From 664c6c2b7c99402c27be7b79d916443db41578da Mon Sep 17 00:00:00 2001 From: LPW Date: Fri, 19 Dec 2025 17:24:48 -0500 Subject: [PATCH] Pending detection, FX metadata, Pending UI badge. (#374) * - Add support for `SIMPLEFIN_INCLUDE_PENDING` to control pending behavior via ENV. - Enhance debug logging for SimpleFin API requests and raw payloads. - Refine pending flag handling in `SimplefinEntry::Processor` based on provider data and inferred conditions. - Improve FX metadata processing for transactions with currency mismatches. - Add new tests for pending detection, FX metadata, and edge cases involving `posted` values. - Add pending indicator UI to transaction view. * Document pending transaction detection, storage, and UI behavior for SimpleFIN and Plaid integrations. Add debug flags for troubleshooting. * Add `pending?` method to `Transaction` model, refactor UI indicator, and centralize SimpleFIN configuration - Introduced `pending?` method in `Transaction` for unified pending state detection. - Refactored transaction pending indicator in the UI to use `pending?` method. - Centralized SimpleFIN configuration in initializer with ENV-backed toggles. - Updated tests for `pending?` behavior and clarified docs for pending detection logic * Add SimpleFIN debug and runtime flags to `.env.local.example` and `.env.test.example` - Introduced `SIMPLEFIN_INCLUDE_PENDING` and `SIMPLEFIN_DEBUG_RAW` flags for controlling pending behavior and debugging. - Updated example environment files with descriptions for new configuration options. * Normalize formatting for `SIMPLEFIN_INCLUDE_PENDING` and `SIMPLEFIN_DEBUG_RAW` flags in `.env.local.example` and `.env.test.example`. --------- Co-authored-by: Josh Waldrep --- .env.local.example | 7 ++ .env.test.example | 7 ++ AGENTS.md | 26 ++++++ CLAUDE.md | 17 ++++ app/models/plaid_entry/processor.rb | 7 +- app/models/provider/simplefin.rb | 5 ++ app/models/simplefin_entry/processor.rb | 20 +++++ app/models/simplefin_item/importer.rb | 13 ++- app/models/transaction.rb | 8 ++ app/views/transactions/_transaction.html.erb | 8 ++ config/initializers/simplefin.rb | 7 ++ test/models/simplefin_entry/processor_test.rb | 86 +++++++++++++++++++ test/models/transaction_test.rb | 21 +++++ 13 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 config/initializers/simplefin.rb create mode 100644 test/models/transaction_test.rb diff --git a/.env.local.example b/.env.local.example index 7aacba4a2..4ce070d7d 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,6 +1,13 @@ # To enable / disable self-hosting features. SELF_HOSTED = true +# SimpleFIN runtime flags (default-off) +# Accepted truthy values: 1, true, yes, on +# SIMPLEFIN_DEBUG_RAW: when truthy, logs the raw payload returned by SimpleFIN (debug-only; can be noisy) +SIMPLEFIN_DEBUG_RAW=false +# SIMPLEFIN_INCLUDE_PENDING: when truthy, forces `pending=1` on SimpleFIN fetches when caller doesn't specify `pending:` +SIMPLEFIN_INCLUDE_PENDING=false + # Controls onboarding flow (valid: open, closed, invite_only) ONBOARDING_STATE = open diff --git a/.env.test.example b/.env.test.example index 57d67d064..098a4422f 100644 --- a/.env.test.example +++ b/.env.test.example @@ -1,5 +1,12 @@ SELF_HOSTED=false +# SimpleFIN runtime flags (default-off) +# Accepted truthy values: 1, true, yes, on +# SIMPLEFIN_DEBUG_RAW: when truthy, logs the raw payload returned by SimpleFIN (debug-only; can be noisy) +SIMPLEFIN_DEBUG_RAW=false +# SIMPLEFIN_INCLUDE_PENDING: when truthy, forces `pending=1` on SimpleFIN fetches when caller doesn't specify `pending:` +SIMPLEFIN_INCLUDE_PENDING=false + # Controls onboarding flow (valid: open, closed, invite_only) ONBOARDING_STATE=open diff --git a/AGENTS.md b/AGENTS.md index 9788d9212..1cf482418 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,3 +33,29 @@ ## Security & Configuration Tips - Never commit secrets. Start from `.env.local.example`; use `.env.local` for development only. - Run `bin/brakeman` before major PRs. Prefer environment variables over hard-coded values. + +## Providers: Pending Transactions and FX Metadata (SimpleFIN/Plaid) + +- Pending detection + - SimpleFIN: pending when provider sends `pending: true`, or when `posted` is blank/0 and `transacted_at` is present. + - Plaid: pending when Plaid sends `pending: true` (stored at `transaction.extra["plaid"]["pending"]` for bank/credit transactions imported via `PlaidEntry::Processor`). +- Storage (extras) + - Provider metadata lives on `Transaction#extra`, namespaced (e.g., `extra["simplefin"]["pending"]`). + - SimpleFIN FX: `extra["simplefin"]["fx_from"]`, `extra["simplefin"]["fx_date"]`. +- UI + - Shows a small “Pending” badge when `transaction.pending?` is true. +- Variability + - Some providers don’t expose pendings; in that case nothing is shown. +- Configuration (default-off) + - SimpleFIN runtime toggles live in `config/initializers/simplefin.rb` via `Rails.configuration.x.simplefin.*`. + - ENV-backed keys: + - `SIMPLEFIN_INCLUDE_PENDING=1` (forces `pending=1` on SimpleFIN fetches when caller didn’t specify a `pending:` arg) + - `SIMPLEFIN_DEBUG_RAW=1` (logs raw payload returned by SimpleFIN) + +### Provider support notes + +- SimpleFIN: supports pending + FX metadata; stored under `extra["simplefin"]`. +- Plaid: supports pending when the upstream Plaid payload includes `pending: true`; stored under `extra["plaid"]`. +- Plaid investments: investment transactions currently do not store pending metadata. +- Lunchflow: does not currently store pending metadata. +- Manual/CSV imports: no pending concept. diff --git a/CLAUDE.md b/CLAUDE.md index 575ba82c5..f543d5bd9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,6 +94,23 @@ Two primary data ingestion methods: - Supports transaction and balance imports - Custom field mapping with transformation rules +### Provider Integrations: Pending Transactions and FX (SimpleFIN/Plaid) + +- Detection + - SimpleFIN: pending via `pending: true` or `posted` blank/0 + `transacted_at`. + - Plaid: pending via Plaid `pending: true` (stored at `extra["plaid"]["pending"]` for bank/credit transactions imported via `PlaidEntry::Processor`). +- Storage: provider data on `Transaction#extra` (e.g., `extra["simplefin"]["pending"]`; FX uses `fx_from`, `fx_date`). +- UI: “Pending” badge when `transaction.pending?` is true; no badge if provider omits pendings. +- Configuration (default-off) + - Centralized in `config/initializers/simplefin.rb` via `Rails.configuration.x.simplefin.*`. + - ENV-backed keys: `SIMPLEFIN_INCLUDE_PENDING=1`, `SIMPLEFIN_DEBUG_RAW=1`. + +Provider support notes: +- SimpleFIN: supports pending + FX metadata (stored under `extra["simplefin"]`). +- Plaid: supports pending when the upstream Plaid payload includes `pending: true` (stored under `extra["plaid"]`). +- Plaid investments: investment transactions currently do not store pending metadata. +- Lunchflow: does not currently store pending metadata. + ### Background Processing Sidekiq handles asynchronous tasks: - Account syncing (`SyncJob`) diff --git a/app/models/plaid_entry/processor.rb b/app/models/plaid_entry/processor.rb index 13d35a93e..e3bc8c74c 100644 --- a/app/models/plaid_entry/processor.rb +++ b/app/models/plaid_entry/processor.rb @@ -15,7 +15,12 @@ class PlaidEntry::Processor name: name, source: "plaid", category_id: matched_category&.id, - merchant: merchant + merchant: merchant, + extra: { + plaid: { + pending: plaid_transaction["pending"] + } + } ) end diff --git a/app/models/provider/simplefin.rb b/app/models/provider/simplefin.rb index 93d721514..8fff2a297 100644 --- a/app/models/provider/simplefin.rb +++ b/app/models/provider/simplefin.rb @@ -1,4 +1,9 @@ class Provider::Simplefin + # Pending: some institutions do not return pending transactions even with `pending=1`. + # This is provider variability (not a bug). For troubleshooting, you can set + # `SIMPLEFIN_INCLUDE_PENDING=1` and/or `SIMPLEFIN_DEBUG_RAW=1` (both default-off). + # These are centralized in `Rails.configuration.x.simplefin.*` via + # `config/initializers/simplefin.rb`. include HTTParty headers "User-Agent" => "Sure Finance SimpleFin Client" diff --git a/app/models/simplefin_entry/processor.rb b/app/models/simplefin_entry/processor.rb index d176d4857..de0802c90 100644 --- a/app/models/simplefin_entry/processor.rb +++ b/app/models/simplefin_entry/processor.rb @@ -34,6 +34,24 @@ class SimplefinEntry::Processor # Include provider-supplied extra hash if present sf["extra"] = data[:extra] if data[:extra].is_a?(Hash) + # Pending detection: honor provider flag or infer from missing/zero posted with present transacted_at + posted_val = data[:posted] + posted_missing = posted_val.blank? || posted_val == 0 || posted_val == "0" + if ActiveModel::Type::Boolean.new.cast(data[:pending]) || (posted_missing && data[:transacted_at].present?) + sf["pending"] = true + Rails.logger.debug("SimpleFIN: flagged pending transaction #{external_id}") + end + + # FX metadata: when tx currency differs from account currency + tx_currency = parse_currency(data[:currency]) + acct_currency = account.currency + if tx_currency.present? && acct_currency.present? && tx_currency != acct_currency + sf["fx_from"] = tx_currency + # Prefer transacted_at for fx date, fallback to posted + fx_d = transacted_date || posted_date + sf["fx_date"] = fx_d&.to_s + end + return nil if sf.empty? { "simplefin" => sf } end @@ -124,6 +142,8 @@ class SimplefinEntry::Processor def posted_date val = data[:posted] + # Treat 0 / "0" as missing to avoid Unix epoch 1970-01-01 for pendings + return nil if val == 0 || val == "0" Simplefin::DateUtils.parse_provider_date(val) end diff --git a/app/models/simplefin_item/importer.rb b/app/models/simplefin_item/importer.rb index 18a811471..d7d7e7226 100644 --- a/app/models/simplefin_item/importer.rb +++ b/app/models/simplefin_item/importer.rb @@ -403,6 +403,10 @@ class SimplefinItem::Importer # Returns a Hash payload with keys like :accounts, or nil when an error is # handled internally via `handle_errors`. def fetch_accounts_data(start_date:, end_date: nil, pending: nil) + # Determine whether to include pending based on explicit arg or global config. + # `Rails.configuration.x.simplefin.include_pending` is ENV-backed. + effective_pending = pending.nil? ? Rails.configuration.x.simplefin.include_pending : pending + # Debug logging to track exactly what's being sent to SimpleFin API start_str = start_date.respond_to?(:strftime) ? start_date.strftime("%Y-%m-%d") : "none" end_str = end_date.respond_to?(:strftime) ? end_date.strftime("%Y-%m-%d") : "current" @@ -411,7 +415,7 @@ class SimplefinItem::Importer else "unknown" end - Rails.logger.info "SimplefinItem::Importer - API Request: #{start_str} to #{end_str} (#{days_requested} days)" + Rails.logger.info "SimplefinItem::Importer - API Request: #{start_str} to #{end_str} (#{days_requested} days) pending=#{effective_pending ? 1 : 0}" begin # Track API request count for quota awareness @@ -420,7 +424,7 @@ class SimplefinItem::Importer simplefin_item.access_url, start_date: start_date, end_date: end_date, - pending: pending + pending: effective_pending ) # Soft warning when approaching SimpleFin daily refresh guidance if stats["api_requests"].to_i >= 20 @@ -436,6 +440,11 @@ class SimplefinItem::Importer end end + # Optional raw payload debug logging (guarded by ENV to avoid spam) + if Rails.configuration.x.simplefin.debug_raw + Rails.logger.debug("SimpleFIN raw: #{accounts_data.inspect}") + end + # Handle errors if present in response if accounts_data[:errors] && accounts_data[:errors].any? if accounts_data[:accounts].to_a.any? diff --git a/app/models/transaction.rb b/app/models/transaction.rb index c2eb468b2..368fa6453 100644 --- a/app/models/transaction.rb +++ b/app/models/transaction.rb @@ -31,4 +31,12 @@ class Transaction < ApplicationRecord update!(category: category) end + + def pending? + extra_data = extra.is_a?(Hash) ? extra : {} + ActiveModel::Type::Boolean.new.cast(extra_data.dig("simplefin", "pending")) || + ActiveModel::Type::Boolean.new.cast(extra_data.dig("plaid", "pending")) + rescue + false + end end diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index 9b87c7aff..ea95c4d5f 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -65,6 +65,14 @@ <% end %> + <%# Pending indicator %> + <% if transaction.pending? %> + + <%= icon "clock", size: "sm", color: "current" %> + Pending + + <% end %> + <% if transaction.transfer.present? %> <%= render "transactions/transfer_match", transaction: transaction %> <% end %> diff --git a/config/initializers/simplefin.rb b/config/initializers/simplefin.rb new file mode 100644 index 000000000..07de9aa77 --- /dev/null +++ b/config/initializers/simplefin.rb @@ -0,0 +1,7 @@ +Rails.application.configure do + truthy = %w[1 true yes on] + + config.x.simplefin ||= ActiveSupport::OrderedOptions.new + config.x.simplefin.include_pending = truthy.include?(ENV["SIMPLEFIN_INCLUDE_PENDING"].to_s.strip.downcase) + config.x.simplefin.debug_raw = truthy.include?(ENV["SIMPLEFIN_DEBUG_RAW"].to_s.strip.downcase) +end diff --git a/test/models/simplefin_entry/processor_test.rb b/test/models/simplefin_entry/processor_test.rb index defc48da1..abea68a91 100644 --- a/test/models/simplefin_entry/processor_test.rb +++ b/test/models/simplefin_entry/processor_test.rb @@ -53,4 +53,90 @@ class SimplefinEntry::ProcessorTest < ActiveSupport::TestCase assert_equal "Order #1234", sf["description"] assert_equal({ "category" => "restaurants", "check_number" => nil }, sf["extra"]) end + test "flags pending transaction when posted is nil and transacted_at present" do + tx = { + id: "tx_pending_1", + amount: "-20.00", + currency: "USD", + payee: "Coffee Shop", + description: "Latte", + memo: "Morning run", + posted: nil, + transacted_at: (Date.today - 3).to_s + } + + SimplefinEntry::Processor.new(tx, simplefin_account: @simplefin_account).process + + entry = @account.entries.find_by!(external_id: "simplefin_tx_pending_1", source: "simplefin") + sf = entry.transaction.extra.fetch("simplefin") + + assert_equal true, sf["pending"], "expected pending flag to be true" + end + + test "captures FX metadata when tx currency differs from account currency" do + # Account is USD from setup; use EUR for tx + t_date = (Date.today - 5) + p_date = Date.today + + tx = { + id: "tx_fx_1", + amount: "-42.00", + currency: "EUR", + payee: "Boulangerie", + description: "Croissant", + posted: p_date.to_s, + transacted_at: t_date.to_s + } + + SimplefinEntry::Processor.new(tx, simplefin_account: @simplefin_account).process + + entry = @account.entries.find_by!(external_id: "simplefin_tx_fx_1", source: "simplefin") + sf = entry.transaction.extra.fetch("simplefin") + + assert_equal "EUR", sf["fx_from"] + assert_equal t_date.to_s, sf["fx_date"], "fx_date should prefer transacted_at" + end + test "flags pending when provider pending flag is true (even if posted provided)" do + tx = { + id: "tx_pending_flag_1", + amount: "-9.99", + currency: "USD", + payee: "Test Store", + description: "Auth", + memo: "", + posted: Date.today.to_s, # provider says pending=true should still flag + transacted_at: (Date.today - 1).to_s, + pending: true + } + + SimplefinEntry::Processor.new(tx, simplefin_account: @simplefin_account).process + + entry = @account.entries.find_by!(external_id: "simplefin_tx_pending_flag_1", source: "simplefin") + sf = entry.transaction.extra.fetch("simplefin") + assert_equal true, sf["pending"], "expected pending flag to be true when provider sends pending=true" + end + + test "posted==0 treated as missing, entry uses transacted_at date and flags pending" do + # Simulate provider sending epoch-like zeros for posted and an integer transacted_at + t_epoch = (Date.today - 2).to_time.to_i + tx = { + id: "tx_pending_zero_posted_1", + amount: "-6.48", + currency: "USD", + payee: "Dunkin'", + description: "DUNKIN #358863", + memo: "", + posted: 0, + transacted_at: t_epoch, + pending: true + } + + SimplefinEntry::Processor.new(tx, simplefin_account: @simplefin_account).process + + entry = @account.entries.find_by!(external_id: "simplefin_tx_pending_zero_posted_1", source: "simplefin") + # For depository accounts, processor prefers posted, then transacted; posted==0 should be treated as missing + assert_equal Time.at(t_epoch).to_date, entry.date, "expected entry.date to use transacted_at when posted==0" + sf = entry.transaction.extra.fetch("simplefin") + assert_equal true, sf["pending"], "expected pending flag to be true when posted==0 and/or pending=true" + end end diff --git a/test/models/transaction_test.rb b/test/models/transaction_test.rb new file mode 100644 index 000000000..7b5f2faeb --- /dev/null +++ b/test/models/transaction_test.rb @@ -0,0 +1,21 @@ +require "test_helper" + +class TransactionTest < ActiveSupport::TestCase + test "pending? is true when extra.simplefin.pending is truthy" do + transaction = Transaction.new(extra: { "simplefin" => { "pending" => true } }) + + assert transaction.pending? + end + + test "pending? is true when extra.plaid.pending is truthy" do + transaction = Transaction.new(extra: { "plaid" => { "pending" => "true" } }) + + assert transaction.pending? + end + + test "pending? is false when no provider pending metadata is present" do + transaction = Transaction.new(extra: { "plaid" => { "pending" => false } }) + + assert_not transaction.pending? + end +end