Files
sure/app/models/simplefin_item.rb
LPW c12c585a0e Harden SimpleFin sync: retries, safer imports, manual relinking, and data-quality reconciliation (#544)
* Add tests and enhance logic for SimpleFin account synchronization and reconciliation

- Added retry logic with exponential backoff for network errors in `Provider::Simplefin`.
- Introduced tests to verify retry functionality and error handling for rate-limit, server errors, and stale data.
- Updated `SimplefinItem` to detect stale sync status and reconciliation issues.
- Enhanced UI to display stale sync warnings and data integrity notices.
- Improved SimpleFin account matching during updates with multi-tier strategy (ID, fingerprint, fuzzy match).
- Added transaction reconciliation logic to detect data gaps, transaction count drops, and duplicate transaction IDs.

* Introduce `SimplefinConnectionUpdateJob` for asynchronous SimpleFin connection updates

- Moved SimpleFin connection update logic to `SimplefinConnectionUpdateJob` to improve response times by offloading network retries, data fetching, and reconciliation tasks.
- Enhanced SimpleFin account matching with a multi-tier strategy (ID, fingerprint, fuzzy name match).
- Added retry logic and bounded latency for token claim requests in `Provider::Simplefin`.
- Updated tests to cover the new job flow and ensure correct account reconciliation during updates.

* Remove unused SimpleFin account matching logic and improve error handling in `SimplefinConnectionUpdateJob`

- Deleted the multi-tier account matching logic from `SimplefinItemsController` as it is no longer used.
- Enhanced error handling in `SimplefinConnectionUpdateJob` to gracefully handle import failures, ensuring orphaned items can be manually resolved.
- Updated job flow to conditionally set item status based on the success of import operations.

* Fix SimpleFin sync: check both legacy FK and AccountProvider for linked accounts

* Add crypto, checking, savings, and cash account detection; refine subtype selection and linking

- Enhanced `Simplefin::AccountTypeMapper` to include detection for crypto, checking, savings, and standalone cash accounts.
- Improved subtype selection UI with validation and warning indicators for missing selections.
- Updated SimpleFin account linking to handle both legacy FK and `AccountProvider` associations consistently.
- Refined job flow and importer logic for better handling of linked accounts and subtype inference.

* Improve `SimplefinConnectionUpdateJob` and holdings processing logic

- Fixed race condition in `SimplefinConnectionUpdateJob` by moving `destroy_later` calls outside of transactions.
- Updated fuzzy name match logic to use Levenshtein distance for better accuracy.
- Enhanced synthetic ticker generation in holdings processor with hash suffix for uniqueness.

* Refine SimpleFin entry processing logic and ensure `extra` data persistence

- Simplified pending flag determination to rely solely on provider-supplied values.
- Fixed potential stale values in `extra` by ensuring deep merge overwrite with `entry.transaction.save!`.

* Replace hardcoded fallback transaction description with localized string

* Refine pending flag logic in SimpleFin processor tests

- Adjust test to prevent falsely inferring pending status from missing posted dates.
- Ensure provider explicitly sets pending flag for transactions.

* Add `has_many :holdings` association to `AccountProvider` with `dependent: :nullify`

---------

Co-authored-by: Josh Waldrep <joshua.waldrep5+github@gmail.com>
2026-01-05 22:11:47 +01:00

256 lines
8.0 KiB
Ruby

class SimplefinItem < ApplicationRecord
include Syncable, Provided
include SimplefinItem::Unlinking
enum :status, { good: "good", requires_update: "requires_update" }, default: :good
# Virtual attribute for the setup token form field
attr_accessor :setup_token
# Helper to detect if ActiveRecord Encryption is configured for this app
def self.encryption_ready?
creds_ready = Rails.application.credentials.active_record_encryption.present?
env_ready = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"].present? &&
ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"].present? &&
ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"].present?
creds_ready || env_ready
end
# Encrypt sensitive credentials if ActiveRecord encryption is configured (credentials OR env vars)
if encryption_ready?
encrypts :access_url, deterministic: true
end
validates :name, presence: true
validates :access_url, presence: true, on: :create
before_destroy :remove_simplefin_item
belongs_to :family
has_one_attached :logo
has_many :simplefin_accounts, dependent: :destroy
has_many :legacy_accounts, through: :simplefin_accounts, source: :account
scope :active, -> { where(scheduled_for_deletion: false) }
scope :ordered, -> { order(created_at: :desc) }
scope :needs_update, -> { where(status: :requires_update) }
# Get accounts from both new and legacy systems
def accounts
# Preload associations to avoid N+1 queries
simplefin_accounts
.includes(:account, account_provider: :account)
.map(&:current_account)
.compact
.uniq
end
def destroy_later
update!(scheduled_for_deletion: true)
DestroyJob.perform_later(self)
end
def import_latest_simplefin_data(sync: nil)
SimplefinItem::Importer.new(self, simplefin_provider: simplefin_provider, sync: sync).import
end
def process_accounts
# Process accounts linked via BOTH legacy FK and AccountProvider
simplefin_accounts.includes(:account, account_provider: :account).each do |simplefin_account|
# Only process if there's a linked account (via either system)
next unless simplefin_account.current_account.present?
SimplefinAccount::Processor.new(simplefin_account).process
end
end
def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil)
accounts.each do |account|
account.sync_later(
parent_sync: parent_sync,
window_start_date: window_start_date,
window_end_date: window_end_date
)
end
end
def upsert_simplefin_snapshot!(accounts_snapshot)
assign_attributes(
raw_payload: accounts_snapshot,
)
# Do not populate item-level institution fields from account data.
# Institution metadata belongs to each simplefin_account (in org_data).
save!
end
def upsert_institution_data!(org_data)
org = org_data.to_h.with_indifferent_access
url = org[:url] || org[:"sfin-url"]
domain = org[:domain]
# Derive domain from URL if missing
if domain.blank? && url.present?
begin
domain = URI.parse(url).host&.gsub(/^www\./, "")
rescue URI::InvalidURIError
Rails.logger.warn("Invalid SimpleFin institution URL: #{url.inspect}")
end
end
assign_attributes(
institution_id: org[:id],
institution_name: org[:name],
institution_domain: domain,
institution_url: url,
raw_institution_payload: org_data
)
end
def has_completed_initial_setup?
# Setup is complete if we have any linked accounts
accounts.any?
end
def sync_status_summary
latest = latest_sync
return nil unless latest
# If sync has statistics, use them
if latest.sync_stats.present?
stats = latest.sync_stats
total = stats["total_accounts"] || 0
linked = stats["linked_accounts"] || 0
unlinked = stats["unlinked_accounts"] || 0
if total == 0
"No accounts found"
elsif unlinked == 0
"#{linked} #{'account'.pluralize(linked)} synced"
else
"#{linked} synced, #{unlinked} need setup"
end
else
# Fallback to current account counts
total_accounts = simplefin_accounts.count
linked_count = accounts.count
unlinked_count = total_accounts - linked_count
if total_accounts == 0
"No accounts found"
elsif unlinked_count == 0
"#{linked_count} #{'account'.pluralize(linked_count)} synced"
else
"#{linked_count} synced, #{unlinked_count} need setup"
end
end
end
def institution_display_name
# Try to get institution name from stored metadata
institution_name.presence || institution_domain.presence || name
end
def connected_institutions
# Get unique institutions from all accounts
simplefin_accounts.includes(:account)
.where.not(org_data: nil)
.map { |acc| acc.org_data }
.uniq { |org| org["domain"] || org["name"] }
end
def institution_summary
institutions = connected_institutions
case institutions.count
when 0
"No institutions connected"
when 1
institutions.first["name"] || institutions.first["domain"] || "1 institution"
else
"#{institutions.count} institutions"
end
end
# Detect a recent rate-limited sync and return a friendly message, else nil
def rate_limited_message
latest = latest_sync
return nil unless latest
# Some Sync records may not have a status_text column; guard with respond_to?
parts = []
parts << latest.error if latest.respond_to?(:error)
parts << latest.status_text if latest.respond_to?(:status_text)
msg = parts.compact.join("")
return nil if msg.blank?
down = msg.downcase
if down.include?("make fewer requests") || down.include?("only refreshed once every 24 hours") || down.include?("rate limit")
"You've hit SimpleFin's daily refresh limit. Please try again after the bridge refreshes (up to 24 hours)."
else
nil
end
end
# Detect if sync data appears stale (no new transactions for extended period)
# Returns a hash with :stale (boolean) and :message (string) if stale
def stale_sync_status
return { stale: false } unless last_synced_at.present?
# Check if last sync was more than 3 days ago
days_since_sync = (Date.current - last_synced_at.to_date).to_i
if days_since_sync > 3
return {
stale: true,
days_since_sync: days_since_sync,
message: "Last successful sync was #{days_since_sync} days ago. Your SimpleFin connection may need attention."
}
end
# Check if linked accounts have recent transactions
linked_accounts = accounts
return { stale: false } if linked_accounts.empty?
# Find the most recent transaction date across all linked accounts
latest_transaction_date = Entry.where(account_id: linked_accounts.map(&:id))
.where(entryable_type: "Transaction")
.maximum(:date)
if latest_transaction_date.present?
days_since_transaction = (Date.current - latest_transaction_date).to_i
if days_since_transaction > 14
return {
stale: true,
days_since_transaction: days_since_transaction,
message: "No new transactions in #{days_since_transaction} days. Check your SimpleFin dashboard to ensure your bank connections are active."
}
end
end
{ stale: false }
end
# Check if the SimpleFin connection needs user attention
def needs_attention?
requires_update? || stale_sync_status[:stale] || pending_account_setup?
end
# Get a summary of issues requiring attention
def attention_summary
issues = []
issues << "Connection needs update" if requires_update?
issues << stale_sync_status[:message] if stale_sync_status[:stale]
issues << "Accounts need setup" if pending_account_setup?
issues
end
private
def remove_simplefin_item
# SimpleFin doesn't require server-side cleanup like Plaid
# The access URL just becomes inactive
end
end