diff --git a/app/controllers/simplefin_items_controller.rb b/app/controllers/simplefin_items_controller.rb index 8e626859f..3b2696ed3 100644 --- a/app/controllers/simplefin_items_controller.rb +++ b/app/controllers/simplefin_items_controller.rb @@ -1,5 +1,5 @@ class SimplefinItemsController < ApplicationController - before_action :set_simplefin_item, only: [ :show, :destroy, :sync, :setup_accounts, :complete_account_setup ] + before_action :set_simplefin_item, only: [ :show, :edit, :update, :destroy, :sync, :setup_accounts, :complete_account_setup ] def index @simplefin_items = Current.family.simplefin_items.active.ordered @@ -9,6 +9,64 @@ class SimplefinItemsController < ApplicationController def show end + def edit + # For SimpleFin, editing means providing a new setup token to replace expired access + @simplefin_item.setup_token = nil # Clear any existing setup token + end + + def update + setup_token = simplefin_params[:setup_token] + + return render_error(t(".errors.blank_token"), context: :edit) if setup_token.blank? + + begin + # Create new SimpleFin item data with updated token + updated_item = Current.family.create_simplefin_item!( + setup_token: setup_token, + item_name: @simplefin_item.name + ) + + # Ensure new simplefin_accounts are created & have account_id set + updated_item.import_latest_simplefin_data + + # Transfer accounts from old item to new item + ActiveRecord::Base.transaction do + @simplefin_item.simplefin_accounts.each do |old_account| + if old_account.account.present? + # Find matching account in new item by account_id + new_account = updated_item.simplefin_accounts.find_by(account_id: old_account.account_id) + if new_account + # Transfer the account directly to the new SimpleFin account + # This will automatically break the old association + old_account.account.update!(simplefin_account_id: new_account.id) + end + end + end + + # Mark old item for deletion + @simplefin_item.destroy_later + end + + # Clear any requires_update status on new item + updated_item.update!(status: :good) + + redirect_to accounts_path, notice: t(".success") + rescue ArgumentError, URI::InvalidURIError + render_error(t(".errors.invalid_token"), setup_token, context: :edit) + rescue Provider::Simplefin::SimplefinError => e + error_message = case e.error_type + when :token_compromised + t(".errors.token_compromised") + else + t(".errors.update_failed", message: e.message) + end + render_error(error_message, setup_token, context: :edit) + rescue => e + Rails.logger.error("SimpleFin connection update error: #{e.message}") + render_error(t(".errors.unexpected"), setup_token, context: :edit) + end + end + def new @simplefin_item = Current.family.simplefin_items.build end @@ -16,7 +74,7 @@ class SimplefinItemsController < ApplicationController def create setup_token = simplefin_params[:setup_token] - return render_error("Please enter a SimpleFin setup token.") if setup_token.blank? + return render_error(t(".errors.blank_token")) if setup_token.blank? begin @simplefin_item = Current.family.create_simplefin_item!( @@ -24,31 +82,37 @@ class SimplefinItemsController < ApplicationController item_name: "SimpleFin Connection" ) - redirect_to simplefin_items_path, notice: "SimpleFin connection added successfully! Your accounts will appear shortly as they sync in the background." + redirect_to accounts_path, notice: t(".success") rescue ArgumentError, URI::InvalidURIError - render_error("Invalid setup token. Please check that you copied the complete token from SimpleFin Bridge.", setup_token) + render_error(t(".errors.invalid_token"), setup_token) rescue Provider::Simplefin::SimplefinError => e error_message = case e.error_type when :token_compromised - "The setup token may be compromised, expired, or already used. Please create a new one." + t(".errors.token_compromised") else - "Failed to connect: #{e.message}" + t(".errors.create_failed", message: e.message) end render_error(error_message, setup_token) rescue => e Rails.logger.error("SimpleFin connection error: #{e.message}") - render_error("An unexpected error occurred. Please try again or contact support.", setup_token) + render_error(t(".errors.unexpected"), setup_token) end end def destroy @simplefin_item.destroy_later - redirect_to simplefin_items_path, notice: "SimpleFin connection will be removed" + redirect_to accounts_path, notice: t(".success") end def sync - @simplefin_item.sync_later - redirect_to simplefin_item_path(@simplefin_item), notice: "Sync started" + unless @simplefin_item.syncing? + @simplefin_item.sync_later + end + + respond_to do |format| + format.html { redirect_back_or_to accounts_path } + format.json { head :ok } + end end def setup_accounts @@ -58,8 +122,7 @@ class SimplefinItemsController < ApplicationController [ "Credit Card", "CreditCard" ], [ "Investment Account", "Investment" ], [ "Loan or Mortgage", "Loan" ], - [ "Other Asset", "OtherAsset" ], - [ "Skip - don't add", "Skip" ] + [ "Other Asset", "OtherAsset" ] ] # Subtype options for each account type @@ -93,10 +156,12 @@ class SimplefinItemsController < ApplicationController account_types = params[:account_types] || {} account_subtypes = params[:account_subtypes] || {} - account_types.each do |simplefin_account_id, selected_type| - # Skip accounts that the user chose not to add - next if selected_type == "Skip" + # Update sync start date from form + if params[:sync_start_date].present? + @simplefin_item.update!(sync_start_date: params[:sync_start_date]) + end + account_types.each do |simplefin_account_id, selected_type| simplefin_account = @simplefin_item.simplefin_accounts.find(simplefin_account_id) selected_subtype = account_subtypes[simplefin_account_id] @@ -115,10 +180,10 @@ class SimplefinItemsController < ApplicationController # Clear pending status and mark as complete @simplefin_item.update!(pending_account_setup: false) - # Schedule account syncs for the newly created accounts - @simplefin_item.schedule_account_syncs + # Trigger a sync to process the imported SimpleFin data (transactions and holdings) + @simplefin_item.sync_later - redirect_to simplefin_items_path, notice: "SimpleFin accounts have been set up successfully!" + redirect_to accounts_path, notice: t(".success") end private @@ -128,12 +193,17 @@ class SimplefinItemsController < ApplicationController end def simplefin_params - params.require(:simplefin_item).permit(:setup_token) + params.require(:simplefin_item).permit(:setup_token, :sync_start_date) end - def render_error(message, setup_token = nil) - @simplefin_item = Current.family.simplefin_items.build(setup_token: setup_token) + def render_error(message, setup_token = nil, context: :new) + if context == :edit + # Keep the persisted record and assign the token for re-render + @simplefin_item.setup_token = setup_token if @simplefin_item + else + @simplefin_item = Current.family.simplefin_items.build(setup_token: setup_token) + end @error_message = message - render :new, status: :unprocessable_entity + render context, status: :unprocessable_entity end end diff --git a/app/javascript/controllers/loading_button_controller.js b/app/javascript/controllers/loading_button_controller.js new file mode 100644 index 000000000..7a1e03d8d --- /dev/null +++ b/app/javascript/controllers/loading_button_controller.js @@ -0,0 +1,21 @@ +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["button"] + static values = { loadingText: String } + + showLoading() { + // Don't prevent form submission, just show loading state + if (this.hasButtonTarget) { + this.buttonTarget.disabled = true + this.buttonTarget.setAttribute("aria-disabled", "true") + this.buttonTarget.setAttribute("aria-busy", "true") + this.buttonTarget.innerHTML = ` + + + ${this.loadingTextValue || 'Loading...'} + + ` + } + } +} \ No newline at end of file diff --git a/app/models/account.rb b/app/models/account.rb index e9a7e9a34..4c2e7eb06 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -85,10 +85,28 @@ class Account < ApplicationRecord balance = balance.abs end + # Calculate cash balance correctly for investment accounts + cash_balance = balance + if account_type == "Investment" + begin + calculator = SimplefinAccount::Investments::BalanceCalculator.new(simplefin_account) + calculated = calculator.cash_balance + cash_balance = calculated unless calculated.nil? + rescue => e + Rails.logger.warn( + "Investment cash_balance calculation failed for " \ + "SimpleFin account #{simplefin_account.id}: #{e.class} - #{e.message}" + ) + # Fallback to zero as suggested + cash_balance = 0 + end + end + attributes = { family: simplefin_account.simplefin_item.family, name: simplefin_account.name, balance: balance, + cash_balance: cash_balance, currency: simplefin_account.currency, accountable_type: account_type, accountable_attributes: build_simplefin_accountable_attributes(simplefin_account, account_type, subtype), diff --git a/app/models/concerns/syncable.rb b/app/models/concerns/syncable.rb index 298aa6207..92b0a3965 100644 --- a/app/models/concerns/syncable.rb +++ b/app/models/concerns/syncable.rb @@ -19,6 +19,11 @@ module Syncable if sync Rails.logger.info("There is an existing sync, expanding window if needed (#{sync.id})") sync.expand_window_if_needed(window_start_date, window_end_date) + + # Update parent relationship if one is provided and sync doesn't already have a parent + if parent_sync && !sync.parent_id + sync.update!(parent: parent_sync) + end else sync = self.syncs.create!( parent: parent_sync, diff --git a/app/models/data_enrichment.rb b/app/models/data_enrichment.rb index df6a0d67f..ead34908f 100644 --- a/app/models/data_enrichment.rb +++ b/app/models/data_enrichment.rb @@ -1,5 +1,5 @@ class DataEnrichment < ApplicationRecord belongs_to :enrichable, polymorphic: true - enum :source, { rule: "rule", plaid: "plaid", synth: "synth", ai: "ai" } + enum :source, { rule: "rule", plaid: "plaid", simplefin: "simplefin", synth: "synth", ai: "ai" } end diff --git a/app/models/holding.rb b/app/models/holding.rb index af659f3e8..90bf4f49c 100644 --- a/app/models/holding.rb +++ b/app/models/holding.rb @@ -8,6 +8,7 @@ class Holding < ApplicationRecord validates :qty, :currency, :date, :price, :amount, presence: true validates :qty, :price, :amount, numericality: { greater_than_or_equal_to: 0 } + validates :external_id, uniqueness: { scope: :account_id }, allow_blank: true scope :chronological, -> { order(:date) } scope :for, ->(security) { where(security_id: security).order(:date) } diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index 19970ce41..abd11beb3 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -99,6 +99,8 @@ class PlaidItem < ApplicationRecord private def remove_plaid_item + return unless plaid_provider.present? + plaid_provider.remove_item(access_token) rescue Plaid::ApiError => e json_response = JSON.parse(e.response_body) diff --git a/app/models/provider/simplefin.rb b/app/models/provider/simplefin.rb index eca6e77fe..93d721514 100644 --- a/app/models/provider/simplefin.rb +++ b/app/models/provider/simplefin.rb @@ -1,8 +1,8 @@ class Provider::Simplefin include HTTParty - headers "User-Agent" => "#{Rails.configuration.x.product_name} Finance SimpleFin Client" - default_options.merge!(verify: true, ssl_verify_mode: :peer) + headers "User-Agent" => "Sure Finance SimpleFin Client" + default_options.merge!(verify: true, ssl_verify_mode: OpenSSL::SSL::VERIFY_PEER, timeout: 120) def initialize end @@ -46,7 +46,15 @@ class Provider::Simplefin # The access URL already contains HTTP Basic Auth credentials - response = HTTParty.get(accounts_url) + begin + response = HTTParty.get(accounts_url) + rescue SocketError, Net::OpenTimeout, Net::ReadTimeout => e + Rails.logger.error "SimpleFin API: GET /accounts failed: #{e.class}: #{e.message}" + raise SimplefinError.new("Exception during GET request: #{e.message}", :request_failed) + rescue => e + Rails.logger.error "SimpleFin API: Unexpected error during GET /accounts: #{e.class}: #{e.message}" + raise SimplefinError.new("Exception during GET request: #{e.message}", :request_failed) + end case response.code diff --git a/app/models/provider_merchant.rb b/app/models/provider_merchant.rb index 7f6779812..2ec1c88ba 100644 --- a/app/models/provider_merchant.rb +++ b/app/models/provider_merchant.rb @@ -1,5 +1,5 @@ class ProviderMerchant < Merchant - enum :source, { plaid: "plaid", synth: "synth", ai: "ai" } + enum :source, { plaid: "plaid", simplefin: "simplefin", synth: "synth", ai: "ai" } validates :name, uniqueness: { scope: [ :source ] } validates :source, presence: true diff --git a/app/models/simplefin_account/investments/balance_calculator.rb b/app/models/simplefin_account/investments/balance_calculator.rb new file mode 100644 index 000000000..1e4780bc7 --- /dev/null +++ b/app/models/simplefin_account/investments/balance_calculator.rb @@ -0,0 +1,49 @@ +# SimpleFin Investment balance calculator +# SimpleFin provides clear balance and holdings data, so calculations are simpler than Plaid +class SimplefinAccount::Investments::BalanceCalculator + def initialize(simplefin_account) + @simplefin_account = simplefin_account + end + + def balance + # SimpleFin provides direct balance data + simplefin_account.current_balance || BigDecimal("0") + end + + def cash_balance + # Calculate cash balance as total balance minus holdings value + total_balance = balance + holdings_value = total_holdings_value + + cash = total_balance - holdings_value + + # Ensure non-negative cash balance + [ cash, BigDecimal("0") ].max + end + + private + attr_reader :simplefin_account + + def total_holdings_value + return BigDecimal("0") unless simplefin_account.raw_payload&.dig("holdings") + + holdings_data = simplefin_account.raw_payload["holdings"] + + holdings_data.sum do |holding| + market_value = holding["market_value"] + begin + case market_value + when String + BigDecimal(market_value) + when Numeric + BigDecimal(market_value.to_s) + else + BigDecimal("0") + end + rescue ArgumentError => e + Rails.logger.warn "SimpleFin holdings market_value parse error for account #{simplefin_account.account_id || simplefin_account.id}: #{e.message} (value: #{market_value.inspect})" + BigDecimal("0") + end + end + end +end diff --git a/app/models/simplefin_account/investments/holdings_processor.rb b/app/models/simplefin_account/investments/holdings_processor.rb new file mode 100644 index 000000000..8cf370860 --- /dev/null +++ b/app/models/simplefin_account/investments/holdings_processor.rb @@ -0,0 +1,121 @@ +class SimplefinAccount::Investments::HoldingsProcessor + def initialize(simplefin_account) + @simplefin_account = simplefin_account + end + + def process + return if holdings_data.empty? + return unless account&.accountable_type == "Investment" + + holdings_data.each do |simplefin_holding| + begin + symbol = simplefin_holding["symbol"] + holding_id = simplefin_holding["id"] + + next unless symbol.present? && holding_id.present? + + security = resolve_security(symbol, simplefin_holding["description"]) + next unless security.present? + + # Use external_id for precise matching + external_id = "simplefin_#{holding_id}" + + # Use the created timestamp as the holding date, fallback to current date + holding_date = parse_holding_date(simplefin_holding["created"]) || Date.current + + holding = account.holdings.find_or_initialize_by( + external_id: external_id + ) do |h| + # Set required fields on initialization + h.security = security + h.date = holding_date + h.currency = simplefin_holding["currency"] || "USD" + end + + # Parse all the data SimpleFin provides + qty = parse_decimal(simplefin_holding["shares"]) + market_value = parse_decimal(simplefin_holding["market_value"]) + cost_basis = parse_decimal(simplefin_holding["cost_basis"]) + + # Calculate price from market_value if we have shares, fallback to purchase_price + price = if qty > 0 && market_value > 0 + market_value / qty + else + parse_decimal(simplefin_holding["purchase_price"]) || 0 + end + + holding.assign_attributes( + security: security, + date: holding_date, + currency: simplefin_holding["currency"] || "USD", + qty: qty, + price: price, + amount: market_value, + cost_basis: cost_basis + ) + + ActiveRecord::Base.transaction do + holding.save! + + # With external_id matching, each holding is uniquely tracked + # No need to delete other holdings since each has its own lifecycle + end + rescue => e + ctx = (defined?(symbol) && symbol.present?) ? " #{symbol}" : "" + Rails.logger.error "Error processing SimpleFin holding#{ctx}: #{e.message}" + end + end + end + + private + attr_reader :simplefin_account + + def account + simplefin_account.account + end + + def holdings_data + # Use the dedicated raw_holdings_payload field + simplefin_account.raw_holdings_payload || [] + end + + def resolve_security(symbol, description) + # Use Security::Resolver to find or create the security + Security::Resolver.new(symbol).resolve + rescue ArgumentError => e + Rails.logger.error "Failed to resolve SimpleFin security #{symbol}: #{e.message}" + nil + end + + def parse_holding_date(created_timestamp) + return nil unless created_timestamp + + case created_timestamp + when Integer + Time.at(created_timestamp).to_date + when String + Date.parse(created_timestamp) + else + nil + end + rescue ArgumentError => e + Rails.logger.error "Failed to parse SimpleFin holding date #{created_timestamp}: #{e.message}" + nil + end + + def parse_decimal(value) + return 0 unless value.present? + + case value + when String + BigDecimal(value) + when Numeric + BigDecimal(value.to_s) + else + BigDecimal("0") + end + rescue ArgumentError => e + Rails.logger.error "Failed to parse SimpleFin decimal value #{value}: #{e.message}" + BigDecimal("0") + end +end diff --git a/app/models/simplefin_account/investments/transactions_processor.rb b/app/models/simplefin_account/investments/transactions_processor.rb new file mode 100644 index 000000000..eb76503d1 --- /dev/null +++ b/app/models/simplefin_account/investments/transactions_processor.rb @@ -0,0 +1,90 @@ +# SimpleFin Investment transactions processor +# Processes investment-specific transactions like trades, dividends, etc. +class SimplefinAccount::Investments::TransactionsProcessor + def initialize(simplefin_account) + @simplefin_account = simplefin_account + end + + def process + return unless simplefin_account.account&.accountable_type == "Investment" + return unless simplefin_account.raw_transactions_payload.present? + + transactions_data = simplefin_account.raw_transactions_payload + + transactions_data.each do |transaction_data| + process_investment_transaction(transaction_data) + end + end + + private + attr_reader :simplefin_account + + def account + simplefin_account.account + end + + def process_investment_transaction(transaction_data) + data = transaction_data.with_indifferent_access + + amount = parse_amount(data[:amount]) + posted_date = parse_date(data[:posted]) + external_id = "simplefin_#{data[:id]}" + + # Check if entry already exists + existing_entry = Entry.find_by(plaid_id: external_id) + + unless existing_entry + # For investment accounts, create as regular transaction + # In the future, we could detect trade patterns and create Trade entries + transaction = Transaction.new(external_id: external_id) + + Entry.create!( + account: account, + name: data[:description] || "Investment transaction", + amount: amount, + date: posted_date, + currency: account.currency, + entryable: transaction, + plaid_id: external_id + ) + end + rescue => e + Rails.logger.error("Failed to process SimpleFin investment transaction #{data[:id]}: #{e.message}") + end + + def parse_amount(amount_value) + parsed_amount = case amount_value + when String + BigDecimal(amount_value) + when Numeric + BigDecimal(amount_value.to_s) + else + BigDecimal("0") + end + + # SimpleFin uses banking convention, Maybe expects opposite + -parsed_amount + rescue ArgumentError => e + Rails.logger.error "Failed to parse SimpleFin investment transaction amount: #{amount_value.inspect} - #{e.message}" + BigDecimal("0") + end + + def parse_date(date_value) + case date_value + when String + Date.parse(date_value) + when Integer, Float + Time.at(date_value).to_date + when Time, DateTime + date_value.to_date + when Date + date_value + else + Rails.logger.error("SimpleFin investment transaction has invalid date value: #{date_value.inspect}") + raise ArgumentError, "Invalid date format: #{date_value.inspect}" + end + rescue ArgumentError, TypeError => e + Rails.logger.error("Failed to parse SimpleFin investment transaction date '#{date_value}': #{e.message}") + raise ArgumentError, "Unable to parse transaction date: #{date_value.inspect}" + end +end diff --git a/app/models/simplefin_account/liabilities/credit_processor.rb b/app/models/simplefin_account/liabilities/credit_processor.rb new file mode 100644 index 000000000..7534fa7ed --- /dev/null +++ b/app/models/simplefin_account/liabilities/credit_processor.rb @@ -0,0 +1,47 @@ +# SimpleFin Credit Card processor for liability-specific features +class SimplefinAccount::Liabilities::CreditProcessor + def initialize(simplefin_account) + @simplefin_account = simplefin_account + end + + def process + return unless simplefin_account.account&.accountable_type == "CreditCard" + + # Update credit card specific attributes if available + update_credit_attributes + end + + private + attr_reader :simplefin_account + + def account + simplefin_account.account + end + + def update_credit_attributes + # SimpleFin provides available_balance which could be credit limit for cards + available_balance = simplefin_account.raw_payload&.dig("available-balance") + + if available_balance.present? && account.accountable.respond_to?(:available_credit=) + credit_limit = parse_decimal(available_balance) + account.accountable.available_credit = credit_limit if credit_limit > 0 + account.accountable.save! + end + end + + def parse_decimal(value) + return 0 unless value.present? + + case value + when String + BigDecimal(value) + when Numeric + BigDecimal(value.to_s) + else + BigDecimal("0") + end + rescue ArgumentError => e + Rails.logger.error "Failed to parse SimpleFin credit value #{value}: #{e.message}" + BigDecimal("0") + end +end diff --git a/app/models/simplefin_account/liabilities/loan_processor.rb b/app/models/simplefin_account/liabilities/loan_processor.rb new file mode 100644 index 000000000..7bb8854de --- /dev/null +++ b/app/models/simplefin_account/liabilities/loan_processor.rb @@ -0,0 +1,28 @@ +# SimpleFin Loan processor for loan-specific features +class SimplefinAccount::Liabilities::LoanProcessor + def initialize(simplefin_account) + @simplefin_account = simplefin_account + end + + def process + return unless simplefin_account.account&.accountable_type == "Loan" + + # Update loan specific attributes if available + update_loan_attributes + end + + private + attr_reader :simplefin_account + + def account + simplefin_account.account + end + + def update_loan_attributes + # I don't know if SimpleFin typically provide detailed loan metadata + # like interest rates, terms, etc. but we can update what's available + + # Balance normalization is handled by SimplefinAccount::Processor.process_account! + # Any other loan-specific attribute updates would go here + end +end diff --git a/app/models/simplefin_account/processor.rb b/app/models/simplefin_account/processor.rb index 571d6bfde..a41575212 100644 --- a/app/models/simplefin_account/processor.rb +++ b/app/models/simplefin_account/processor.rb @@ -5,105 +5,85 @@ class SimplefinAccount::Processor @simplefin_account = simplefin_account end + # Each step represents different SimpleFin data processing + # Processing the account is the first step and if it fails, we halt + # Each subsequent step can fail independently, but we continue processing def process - ensure_account_exists + unless simplefin_account.account.present? + return + end + + process_account! process_transactions + process_investments + process_liabilities end private - def ensure_account_exists - return if simplefin_account.account.present? - + def process_account! # This should not happen in normal flow since accounts are created manually # during setup, but keeping as safety check - Rails.logger.error("SimpleFin account #{simplefin_account.id} has no associated Account - this should not happen after manual setup") + if simplefin_account.account.blank? + Rails.logger.error("SimpleFin account #{simplefin_account.id} has no associated Account - this should not happen after manual setup") + return + end + + # Update account balance and cash balance from latest SimpleFin data + account = simplefin_account.account + balance = simplefin_account.current_balance || simplefin_account.available_balance || 0 + + # SimpleFin returns negative balances for credit cards (liabilities) + # But Maybe expects positive balances for liabilities + if account.accountable_type == "CreditCard" || account.accountable_type == "Loan" + balance = balance.abs + end + + # Calculate cash balance correctly for investment accounts + cash_balance = if account.accountable_type == "Investment" + calculator = SimplefinAccount::Investments::BalanceCalculator.new(simplefin_account) + calculator.cash_balance + else + balance + end + + account.update!( + balance: balance, + cash_balance: cash_balance + ) end def process_transactions - return unless simplefin_account.raw_transactions_payload.present? - - account = simplefin_account.account - transactions_data = simplefin_account.raw_transactions_payload - - transactions_data.each do |transaction_data| - process_transaction(account, transaction_data) - end + SimplefinAccount::Transactions::Processor.new(simplefin_account).process + rescue => e + report_exception(e, "transactions") end - def process_transaction(account, transaction_data) - # Handle both string and symbol keys - data = transaction_data.with_indifferent_access + def process_investments + return unless simplefin_account.account&.accountable_type == "Investment" + SimplefinAccount::Investments::TransactionsProcessor.new(simplefin_account).process + SimplefinAccount::Investments::HoldingsProcessor.new(simplefin_account).process + rescue => e + report_exception(e, "investments") + end - - # Convert SimpleFin transaction to internal Transaction format - amount = parse_amount(data[:amount], account.currency) - posted_date = parse_date(data[:posted]) - - # Use plaid_id field for external ID (works for both Plaid and SimpleFin) - external_id = "simplefin_#{data[:id]}" - - # Check if entry already exists - existing_entry = Entry.find_by(plaid_id: external_id) - - unless existing_entry - # Create the transaction (entryable) - transaction = Transaction.new( - external_id: external_id - ) - - # Create the entry with the transaction - Entry.create!( - account: account, - name: data[:description] || "Unknown transaction", - amount: amount, - date: posted_date, - currency: account.currency, - entryable: transaction, - plaid_id: external_id - ) + def process_liabilities + case simplefin_account.account&.accountable_type + when "CreditCard" + SimplefinAccount::Liabilities::CreditProcessor.new(simplefin_account).process + when "Loan" + SimplefinAccount::Liabilities::LoanProcessor.new(simplefin_account).process end rescue => e - Rails.logger.error("Failed to process SimpleFin transaction #{data[:id]}: #{e.message}") - # Don't fail the entire sync for one bad transaction + report_exception(e, "liabilities") end - def parse_amount(amount_value, currency) - parsed_amount = case amount_value - when String - BigDecimal(amount_value) - when Numeric - BigDecimal(amount_value.to_s) - else - BigDecimal("0") + def report_exception(error, context) + Sentry.capture_exception(error) do |scope| + scope.set_tags( + simplefin_account_id: simplefin_account.id, + context: context + ) end - - # SimpleFin uses banking convention (expenses negative, income positive) - # Sure expects opposite convention (expenses positive, income negative) - # So we negate the amount to convert from SimpleFin to Sure format - -parsed_amount - rescue ArgumentError => e - Rails.logger.error "Failed to parse SimpleFin transaction amount: #{amount_value.inspect} - #{e.message}" - BigDecimal("0") - end - - def parse_date(date_value) - case date_value - when String - Date.parse(date_value) - when Integer, Float - # Unix timestamp - Time.at(date_value).to_date - when Time, DateTime - date_value.to_date - when Date - date_value - else - Rails.logger.error("SimpleFin transaction has invalid date value: #{date_value.inspect}") - raise ArgumentError, "Invalid date format: #{date_value.inspect}" - end - rescue ArgumentError, TypeError => e - Rails.logger.error("Failed to parse SimpleFin transaction date '#{date_value}': #{e.message}") - raise ArgumentError, "Unable to parse transaction date: #{date_value.inspect}" end end diff --git a/app/models/simplefin_account/transactions/merchant_detector.rb b/app/models/simplefin_account/transactions/merchant_detector.rb new file mode 100644 index 000000000..abc2cb893 --- /dev/null +++ b/app/models/simplefin_account/transactions/merchant_detector.rb @@ -0,0 +1,29 @@ +# Detects and creates merchant records from SimpleFin transaction data +# SimpleFin provides clean payee data that works well for merchant identification +class SimplefinAccount::Transactions::MerchantDetector + def initialize(transaction_data) + @transaction_data = transaction_data.with_indifferent_access + end + + def detect_merchant + # SimpleFin provides clean payee data - use it directly + payee = (transaction_data[:payee] || transaction_data["payee"])&.strip + return nil unless payee.present? + + # Find or create merchant record using payee data + ProviderMerchant.find_or_create_by!( + source: "simplefin", + name: payee + ) do |merchant| + merchant.provider_merchant_id = generate_merchant_id(payee) + end + end + + private + attr_reader :transaction_data + + def generate_merchant_id(merchant_name) + # Generate a consistent ID for merchants without explicit IDs + "simplefin_#{Digest::MD5.hexdigest(merchant_name.downcase)}" + end +end diff --git a/app/models/simplefin_account/transactions/processor.rb b/app/models/simplefin_account/transactions/processor.rb new file mode 100644 index 000000000..16caba2ba --- /dev/null +++ b/app/models/simplefin_account/transactions/processor.rb @@ -0,0 +1,42 @@ +class SimplefinAccount::Transactions::Processor + attr_reader :simplefin_account + + def initialize(simplefin_account) + @simplefin_account = simplefin_account + end + + def process + return unless simplefin_account.raw_transactions_payload.present? + + # Each entry is processed inside a transaction, but to avoid locking up the DB when + # there are hundreds or thousands of transactions, we process them individually. + simplefin_account.raw_transactions_payload.each do |transaction_data| + SimplefinEntry::Processor.new( + transaction_data, + simplefin_account: simplefin_account + ).process + rescue => e + Rails.logger.error "Error processing SimpleFin transaction: #{e.message}" + end + end + + private + + def category_matcher + @category_matcher ||= SimplefinAccount::Transactions::CategoryMatcher.new(family_categories) + end + + def family_categories + @family_categories ||= begin + if account.family.categories.none? + account.family.categories.bootstrap! + end + + account.family.categories + end + end + + def account + simplefin_account.account + end +end diff --git a/app/models/simplefin_entry/processor.rb b/app/models/simplefin_entry/processor.rb new file mode 100644 index 000000000..51225520c --- /dev/null +++ b/app/models/simplefin_entry/processor.rb @@ -0,0 +1,121 @@ +class SimplefinEntry::Processor + # simplefin_transaction is the raw hash fetched from SimpleFin API and converted to JSONB + def initialize(simplefin_transaction, simplefin_account:) + @simplefin_transaction = simplefin_transaction + @simplefin_account = simplefin_account + end + + def process + SimplefinAccount.transaction do + entry = account.entries.find_or_initialize_by(plaid_id: external_id) do |e| + e.entryable = Transaction.new + end + + entry.assign_attributes( + amount: amount, + currency: currency, + date: date + ) + + entry.enrich_attribute( + :name, + name, + source: "simplefin" + ) + + # SimpleFin provides no category data - categories will be set by AI or rules + + if merchant + entry.transaction.enrich_attribute( + :merchant_id, + merchant.id, + source: "simplefin" + ) + end + + entry.save! + end + end + + private + attr_reader :simplefin_transaction, :simplefin_account + + def account + simplefin_account.account + end + + def data + @data ||= simplefin_transaction.with_indifferent_access + end + + def external_id + id = data[:id].presence + raise ArgumentError, "SimpleFin transaction missing id: #{data.inspect}" unless id + "simplefin_#{id}" + end + + def name + # Use SimpleFin's rich, clean data to create informative transaction names + payee = data[:payee] + description = data[:description] + + # Combine payee + description when both are present and different + if payee.present? && description.present? && payee != description + "#{payee} - #{description}" + elsif payee.present? + payee + elsif description.present? + description + else + data[:memo] || "Unknown transaction" + end + end + + def amount + parsed_amount = case data[:amount] + when String + BigDecimal(data[:amount]) + when Numeric + BigDecimal(data[:amount].to_s) + else + BigDecimal("0") + end + + # SimpleFin uses banking convention (expenses negative, income positive) + # Maybe expects opposite convention (expenses positive, income negative) + # So we negate the amount to convert from SimpleFin to Maybe format + -parsed_amount + rescue ArgumentError => e + Rails.logger.error "Failed to parse SimpleFin transaction amount: #{data[:amount].inspect} - #{e.message}" + raise + end + + def currency + data[:currency] || account.currency + end + + def date + case data[:posted] + when String + Date.parse(data[:posted]) + when Integer, Float + # Unix timestamp + Time.at(data[:posted]).to_date + when Time, DateTime + data[:posted].to_date + when Date + data[:posted] + else + Rails.logger.error("SimpleFin transaction has invalid date value: #{data[:posted].inspect}") + raise ArgumentError, "Invalid date format: #{data[:posted].inspect}" + end + rescue ArgumentError, TypeError => e + Rails.logger.error("Failed to parse SimpleFin transaction date '#{data[:posted]}': #{e.message}") + raise ArgumentError, "Unable to parse transaction date: #{data[:posted].inspect}" + end + + + def merchant + @merchant ||= SimplefinAccount::Transactions::MerchantDetector.new(data).detect_merchant + end +end diff --git a/app/models/simplefin_item.rb b/app/models/simplefin_item.rb index c896438cc..1e194f76b 100644 --- a/app/models/simplefin_item.rb +++ b/app/models/simplefin_item.rb @@ -34,7 +34,7 @@ class SimplefinItem < ApplicationRecord end def process_accounts - simplefin_accounts.each do |simplefin_account| + simplefin_accounts.joins(:account).each do |simplefin_account| SimplefinAccount::Processor.new(simplefin_account).process end end @@ -54,18 +54,103 @@ class SimplefinItem < ApplicationRecord raw_payload: accounts_snapshot, ) + # Extract institution data from the first account if available + snapshot = accounts_snapshot.to_h.with_indifferent_access + if snapshot[:accounts].present? + first_account = snapshot[:accounts].first + org = first_account[:org] + upsert_institution_data!(org) if org.present? + end + save! end - def upsert_simplefin_institution_snapshot!(institution_snapshot) - assign_attributes( - institution_id: institution_snapshot[:id], - institution_name: institution_snapshot[:name], - institution_url: institution_snapshot[:url], - raw_institution_payload: institution_snapshot - ) + def upsert_institution_data!(org_data) + org = org_data.to_h.with_indifferent_access + url = org[:url] || org[:"sfin-url"] + domain = org[:domain] - save! + # Derive domain from URL if missing + if domain.blank? && url.present? + begin + domain = URI.parse(url).host&.gsub(/^www\./, "") + rescue URI::InvalidURIError + Rails.logger.warn("Invalid SimpleFin institution URL: #{url.inspect}") + end + end + + assign_attributes( + institution_id: org[:id], + institution_name: org[:name], + institution_domain: domain, + institution_url: url, + raw_institution_payload: org_data + ) + end + + + def has_completed_initial_setup? + # Setup is complete if we have any linked accounts + accounts.any? + end + + def sync_status_summary + latest = latest_sync + return nil unless latest + + # If sync has statistics, use them + if latest.sync_stats.present? + stats = latest.sync_stats + total = stats["total_accounts"] || 0 + linked = stats["linked_accounts"] || 0 + unlinked = stats["unlinked_accounts"] || 0 + + if total == 0 + "No accounts found" + elsif unlinked == 0 + "#{linked} #{'account'.pluralize(linked)} synced" + else + "#{linked} synced, #{unlinked} need setup" + end + else + # Fallback to current account counts + total_accounts = simplefin_accounts.count + linked_count = accounts.count + unlinked_count = total_accounts - linked_count + + if total_accounts == 0 + "No accounts found" + elsif unlinked_count == 0 + "#{linked_count} #{'account'.pluralize(linked_count)} synced" + else + "#{linked_count} synced, #{unlinked_count} need setup" + end + end + end + + def institution_display_name + # Try to get institution name from stored metadata + institution_name.presence || institution_domain.presence || name + end + + def connected_institutions + # Get unique institutions from all accounts + simplefin_accounts.includes(:account) + .where.not(org_data: nil) + .map { |acc| acc.org_data } + .uniq { |org| org["domain"] || org["name"] } + end + + def institution_summary + institutions = connected_institutions + case institutions.count + when 0 + "No institutions connected" + when 1 + institutions.first["name"] || institutions.first["domain"] || "1 institution" + else + "#{institutions.count} institutions" + end end private diff --git a/app/models/simplefin_item/importer.rb b/app/models/simplefin_item/importer.rb index 901f1195e..d17b7df2a 100644 --- a/app/models/simplefin_item/importer.rb +++ b/app/models/simplefin_item/importer.rb @@ -7,95 +7,220 @@ class SimplefinItem::Importer end def import - # Determine start date based on sync history - start_date = determine_sync_start_date + Rails.logger.info "SimplefinItem::Importer - Starting import for item #{simplefin_item.id}" + Rails.logger.info "SimplefinItem::Importer - last_synced_at: #{simplefin_item.last_synced_at.inspect}" + Rails.logger.info "SimplefinItem::Importer - sync_start_date: #{simplefin_item.sync_start_date.inspect}" - if start_date + if simplefin_item.last_synced_at.nil? + # First sync - use chunked approach to get full history + Rails.logger.info "SimplefinItem::Importer - Using chunked history import" + import_with_chunked_history else - end - - accounts_data = simplefin_provider.get_accounts( - simplefin_item.access_url, - start_date: start_date - ) - - # Handle errors if present - if accounts_data[:errors] && accounts_data[:errors].any? - handle_errors(accounts_data[:errors]) - return - end - - # Store raw payload - simplefin_item.upsert_simplefin_snapshot!(accounts_data) - - # Import accounts - accounts_data[:accounts] is an array - accounts_data[:accounts]&.each do |account_data| - import_account(account_data) + # Regular sync - use single request with buffer + Rails.logger.info "SimplefinItem::Importer - Using regular sync" + import_regular_sync end end private + def import_with_chunked_history + # SimpleFin's actual limit is 60 days (not 365 as documented) + # Use 60-day chunks to stay within limits + chunk_size_days = 60 + max_requests = 22 + current_end_date = Time.current + + # Use user-selected sync_start_date if available, otherwise use default lookback + user_start_date = simplefin_item.sync_start_date + default_start_date = initial_sync_lookback_period.days.ago + target_start_date = user_start_date ? user_start_date.beginning_of_day : default_start_date + + # Enforce maximum 3-year lookback to respect SimpleFin's actual 60-day limit per request + # With 22 requests max: 60 days × 22 = 1,320 days = 3.6 years, so 3 years is safe + max_lookback_date = 3.years.ago.beginning_of_day + if target_start_date < max_lookback_date + Rails.logger.info "SimpleFin: Limiting sync start date from #{target_start_date.strftime('%Y-%m-%d')} to #{max_lookback_date.strftime('%Y-%m-%d')} due to rate limits" + target_start_date = max_lookback_date + end + + total_accounts_imported = 0 + chunk_count = 0 + + Rails.logger.info "SimpleFin chunked sync: syncing from #{target_start_date.strftime('%Y-%m-%d')} to #{current_end_date.strftime('%Y-%m-%d')}" + + # Walk backwards from current_end_date in proper chunks + chunk_end_date = current_end_date + + while chunk_count < max_requests && chunk_end_date > target_start_date + chunk_count += 1 + + # Calculate chunk start date - always use exactly chunk_size_days to stay within limits + chunk_start_date = chunk_end_date - chunk_size_days.days + + # Don't go back further than the target start date + if chunk_start_date < target_start_date + chunk_start_date = target_start_date + end + + # Verify we're within SimpleFin's limits + actual_days = (chunk_end_date.to_date - chunk_start_date.to_date).to_i + if actual_days > 365 + Rails.logger.error "SimpleFin: Chunk exceeds 365 days (#{actual_days} days). This should not happen." + chunk_start_date = chunk_end_date - 365.days + end + + Rails.logger.info "SimpleFin chunked sync: fetching chunk #{chunk_count}/#{max_requests} (#{chunk_start_date.strftime('%Y-%m-%d')} to #{chunk_end_date.strftime('%Y-%m-%d')}) - #{actual_days} days" + + accounts_data = fetch_accounts_data(start_date: chunk_start_date, end_date: chunk_end_date) + return if accounts_data.nil? # Error already handled + + # Store raw payload on first chunk only + if chunk_count == 1 + simplefin_item.upsert_simplefin_snapshot!(accounts_data) + end + + # Import accounts and transactions for this chunk + accounts_data[:accounts]&.each do |account_data| + import_account(account_data) + end + total_accounts_imported += accounts_data[:accounts]&.size || 0 + + # Stop if we've reached our target start date + if chunk_start_date <= target_start_date + Rails.logger.info "SimpleFin chunked sync: reached target start date, stopping" + break + end + + # Continue to next chunk - move the end date backwards + chunk_end_date = chunk_start_date + end + + Rails.logger.info "SimpleFin chunked sync completed: #{chunk_count} chunks processed, #{total_accounts_imported} account records imported" + end + + def import_regular_sync + start_date = determine_sync_start_date + + accounts_data = fetch_accounts_data(start_date: start_date) + return if accounts_data.nil? # Error already handled + + # Store raw payload + simplefin_item.upsert_simplefin_snapshot!(accounts_data) + + # Import accounts + accounts_data[:accounts]&.each do |account_data| + import_account(account_data) + end + end + + def fetch_accounts_data(start_date:, end_date: nil) + # Debug logging to track exactly what's being sent to SimpleFin API + days_requested = end_date ? (end_date.to_date - start_date.to_date).to_i : "unknown" + Rails.logger.info "SimplefinItem::Importer - API Request: #{start_date.strftime('%Y-%m-%d')} to #{end_date&.strftime('%Y-%m-%d') || 'current'} (#{days_requested} days)" + + begin + accounts_data = simplefin_provider.get_accounts( + simplefin_item.access_url, + start_date: start_date, + end_date: end_date + ) + rescue Provider::Simplefin::SimplefinError => e + # Handle authentication errors by marking item as requiring update + if e.error_type == :access_forbidden + simplefin_item.update!(status: :requires_update) + raise e + else + raise e + end + end + + # Handle errors if present in response + if accounts_data[:errors] && accounts_data[:errors].any? + handle_errors(accounts_data[:errors]) + return nil + end + + accounts_data + end + def determine_sync_start_date - # For the first sync, get all available data by using a very wide date range + # For the first sync, get only a limited amount of data to avoid SimpleFin API limits # SimpleFin requires a start_date parameter - without it, only returns recent transactions unless simplefin_item.last_synced_at - return 100.years.ago # Set to 100 years for first sync to get everything just to be sure + return initial_sync_lookback_period.days.ago end # For subsequent syncs, fetch from last sync date with a buffer - # Use 7 days buffer to ensure we don't miss any late-posting transactions - simplefin_item.last_synced_at - 7.days + # Use buffer to ensure we don't miss any late-posting transactions + simplefin_item.last_synced_at - sync_buffer_period.days end def import_account(account_data) - # Import organization data from the account if present and not already imported - if account_data[:org] && simplefin_item.institution_id.blank? - import_organization(account_data[:org]) - end + account_id = account_data[:id] + + # Validate required account_id to prevent duplicate creation + return if account_id.blank? simplefin_account = simplefin_item.simplefin_accounts.find_or_initialize_by( - account_id: account_data[:id] + account_id: account_id ) - # Store transactions temporarily + # Store transactions and holdings separately from account data to avoid overwriting transactions = account_data[:transactions] + holdings = account_data[:holdings] - # Update account snapshot first (without transactions) - simplefin_account.upsert_simplefin_snapshot!(account_data) - - # Then save transactions separately (so they don't get overwritten) - if transactions && transactions.any? - simplefin_account.update!(raw_transactions_payload: transactions) - else - end - end - - def import_organization(org_data) - # Create normalized institution data for compatibility - normalized_data = { - id: org_data[:domain] || org_data[:"sfin-url"], - name: org_data[:name] || extract_domain_name(org_data[:domain]), - url: org_data[:domain] || org_data[:"sfin-url"], - # Store the complete raw organization data - raw_org_data: org_data + # Update all attributes; only update transactions if present to avoid wiping prior data + attrs = { + name: account_data[:name], + account_type: account_data["type"] || account_data[:type] || "unknown", + currency: account_data[:currency] || "USD", + current_balance: account_data[:balance], + available_balance: account_data[:"available-balance"], + balance_date: account_data[:"balance-date"], + raw_payload: account_data, + org_data: account_data[:org] } - simplefin_item.upsert_simplefin_institution_snapshot!(normalized_data) + # Merge transactions from chunked imports (accumulate historical data) + if transactions.is_a?(Array) && transactions.any? + existing_transactions = simplefin_account.raw_transactions_payload.to_a + merged_transactions = (existing_transactions + transactions).uniq do |tx| + tx = tx.with_indifferent_access + tx[:id] || tx[:fitid] || [ tx[:posted], tx[:amount], tx[:description] ] + end + attrs[:raw_transactions_payload] = merged_transactions + end + + # Preserve most recent holdings (don't overwrite current positions with older data) + if holdings.is_a?(Array) && holdings.any? && simplefin_account.raw_holdings_payload.blank? + attrs[:raw_holdings_payload] = holdings + end + simplefin_account.assign_attributes(attrs) + + # Final validation before save to prevent duplicates + if simplefin_account.account_id.blank? + simplefin_account.account_id = account_id + end + + simplefin_account.save! end - def extract_domain_name(domain) - return "Unknown Institution" if domain.blank? - - # Extract a readable name from domain like "mybank.com" -> "Mybank" - domain.split(".").first.capitalize - end def handle_errors(errors) - error_messages = errors.map { |error| error[:description] || error[:message] }.join(", ") + error_messages = errors.map { |error| error.is_a?(String) ? error : (error[:description] || error[:message]) }.join(", ") - # Mark item as requiring update for certain error types - if errors.any? { |error| error[:code] == "auth_failure" || error[:code] == "token_expired" } + # Mark item as requiring update for authentication-related errors + needs_update = errors.any? do |error| + if error.is_a?(String) + error.downcase.include?("reauthenticate") || error.downcase.include?("authentication") + else + error[:code] == "auth_failure" || error[:code] == "token_expired" || + error[:type] == "authentication_error" + end + end + + if needs_update simplefin_item.update!(status: :requires_update) end @@ -104,4 +229,14 @@ class SimplefinItem::Importer :api_error ) end + + def initial_sync_lookback_period + # Default to 7 days for initial sync to avoid API limits + 7 + end + + def sync_buffer_period + # Default to 7 days buffer for subsequent syncs + 7 + end end diff --git a/app/models/simplefin_item/syncer.rb b/app/models/simplefin_item/syncer.rb index cf20a9912..da0275d93 100644 --- a/app/models/simplefin_item/syncer.rb +++ b/app/models/simplefin_item/syncer.rb @@ -6,26 +6,49 @@ class SimplefinItem::Syncer end def perform_sync(sync) - # Loads item metadata, accounts, transactions from SimpleFin API + # Phase 1: Import data from SimpleFin API + sync.update!(status_text: "Importing accounts from SimpleFin...") if sync.respond_to?(:status_text) simplefin_item.import_latest_simplefin_data - # Check if we have new SimpleFin accounts that need setup + # Phase 2: Check account setup status and collect sync statistics + sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text) + total_accounts = simplefin_item.simplefin_accounts.count + linked_accounts = simplefin_item.simplefin_accounts.joins(:account) unlinked_accounts = simplefin_item.simplefin_accounts.includes(:account).where(accounts: { id: nil }) + + # Store sync statistics for display + sync_stats = { + total_accounts: total_accounts, + linked_accounts: linked_accounts.count, + unlinked_accounts: unlinked_accounts.count + } + + # Set pending_account_setup if there are unlinked accounts if unlinked_accounts.any? - # Mark as pending account setup so user can choose account types simplefin_item.update!(pending_account_setup: true) - return + sync.update!(status_text: "#{unlinked_accounts.count} accounts need setup...") if sync.respond_to?(:status_text) + else + simplefin_item.update!(pending_account_setup: false) end - # Processes the raw SimpleFin data and updates internal domain objects - simplefin_item.process_accounts + # Phase 3: Process transactions and holdings for linked accounts only + if linked_accounts.any? + sync.update!(status_text: "Processing transactions and holdings...") if sync.respond_to?(:status_text) + simplefin_item.process_accounts - # All data is synced, so we can now run an account sync to calculate historical balances and more - simplefin_item.schedule_account_syncs( - parent_sync: sync, - window_start_date: sync.window_start_date, - window_end_date: sync.window_end_date - ) + # Phase 4: Schedule balance calculations for linked accounts + sync.update!(status_text: "Calculating balances...") if sync.respond_to?(:status_text) + simplefin_item.schedule_account_syncs( + parent_sync: sync, + window_start_date: sync.window_start_date, + window_end_date: sync.window_end_date + ) + end + + # Store sync statistics in the sync record for status display + if sync.respond_to?(:sync_stats) + sync.update!(sync_stats: sync_stats) + end end def perform_post_sync diff --git a/app/models/sync.rb b/app/models/sync.rb index ea66ebb4d..3e2abfe6e 100644 --- a/app/models/sync.rb +++ b/app/models/sync.rb @@ -21,6 +21,8 @@ class Sync < ApplicationRecord after_commit :update_family_sync_timestamp + serialize :sync_stats, coder: JSON + validate :window_valid # Sync state machine diff --git a/app/views/simplefin_items/_simplefin_item.html.erb b/app/views/simplefin_items/_simplefin_item.html.erb index 43db5e57e..3f2768d3f 100644 --- a/app/views/simplefin_items/_simplefin_item.html.erb +++ b/app/views/simplefin_items/_simplefin_item.html.erb @@ -6,55 +6,69 @@
<%= simplefin_item.institution_name %>
- <% end %> -(deletion in progress...)
+<%= t(".deletion_in_progress") %>
<% end %>+ <%= simplefin_item.institution_summary %> +
+ <% end %> <% if simplefin_item.syncing? %>- <%= simplefin_item.last_synced_at ? "Last synced #{time_ago_in_words(simplefin_item.last_synced_at)} ago" : "Never synced" %> + <% if simplefin_item.last_synced_at %> + <% if simplefin_item.sync_status_summary %> + <%= t(".status_with_summary", timestamp: time_ago_in_words(simplefin_item.last_synced_at), summary: simplefin_item.sync_status_summary) %> + <% else %> + <%= t(".status", timestamp: time_ago_in_words(simplefin_item.last_synced_at)) %> + <% end %> + <% else %> + <%= t(".status_never") %> + <% end %>
<% end %>New accounts ready to set up
-Choose account types for your newly imported SimpleFin accounts.
+<%= t(".setup_needed") %>
+<%= t(".setup_description") %>
<%= render DS::Link.new( - text: "Set Up New Accounts", + text: t(".setup_action"), icon: "settings", variant: "primary", href: setup_accounts_simplefin_item_path(simplefin_item) @@ -94,8 +108,8 @@No accounts found
-This connection doesn't have any synchronized accounts yet.
+<%= t(".no_accounts_title") %>
+<%= t(".no_accounts_description") %>
+ Your SimpleFin connection needs to be updated: +
+<%= @error_message %>
+<%= t(".setup_token.help_text") %>
++ Historical Data Range: +
+ <%= form.date_field :sync_start_date, + label: "Start syncing transactions from:", + value: @simplefin_item.sync_start_date || 1.year.ago.to_date, + min: 3.years.ago.to_date, + max: Date.current, + class: "w-full max-w-xs", + help_text: "Select how far back you want to sync transaction history. Maximum 3 years of history available." %> +@@ -82,7 +103,8 @@ variant: "primary", icon: "plus", type: "submit", - class: "flex-1" + class: "flex-1", + data: { loading_button_target: "button" } ) %> <%= render DS::Link.new( text: "Cancel", diff --git a/config/locales/views/simplefin_items/en.yml b/config/locales/views/simplefin_items/en.yml new file mode 100644 index 000000000..866e8cfcf --- /dev/null +++ b/config/locales/views/simplefin_items/en.yml @@ -0,0 +1,47 @@ +--- +en: + simplefin_items: + create: + success: SimpleFin connection added successfully! Your accounts will appear shortly as they sync in the background. + errors: + blank_token: Please enter a SimpleFin setup token. + invalid_token: Invalid setup token. Please check that you copied the complete token from SimpleFin Bridge. + token_compromised: The setup token may be compromised, expired, or already used. Please create a new one. + create_failed: "Failed to connect: %{message}" + unexpected: An unexpected error occurred. Please try again or contact support. + destroy: + success: SimpleFin connection will be removed + update: + success: SimpleFin connection updated successfully! Your accounts are being reconnected. + errors: + blank_token: Please enter a SimpleFin setup token. + invalid_token: Invalid setup token. Please check that you copied the complete token from SimpleFin Bridge. + token_compromised: The setup token may be compromised, expired, or already used. Please create a new one. + update_failed: "Failed to update connection: %{message}" + unexpected: An unexpected error occurred. Please try again or contact support. + edit: + setup_token: + label: "SimpleFin Setup Token:" + placeholder: "Paste your SimpleFin setup token here..." + help_text: "The token should be a long string starting with letters and numbers" + complete_account_setup: + success: SimpleFin accounts have been set up successfully! Your transactions and holdings are being imported in the background. + simplefin_item: + add_new: Add new connection + confirm_accept: Delete connection + confirm_body: This will permanently delete all the accounts in this group and all associated data. + confirm_title: Delete SimpleFin connection? + delete: Delete + deletion_in_progress: "(deletion in progress...)" + error: Error occurred while syncing data + no_accounts_description: This connection doesn't have any synchronized accounts yet. + no_accounts_title: No accounts found + requires_update: Requires re-authentication + setup_needed: New accounts ready to set up + setup_description: Choose account types for your newly imported SimpleFin accounts. + setup_action: Set Up New Accounts + status: Last synced %{timestamp} ago + status_never: Never synced + status_with_summary: "Last synced %{timestamp} ago • %{summary}" + syncing: Syncing... + update: Update connection \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 7c049f6c4..3c48d7cfe 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -255,7 +255,7 @@ Rails.application.routes.draw do end end - resources :simplefin_items, only: %i[index new create show destroy] do + resources :simplefin_items, only: %i[index new create show edit update destroy] do member do post :sync get :setup_accounts diff --git a/db/migrate/20250813144520_add_institution_fields_to_simplefin_items.rb b/db/migrate/20250813144520_add_institution_fields_to_simplefin_items.rb new file mode 100644 index 000000000..86c0806ab --- /dev/null +++ b/db/migrate/20250813144520_add_institution_fields_to_simplefin_items.rb @@ -0,0 +1,33 @@ +class AddInstitutionFieldsToSimplefinItems < ActiveRecord::Migration[7.2] + def up + # Only add the new fields that don't already exist + # institution_id, institution_name, institution_url, and raw_institution_payload + # already exist from the original create_simplefin_items migration + add_column :simplefin_items, :institution_domain, :string + add_column :simplefin_items, :institution_color, :string + + # Add indexes for performance on commonly queried institution fields + add_index :simplefin_items, :institution_id + add_index :simplefin_items, :institution_domain + add_index :simplefin_items, :institution_name + + # Enforce NOT NULL constraints on required fields + change_column_null :simplefin_items, :institution_id, false + change_column_null :simplefin_items, :institution_name, false + end + + def down + # Revert NOT NULL constraints + change_column_null :simplefin_items, :institution_id, true + change_column_null :simplefin_items, :institution_name, true + + # Remove indexes + remove_index :simplefin_items, :institution_id + remove_index :simplefin_items, :institution_domain + remove_index :simplefin_items, :institution_name + + # Remove the new columns + remove_column :simplefin_items, :institution_domain + remove_column :simplefin_items, :institution_color + end +end diff --git a/db/migrate/20250901004029_add_sync_start_date_to_simplefin_items.rb b/db/migrate/20250901004029_add_sync_start_date_to_simplefin_items.rb new file mode 100644 index 000000000..6c4cb53b6 --- /dev/null +++ b/db/migrate/20250901004029_add_sync_start_date_to_simplefin_items.rb @@ -0,0 +1,5 @@ +class AddSyncStartDateToSimplefinItems < ActiveRecord::Migration[7.2] + def change + add_column :simplefin_items, :sync_start_date, :date + end +end diff --git a/db/migrate/20250901005519_add_sync_stats_to_syncs.rb b/db/migrate/20250901005519_add_sync_stats_to_syncs.rb new file mode 100644 index 000000000..207508631 --- /dev/null +++ b/db/migrate/20250901005519_add_sync_stats_to_syncs.rb @@ -0,0 +1,5 @@ +class AddSyncStatsToSyncs < ActiveRecord::Migration[7.2] + def change + add_column :syncs, :sync_stats, :text + end +end diff --git a/db/migrate/20250901182423_add_raw_holdings_payload_to_simplefin_accounts.rb b/db/migrate/20250901182423_add_raw_holdings_payload_to_simplefin_accounts.rb new file mode 100644 index 000000000..f836d60fa --- /dev/null +++ b/db/migrate/20250901182423_add_raw_holdings_payload_to_simplefin_accounts.rb @@ -0,0 +1,5 @@ +class AddRawHoldingsPayloadToSimplefinAccounts < ActiveRecord::Migration[7.2] + def change + add_column :simplefin_accounts, :raw_holdings_payload, :jsonb + end +end diff --git a/db/migrate/20250901183328_add_external_id_and_cost_basis_to_holdings.rb b/db/migrate/20250901183328_add_external_id_and_cost_basis_to_holdings.rb new file mode 100644 index 000000000..06ea5f7d5 --- /dev/null +++ b/db/migrate/20250901183328_add_external_id_and_cost_basis_to_holdings.rb @@ -0,0 +1,6 @@ +class AddExternalIdAndCostBasisToHoldings < ActiveRecord::Migration[7.2] + def change + add_column :holdings, :external_id, :string + add_column :holdings, :cost_basis, :decimal + end +end diff --git a/db/migrate/20250903010617_add_unique_index_to_holdings_external_id.rb b/db/migrate/20250903010617_add_unique_index_to_holdings_external_id.rb new file mode 100644 index 000000000..62352b67f --- /dev/null +++ b/db/migrate/20250903010617_add_unique_index_to_holdings_external_id.rb @@ -0,0 +1,5 @@ +class AddUniqueIndexToHoldingsExternalId < ActiveRecord::Migration[7.2] + def change + add_index :holdings, [ :account_id, :external_id ], unique: true, name: 'index_holdings_on_account_and_external_id' + end +end diff --git a/db/migrate/20250903015009_fix_holdings_cost_basis_and_external_id_constraints.rb b/db/migrate/20250903015009_fix_holdings_cost_basis_and_external_id_constraints.rb new file mode 100644 index 000000000..25efc1ae4 --- /dev/null +++ b/db/migrate/20250903015009_fix_holdings_cost_basis_and_external_id_constraints.rb @@ -0,0 +1,9 @@ +class FixHoldingsCostBasisAndExternalIdConstraints < ActiveRecord::Migration[7.2] + def change + change_column :holdings, :cost_basis, :decimal, precision: 19, scale: 4 + add_index :holdings, [ :account_id, :external_id ], + unique: true, + where: "external_id IS NOT NULL", + name: "idx_holdings_on_account_id_external_id_unique" + end +end diff --git a/db/schema.rb b/db/schema.rb index 90d190dcf..315069252 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_08_08_143007) do +ActiveRecord::Schema[7.2].define(version: 2025_09_03_015009) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -29,7 +29,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_08_08_143007) do t.uuid "accountable_id" t.decimal "balance", precision: 19, scale: 4 t.string "currency" - t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true + t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true t.uuid "import_id" t.uuid "plaid_account_id" t.decimal "cash_balance", precision: 19, scale: 4, default: "0.0" @@ -293,6 +293,10 @@ ActiveRecord::Schema[7.2].define(version: 2025_08_08_143007) do t.string "currency", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "external_id" + t.decimal "cost_basis", precision: 19, scale: 4 + t.index ["account_id", "external_id"], name: "idx_holdings_on_account_id_external_id_unique", unique: true, where: "(external_id IS NOT NULL)" + t.index ["account_id", "external_id"], name: "index_holdings_on_account_and_external_id", unique: true t.index ["account_id", "security_id", "date", "currency"], name: "idx_on_account_id_security_id_date_currency_5323e39f8b", unique: true t.index ["account_id"], name: "index_holdings_on_account_id" t.index ["security_id"], name: "index_holdings_on_security_id" @@ -698,6 +702,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_08_08_143007) do t.datetime "balance_date" t.jsonb "extra" t.jsonb "org_data" + t.jsonb "raw_holdings_payload" t.index ["account_id"], name: "index_simplefin_accounts_on_account_id" t.index ["simplefin_item_id"], name: "index_simplefin_accounts_on_simplefin_item_id" end @@ -716,6 +721,9 @@ ActiveRecord::Schema[7.2].define(version: 2025_08_08_143007) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.boolean "pending_account_setup", default: false, null: false + t.string "institution_domain" + t.string "institution_color" + t.date "sync_start_date" t.index ["family_id"], name: "index_simplefin_items_on_family_id" t.index ["status"], name: "index_simplefin_items_on_status" end @@ -749,6 +757,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_08_08_143007) do t.datetime "failed_at" t.date "window_start_date" t.date "window_end_date" + t.text "sync_stats" t.index ["parent_id"], name: "index_syncs_on_parent_id" t.index ["status"], name: "index_syncs_on_status" t.index ["syncable_type", "syncable_id"], name: "index_syncs_on_syncable" diff --git a/test/controllers/simplefin_items_controller_test.rb b/test/controllers/simplefin_items_controller_test.rb index 1ba03fd4a..26d26475a 100644 --- a/test/controllers/simplefin_items_controller_test.rb +++ b/test/controllers/simplefin_items_controller_test.rb @@ -32,14 +32,207 @@ class SimplefinItemsControllerTest < ActionDispatch::IntegrationTest delete simplefin_item_url(@simplefin_item) end - assert_redirected_to simplefin_items_path + assert_redirected_to accounts_path @simplefin_item.reload assert @simplefin_item.scheduled_for_deletion? end test "should sync simplefin item" do post sync_simplefin_item_url(@simplefin_item) - assert_redirected_to simplefin_item_path(@simplefin_item) - assert_equal "Sync started", flash[:notice] + assert_redirected_to accounts_path + end + + test "should get edit" do + @simplefin_item.update!(status: :requires_update) + get edit_simplefin_item_url(@simplefin_item) + assert_response :success + end + + test "should update simplefin item with valid token" do + @simplefin_item.update!(status: :requires_update) + + # Mock the SimpleFin provider to prevent real API calls + mock_provider = mock() + mock_provider.expects(:claim_access_url).with("valid_token").returns("https://example.com/new_access") + mock_provider.expects(:get_accounts).returns({ accounts: [] }).at_least_once + Provider::Simplefin.expects(:new).returns(mock_provider).at_least_once + + # Let the real create_simplefin_item! method run - don't mock it + + patch simplefin_item_url(@simplefin_item), params: { + simplefin_item: { setup_token: "valid_token" } + } + + assert_redirected_to accounts_path + assert_match(/updated successfully/, flash[:notice]) + @simplefin_item.reload + assert @simplefin_item.scheduled_for_deletion? + end + + test "should handle update with invalid token" do + @simplefin_item.update!(status: :requires_update) + + patch simplefin_item_url(@simplefin_item), params: { + simplefin_item: { setup_token: "" } + } + + assert_response :unprocessable_entity + assert_includes response.body, "Please enter a SimpleFin setup token" + end + + test "should transfer accounts when updating simplefin item token" do + @simplefin_item.update!(status: :requires_update) + + # Create old SimpleFin accounts linked to Maybe accounts + old_simplefin_account1 = @simplefin_item.simplefin_accounts.create!( + name: "Test Checking", + account_id: "sf_account_123", + currency: "USD", + current_balance: 1000, + account_type: "depository" + ) + old_simplefin_account2 = @simplefin_item.simplefin_accounts.create!( + name: "Test Savings", + account_id: "sf_account_456", + currency: "USD", + current_balance: 5000, + account_type: "depository" + ) + + # Create Maybe accounts linked to the SimpleFin accounts + maybe_account1 = Account.create!( + family: @family, + name: "Checking Account", + balance: 1000, + currency: "USD", + accountable_type: "Depository", + accountable: Depository.create!(subtype: "checking"), + simplefin_account_id: old_simplefin_account1.id + ) + maybe_account2 = Account.create!( + family: @family, + name: "Savings Account", + balance: 5000, + currency: "USD", + accountable_type: "Depository", + accountable: Depository.create!(subtype: "savings"), + simplefin_account_id: old_simplefin_account2.id + ) + + # Update old SimpleFin accounts to reference the Maybe accounts + old_simplefin_account1.update!(account: maybe_account1) + old_simplefin_account2.update!(account: maybe_account2) + + # Mock only the external API calls, let business logic run + mock_provider = mock() + mock_provider.expects(:claim_access_url).with("valid_token").returns("https://example.com/new_access") + mock_provider.expects(:get_accounts).returns({ + accounts: [ + { + id: "sf_account_123", + name: "Test Checking", + type: "depository", + currency: "USD", + balance: 1000, + transactions: [] + }, + { + id: "sf_account_456", + name: "Test Savings", + type: "depository", + currency: "USD", + balance: 5000, + transactions: [] + } + ] + }).at_least_once + Provider::Simplefin.expects(:new).returns(mock_provider).at_least_once + + # Perform the update + patch simplefin_item_url(@simplefin_item), params: { + simplefin_item: { setup_token: "valid_token" } + } + + assert_redirected_to accounts_path + assert_match(/updated successfully/, flash[:notice]) + + # Verify accounts were transferred to new SimpleFin accounts + assert Account.exists?(maybe_account1.id), "maybe_account1 should still exist" + assert Account.exists?(maybe_account2.id), "maybe_account2 should still exist" + + maybe_account1.reload + maybe_account2.reload + + # Find the new SimpleFin item that was created + new_simplefin_item = @family.simplefin_items.where.not(id: @simplefin_item.id).first + assert_not_nil new_simplefin_item, "New SimpleFin item should have been created" + + new_sf_account1 = new_simplefin_item.simplefin_accounts.find_by(account_id: "sf_account_123") + new_sf_account2 = new_simplefin_item.simplefin_accounts.find_by(account_id: "sf_account_456") + + assert_not_nil new_sf_account1, "New SimpleFin account with ID sf_account_123 should exist" + assert_not_nil new_sf_account2, "New SimpleFin account with ID sf_account_456 should exist" + + assert_equal new_sf_account1.id, maybe_account1.simplefin_account_id + assert_equal new_sf_account2.id, maybe_account2.simplefin_account_id + + # Verify old SimpleFin accounts no longer reference Maybe accounts + old_simplefin_account1.reload + old_simplefin_account2.reload + assert_nil old_simplefin_account1.account + assert_nil old_simplefin_account2.account + + # Verify old SimpleFin item is scheduled for deletion + @simplefin_item.reload + assert @simplefin_item.scheduled_for_deletion? + end + + test "should handle partial account matching during token update" do + @simplefin_item.update!(status: :requires_update) + + # Create old SimpleFin account + old_simplefin_account = @simplefin_item.simplefin_accounts.create!( + name: "Test Checking", + account_id: "sf_account_123", + currency: "USD", + current_balance: 1000, + account_type: "depository" + ) + + # Create Maybe account linked to the SimpleFin account + maybe_account = Account.create!( + family: @family, + name: "Checking Account", + balance: 1000, + currency: "USD", + accountable_type: "Depository", + accountable: Depository.create!(subtype: "checking"), + simplefin_account_id: old_simplefin_account.id + ) + old_simplefin_account.update!(account: maybe_account) + + # Mock only the external API calls, let business logic run + mock_provider = mock() + mock_provider.expects(:claim_access_url).with("valid_token").returns("https://example.com/new_access") + # Return empty accounts list to simulate account was removed from bank + mock_provider.expects(:get_accounts).returns({ accounts: [] }).at_least_once + Provider::Simplefin.expects(:new).returns(mock_provider).at_least_once + + # Perform update + patch simplefin_item_url(@simplefin_item), params: { + simplefin_item: { setup_token: "valid_token" } + } + + assert_redirected_to accounts_path + + # Verify Maybe account still linked to old SimpleFin account (no transfer occurred) + maybe_account.reload + old_simplefin_account.reload + assert_equal old_simplefin_account.id, maybe_account.simplefin_account_id + assert_equal maybe_account, old_simplefin_account.account + + # Old item still scheduled for deletion + @simplefin_item.reload + assert @simplefin_item.scheduled_for_deletion? end end diff --git a/test/models/simplefin_item_test.rb b/test/models/simplefin_item_test.rb index 30a429867..e6f8514c8 100644 --- a/test/models/simplefin_item_test.rb +++ b/test/models/simplefin_item_test.rb @@ -61,4 +61,88 @@ class SimplefinItemTest < ActiveSupport::TestCase assert_equal [ @simplefin_item, item_for_deletion ].sort_by(&:created_at).reverse, ordered_items.to_a end + + test "upserts institution data correctly" do + org_data = { + id: "bank123", + name: "Test Bank", + domain: "testbank.com", + url: "https://testbank.com", + "sfin-url": "https://sfin.testbank.com" + } + + @simplefin_item.upsert_institution_data!(org_data) + + assert_equal "bank123", @simplefin_item.institution_id + assert_equal "Test Bank", @simplefin_item.institution_name + assert_equal "testbank.com", @simplefin_item.institution_domain + assert_equal "https://testbank.com", @simplefin_item.institution_url + assert_equal org_data.stringify_keys, @simplefin_item.raw_institution_payload + end + + test "institution display name fallback works" do + # No institution data + assert_equal @simplefin_item.name, @simplefin_item.institution_display_name + + # With institution name + @simplefin_item.update!(institution_name: "Chase Bank") + assert_equal "Chase Bank", @simplefin_item.institution_display_name + + # With domain fallback + @simplefin_item.update!(institution_name: nil, institution_domain: "chase.com") + assert_equal "chase.com", @simplefin_item.institution_display_name + end + + test "connected institutions returns unique institutions" do + # Create accounts with different institutions + account1 = @simplefin_item.simplefin_accounts.create!( + name: "Checking", + account_id: "acc1", + currency: "USD", + account_type: "checking", + current_balance: 1000, + org_data: { "name" => "Chase Bank", "domain" => "chase.com" } + ) + + account2 = @simplefin_item.simplefin_accounts.create!( + name: "Savings", + account_id: "acc2", + currency: "USD", + account_type: "savings", + current_balance: 2000, + org_data: { "name" => "Wells Fargo", "domain" => "wellsfargo.com" } + ) + + institutions = @simplefin_item.connected_institutions + assert_equal 2, institutions.count + assert_includes institutions.map { |i| i["name"] }, "Chase Bank" + assert_includes institutions.map { |i| i["name"] }, "Wells Fargo" + end + + test "institution summary with multiple institutions" do + # No institutions + assert_equal "No institutions connected", @simplefin_item.institution_summary + + # One institution + @simplefin_item.simplefin_accounts.create!( + name: "Checking", + account_id: "acc1", + currency: "USD", + account_type: "checking", + current_balance: 1000, + org_data: { "name" => "Chase Bank" } + ) + assert_equal "Chase Bank", @simplefin_item.institution_summary + + # Multiple institutions + @simplefin_item.simplefin_accounts.create!( + name: "Savings", + account_id: "acc2", + currency: "USD", + account_type: "savings", + current_balance: 2000, + org_data: { "name" => "Wells Fargo" } + ) + assert_equal "2 institutions", @simplefin_item.institution_summary + end end