Additional cache columns on balances for activity view breakdowns (#2505)

* Initial schema iteration * Add new balance components * Add existing data migrator to backfill components * Update calculator test assertions for new balance components * Update flow assertions for forward calculator * Update reverse calculator flows assumptions * Forward calculator tests passing * Get all calculator tests passing * Assert flows factor
2026-04-07 06:21:23 +00:00 · 2025-07-23 10:06:25 -04:00
parent 347c0a7906
commit da2045dbd8
13 changed files with 1159 additions and 177 deletions
--- a/app/data_migrations/balance_component_migrator.rb
+++ b/app/data_migrations/balance_component_migrator.rb
@@ -0,0 +1,59 @@
+class BalanceComponentMigrator
+  def self.run
+    ActiveRecord::Base.transaction do
+      # Step 1: Update flows factor
+      ActiveRecord::Base.connection.execute <<~SQL
+        UPDATE balances SET
+          flows_factor = CASE WHEN a.classification = 'asset' THEN 1 ELSE -1 END
+        FROM accounts a
+        WHERE a.id = balances.account_id
+      SQL
+
+      # Step 2: Set start values using LOCF (Last Observation Carried Forward)
+      ActiveRecord::Base.connection.execute <<~SQL
+        UPDATE balances b1
+        SET
+          start_cash_balance = COALESCE(prev.cash_balance, 0),
+          start_non_cash_balance = COALESCE(prev.balance - prev.cash_balance, 0)
+        FROM balances b1_inner
+        LEFT JOIN LATERAL (
+          SELECT
+            b2.cash_balance,
+            b2.balance
+          FROM balances b2
+          WHERE b2.account_id = b1_inner.account_id
+          AND b2.currency = b1_inner.currency
+          AND b2.date < b1_inner.date
+          ORDER BY b2.date DESC
+          LIMIT 1
+        ) prev ON true
+        WHERE b1.id = b1_inner.id
+      SQL
+
+      # Step 3: Calculate net inflows
+      # A slight workaround to the fact that we can't easily derive inflows/outflows from our current data model, and
+      # the tradeoff not worth it since each new sync will fix it. So instead, we sum up *net* flows, and throw the signed
+      # amount in the "inflows" column, and zero-out the "outflows" column so our math works correctly with incomplete data.
+      ActiveRecord::Base.connection.execute <<~SQL
+        UPDATE balances SET
+          cash_inflows = (cash_balance - start_cash_balance) * flows_factor,
+          cash_outflows = 0,
+          non_cash_inflows = ((balance - cash_balance) - start_non_cash_balance) * flows_factor,
+          non_cash_outflows = 0,
+          net_market_flows = 0
+      SQL
+
+      # Verify data integrity
+      # All end_balance values should match the original balance
+      invalid_count = ActiveRecord::Base.connection.select_value(<<~SQL)
+        SELECT COUNT(*)
+        FROM balances b
+        WHERE ABS(b.balance - b.end_balance) > 0.0001
+      SQL
+
+      if invalid_count > 0
+        raise "Data migration failed validation: #{invalid_count} balances have incorrect end_balance values"
+      end
+    end
+  end
+end
--- a/app/models/balance.rb
+++ b/app/models/balance.rb
@@ -2,8 +2,16 @@ class Balance < ApplicationRecord
  include Monetizable

  belongs_to :account
+
  validates :account, :date, :balance, presence: true
-  monetize :balance, :cash_balance
+  validates :flows_factor, inclusion: { in: [ -1, 1 ] }
+
+  monetize :balance, :cash_balance,
+           :start_cash_balance, :start_non_cash_balance, :start_balance,
+           :cash_inflows, :cash_outflows, :non_cash_inflows, :non_cash_outflows, :net_market_flows,
+           :cash_adjustments, :non_cash_adjustments,
+           :end_cash_balance, :end_non_cash_balance, :end_balance
+
  scope :in_period, ->(period) { period.nil? ? all : where(date: period.date_range) }
  scope :chronological, -> { order(:date) }
 end
--- a/app/models/balance/base_calculator.rb
+++ b/app/models/balance/base_calculator.rb
@@ -15,8 +15,8 @@ class Balance::BaseCalculator
    end

    def holdings_value_for_date(date)
-      holdings = sync_cache.get_holdings(date)
-      holdings.sum(&:amount)
+      @holdings_value_for_date ||= {}
+      @holdings_value_for_date[date] ||= sync_cache.get_holdings(date).sum(&:amount)
    end

    def derive_cash_balance_on_date_from_total(total_balance:, date:)
@@ -29,6 +29,67 @@ class Balance::BaseCalculator
      end
    end

+    def cash_adjustments_for_date(start_cash, net_cash_flows, valuation)
+      return 0 unless valuation && account.balance_type != :non_cash
+
+      valuation.amount - start_cash - net_cash_flows
+    end
+
+    def non_cash_adjustments_for_date(start_non_cash, non_cash_flows, valuation)
+      return 0 unless valuation && account.balance_type == :non_cash
+
+      valuation.amount - start_non_cash - non_cash_flows
+    end
+
+    # If holdings value goes from $100 -> $200 (change_holdings_value is $100)
+    # And non-cash flows (i.e. "buys") for day are +$50 (net_buy_sell_value is $50)
+    # That means value increased by $100, where $50 of that is due to the change in holdings value, and $50 is due to the buy/sell
+    def market_value_change_on_date(date, flows)
+      return 0 unless account.balance_type == :investment
+
+      start_of_day_holdings_value = holdings_value_for_date(date.prev_day)
+      end_of_day_holdings_value = holdings_value_for_date(date)
+
+      change_holdings_value = end_of_day_holdings_value - start_of_day_holdings_value
+      net_buy_sell_value = flows[:non_cash_inflows] - flows[:non_cash_outflows]
+
+      change_holdings_value - net_buy_sell_value
+    end
+
+    def flows_for_date(date)
+      entries = sync_cache.get_entries(date)
+
+      cash_inflows = 0
+      cash_outflows = 0
+      non_cash_inflows = 0
+      non_cash_outflows = 0
+
+      txn_inflow_sum = entries.select { |e| e.amount < 0 && e.transaction? }.sum(&:amount)
+      txn_outflow_sum = entries.select { |e| e.amount >= 0 && e.transaction? }.sum(&:amount)
+
+      trade_cash_inflow_sum = entries.select { |e| e.amount < 0 && e.trade? }.sum(&:amount)
+      trade_cash_outflow_sum = entries.select { |e| e.amount >= 0 && e.trade? }.sum(&:amount)
+
+      if account.balance_type == :non_cash && account.accountable_type == "Loan"
+        non_cash_inflows = txn_inflow_sum.abs
+        non_cash_outflows = txn_outflow_sum
+      elsif account.balance_type != :non_cash
+        cash_inflows = txn_inflow_sum.abs + trade_cash_inflow_sum.abs
+        cash_outflows = txn_outflow_sum + trade_cash_outflow_sum
+
+        # Trades are inverse (a "buy" is outflow of cash, but "inflow" of non-cash, aka "holdings")
+        non_cash_outflows = trade_cash_inflow_sum.abs
+        non_cash_inflows = trade_cash_outflow_sum
+      end
+
+      {
+        cash_inflows: cash_inflows,
+        cash_outflows: cash_outflows,
+        non_cash_inflows: non_cash_inflows,
+        non_cash_outflows: non_cash_outflows
+      }
+    end
+
    def derive_cash_balance(cash_balance, date)
      entries = sync_cache.get_entries(date)

@@ -57,13 +118,23 @@ class Balance::BaseCalculator
      raise NotImplementedError, "Directional calculators must implement this method"
    end

-    def build_balance(date:, cash_balance:, non_cash_balance:)
+    def build_balance(date:, **args)
      Balance.new(
        account_id: account.id,
+        currency: account.currency,
        date: date,
-        balance: non_cash_balance + cash_balance,
-        cash_balance: cash_balance,
-        currency: account.currency
+        balance: args[:balance],
+        cash_balance: args[:cash_balance],
+        start_cash_balance: args[:start_cash_balance] || 0,
+        start_non_cash_balance: args[:start_non_cash_balance] || 0,
+        cash_inflows: args[:cash_inflows] || 0,
+        cash_outflows: args[:cash_outflows] || 0,
+        non_cash_inflows: args[:non_cash_inflows] || 0,
+        non_cash_outflows: args[:non_cash_outflows] || 0,
+        cash_adjustments: args[:cash_adjustments] || 0,
+        non_cash_adjustments: args[:non_cash_adjustments] || 0,
+        net_market_flows: args[:net_market_flows] || 0,
+        flows_factor: account.classification == "asset" ? 1 : -1
      )
    end
 end
--- a/app/models/balance/forward_calculator.rb
+++ b/app/models/balance/forward_calculator.rb
@@ -2,13 +2,13 @@ class Balance::ForwardCalculator < Balance::BaseCalculator
  def calculate
    Rails.logger.tagged("Balance::ForwardCalculator") do
      start_cash_balance = derive_cash_balance_on_date_from_total(
-        total_balance: account.opening_anchor_balance,
+        total_balance: 0,
        date: account.opening_anchor_date
      )
-      start_non_cash_balance = account.opening_anchor_balance - start_cash_balance
+      start_non_cash_balance = 0

      calc_start_date.upto(calc_end_date).map do |date|
-        valuation = sync_cache.get_reconciliation_valuation(date)
+        valuation = sync_cache.get_valuation(date)

        if valuation
          end_cash_balance = derive_cash_balance_on_date_from_total(
@@ -21,10 +21,22 @@ class Balance::ForwardCalculator < Balance::BaseCalculator
          end_non_cash_balance = derive_end_non_cash_balance(start_non_cash_balance: start_non_cash_balance, date: date)
        end

+        flows = flows_for_date(date)
+        market_value_change = market_value_change_on_date(date, flows)
+
        output_balance = build_balance(
          date: date,
+          balance: end_cash_balance + end_non_cash_balance,
          cash_balance: end_cash_balance,
-          non_cash_balance: end_non_cash_balance
+          start_cash_balance: start_cash_balance,
+          start_non_cash_balance: start_non_cash_balance,
+          cash_inflows: flows[:cash_inflows],
+          cash_outflows: flows[:cash_outflows],
+          non_cash_inflows: flows[:non_cash_inflows],
+          non_cash_outflows: flows[:non_cash_outflows],
+          cash_adjustments: cash_adjustments_for_date(start_cash_balance, flows[:cash_inflows] - flows[:cash_outflows], valuation),
+          non_cash_adjustments: non_cash_adjustments_for_date(start_non_cash_balance, flows[:non_cash_inflows] - flows[:non_cash_outflows], valuation),
+          net_market_flows: market_value_change
        )

        # Set values for the next iteration
--- a/app/models/balance/reverse_calculator.rb
+++ b/app/models/balance/reverse_calculator.rb
@@ -11,6 +11,8 @@ class Balance::ReverseCalculator < Balance::BaseCalculator

      # Calculates in reverse-chronological order (End of day -> Start of day)
      account.current_anchor_date.downto(account.opening_anchor_date).map do |date|
+        flows = flows_for_date(date)
+
        if use_opening_anchor_for_date?(date)
          end_cash_balance = derive_cash_balance_on_date_from_total(
            total_balance: account.opening_anchor_balance,
@@ -20,29 +22,30 @@ class Balance::ReverseCalculator < Balance::BaseCalculator

          start_cash_balance = end_cash_balance
          start_non_cash_balance = end_non_cash_balance
-
-          build_balance(
-            date: date,
-            cash_balance: end_cash_balance,
-            non_cash_balance: end_non_cash_balance
-          )
+          market_value_change = 0
        else
          start_cash_balance = derive_start_cash_balance(end_cash_balance: end_cash_balance, date: date)
          start_non_cash_balance = derive_start_non_cash_balance(end_non_cash_balance: end_non_cash_balance, date: date)
-
-          # Even though we've just calculated "start" balances, we set today equal to end of day, then use those
-          # in our next iteration (slightly confusing, but just the nature of a "reverse" sync)
-          output_balance = build_balance(
-            date: date,
-            cash_balance: end_cash_balance,
-            non_cash_balance: end_non_cash_balance
-          )
-
-          end_cash_balance = start_cash_balance
-          end_non_cash_balance = start_non_cash_balance
-
-          output_balance
+          market_value_change = market_value_change_on_date(date, flows)
        end
+
+        output_balance = build_balance(
+          date: date,
+          balance: end_cash_balance + end_non_cash_balance,
+          cash_balance: end_cash_balance,
+          start_cash_balance: start_cash_balance,
+          start_non_cash_balance: start_non_cash_balance,
+          cash_inflows: flows[:cash_inflows],
+          cash_outflows: flows[:cash_outflows],
+          non_cash_inflows: flows[:non_cash_inflows],
+          non_cash_outflows: flows[:non_cash_outflows],
+          net_market_flows: market_value_change
+        )
+
+        end_cash_balance = start_cash_balance
+        end_non_cash_balance = start_non_cash_balance
+
+        output_balance
      end
    end
  end
@@ -58,13 +61,6 @@ class Balance::ReverseCalculator < Balance::BaseCalculator
      account.asset? ? entry_flows : -entry_flows
    end

-    # Reverse syncs are a bit different than forward syncs because we do not allow "reconciliation" valuations
-    # to be used at all. This is primarily to keep the code and the UI easy to understand. For a more detailed
-    # explanation, see the test suite.
-    def use_opening_anchor_for_date?(date)
-      account.has_opening_anchor? && date == account.opening_anchor_date
-    end
-
    # Alias method, for algorithmic clarity
    # Derives cash balance, starting from the end-of-day, applying entries in reverse to get the start-of-day balance
    def derive_start_cash_balance(end_cash_balance:, date:)
@@ -76,4 +72,11 @@ class Balance::ReverseCalculator < Balance::BaseCalculator
    def derive_start_non_cash_balance(end_non_cash_balance:, date:)
      derive_non_cash_balance(end_non_cash_balance, date, direction: :reverse)
    end
+
+    # Reverse syncs are a bit different than forward syncs because we do not allow "reconciliation" valuations
+    # to be used at all. This is primarily to keep the code and the UI easy to understand. For a more detailed
+    # explanation, see the test suite.
+    def use_opening_anchor_for_date?(date)
+      account.has_opening_anchor? && date == account.opening_anchor_date
+    end
 end
--- a/app/models/balance/sync_cache.rb
+++ b/app/models/balance/sync_cache.rb
@@ -3,8 +3,8 @@ class Balance::SyncCache
    @account = account
  end

-  def get_reconciliation_valuation(date)
-    converted_entries.find { |e| e.date == date && e.valuation? && e.valuation.reconciliation? }
+  def get_valuation(date)
+    converted_entries.find { |e| e.date == date && e.valuation? }
  end

  def get_holdings(date)