Merge branch 'main' into feat/savings-goals

Signed-off-by: Guillem Arias Fauste <accounts@gariasf.com>
This commit is contained in:
Guillem Arias Fauste
2026-05-13 18:22:55 +02:00
committed by GitHub
267 changed files with 19408 additions and 455 deletions

View File

@@ -23,6 +23,14 @@ class Account < ApplicationRecord
has_many :goal_accounts, dependent: :destroy
has_many :goals, through: :goal_accounts
has_many :goal_contributions, dependent: :destroy
# Inverse for recurring transfers where this account is the destination.
# Account#recurring_transactions only matches account_id; without this
# association, destroying the destination account would hit the FK
# cascade silently and the AR cache wouldn't reflect the deletion.
has_many :inbound_recurring_transfers,
class_name: "RecurringTransaction",
foreign_key: :destination_account_id,
dependent: :destroy
monetize :balance, :cash_balance
@@ -269,6 +277,47 @@ class Account < ApplicationRecord
create_and_sync(attributes, skip_initial_sync: true)
end
def create_from_ibkr_account(ibkr_account)
family = ibkr_account.ibkr_item.family
default_name = if ibkr_account.ibkr_account_id.present?
"Interactive Brokers (#{ibkr_account.ibkr_account_id})"
else
"Interactive Brokers"
end
attributes = {
family: family,
name: default_name,
balance: 0,
cash_balance: 0,
currency: ibkr_account.currency.presence || family.currency,
accountable_type: "Investment",
accountable_attributes: {
subtype: "brokerage"
}
}
create_and_sync(attributes, skip_initial_sync: true)
end
def create_from_kraken_account(kraken_account)
family = kraken_account.kraken_item.family
attributes = {
family: family,
name: kraken_account.name,
balance: (kraken_account.current_balance || 0).to_d,
cash_balance: 0,
currency: kraken_account.currency.presence || family.currency,
accountable_type: "Crypto",
accountable_attributes: {
subtype: "exchange",
tax_treatment: "taxable"
}
}
create_and_sync(attributes, skip_initial_sync: true)
end
private
@@ -301,6 +350,14 @@ class Account < ApplicationRecord
read_attribute(:institution_domain).presence || provider&.institution_domain
end
def manual_crypto_exchange?
accountable_type == "Crypto" &&
accountable&.subtype == "exchange" &&
account_providers.none? &&
plaid_account_id.blank? &&
simplefin_account_id.blank?
end
def logo_url
if institution_domain.present? && Setting.brand_fetch_client_id.present?
logo_size = Setting.brand_fetch_logo_size
@@ -331,15 +388,23 @@ class Account < ApplicationRecord
end
def current_holdings
holdings
.where(currency: currency)
.where.not(qty: 0)
.where(
id: holdings.select("DISTINCT ON (security_id) id")
.where(currency: currency)
.order(:security_id, date: :desc)
)
.order(amount: :desc)
if (provider_snapshot_date = latest_provider_holdings_snapshot_date)
holdings
.where.not(account_provider_id: nil)
.where(date: provider_snapshot_date)
.where.not(qty: 0)
.order(amount: :desc)
else
holdings
.where(currency: currency)
.where.not(qty: 0)
.where(
id: holdings.select("DISTINCT ON (security_id) id")
.where(currency: currency)
.order(:security_id, date: :desc)
)
.order(amount: :desc)
end
end
def latest_provider_holdings_snapshot_date

View File

@@ -51,7 +51,11 @@ class Account::OpeningBalanceManager
end
def oldest_entry_date
@oldest_entry_date ||= account.entries.minimum(:date)
if opening_anchor_valuation&.entry
account.entries.where.not(id: opening_anchor_valuation.entry.id).minimum(:date)
else
account.entries.minimum(:date)
end
end
def default_date

View File

@@ -109,8 +109,28 @@ class Account::ProviderImportAdapter
end
if pending_match
old_pending_external_id = pending_match.external_id
pending_entry_date = pending_match.date
entry = pending_match
entry.assign_attributes(external_id: external_id)
# Clear the pending flag so this entry no longer shows as pending after being claimed
# by a booked transaction. Also record the old external_id so the sync engine can
# exclude it from re-import (preventing the old pending from being recreated on the
# next sync when the stored raw payload still contains the pending transaction data).
if entry.entryable.is_a?(Transaction)
ex = (entry.transaction.extra || {}).deep_dup
Transaction::PENDING_PROVIDERS.each do |provider|
next unless ex.key?(provider)
ex[provider].delete("pending")
ex.delete(provider) if ex[provider].empty?
end
if old_pending_external_id.present?
existing_claims = Array.wrap(ex["auto_claimed_pending_ids"])
ex["auto_claimed_pending_ids"] = (existing_claims + [ old_pending_external_id ]).uniq
end
entry.transaction.extra = ex
end
end
end
@@ -120,7 +140,7 @@ class Account::ProviderImportAdapter
entry.assign_attributes(
amount: amount,
currency: currency,
date: date
date: pending_entry_date || date
)
# Use enrichment pattern to respect user overrides
@@ -551,8 +571,9 @@ class Account::ProviderImportAdapter
# @param external_id [String, nil] Provider's unique ID (optional, for deduplication)
# @param source [String] Provider name
# @param activity_label [String, nil] Investment activity label (e.g., "Buy", "Sell", "Reinvestment")
# @param exchange_rate [BigDecimal, Numeric, nil] Optional provider-supplied FX rate into the account currency
# @return [Entry] The created entry with trade
def import_trade(security:, quantity:, price:, amount:, currency:, date:, name: nil, external_id: nil, source:, activity_label: nil)
def import_trade(security:, quantity:, price:, amount:, currency:, date:, name: nil, external_id: nil, source:, activity_label: nil, exchange_rate: nil)
raise ArgumentError, "security is required" if security.nil?
raise ArgumentError, "source is required" if source.blank?
@@ -585,13 +606,16 @@ class Account::ProviderImportAdapter
end
# Always update Trade attributes (works for both new and existing records)
entry.entryable.assign_attributes(
trade_attributes = {
security: security,
qty: quantity,
price: price,
currency: currency,
investment_activity_label: activity_label || (quantity > 0 ? "Buy" : "Sell")
)
}
trade_attributes[:exchange_rate] = exchange_rate unless exchange_rate.nil?
entry.entryable.assign_attributes(trade_attributes)
entry.assign_attributes(
date: date,

View File

@@ -9,6 +9,7 @@ class Account::Syncer
Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})")
import_market_data
materialize_balances(window_start_date: sync.window_start_date)
apply_provider_balance_overrides
end
def perform_post_sync
@@ -34,4 +35,16 @@ class Account::Syncer
Rails.logger.error("Error syncing market data for account #{account.id}: #{e.message}")
Sentry.capture_exception(e)
end
def apply_provider_balance_overrides
return unless account.linked_to?("IbkrAccount")
ibkr_account = account.account_providers.find_by(provider_type: "IbkrAccount")&.provider
return unless ibkr_account
IbkrAccount::HistoricalBalancesSync.new(ibkr_account).sync!
rescue => e
Rails.logger.error("Error syncing IBKR historical balances for account #{account.id}: #{e.class} - #{e.message}")
Sentry.capture_exception(e)
end
end

View File

@@ -37,10 +37,7 @@ class Balance::SyncCache
@converted_entries ||= account.entries.excluding_split_parents.includes(:entryable).order(:date).to_a.map do |e|
converted_entry = e.dup
# Extract custom exchange rate if present on Transaction
custom_rate = if e.entryable.is_a?(Transaction)
e.entryable.extra&.dig("exchange_rate")
end
custom_rate = e.entryable.exchange_rate if e.entryable.respond_to?(:exchange_rate)
# Use Money#exchange_to with custom rate if available, standard lookup otherwise
converted_entry.amount = converted_entry.amount_money.exchange_to(

204
app/models/brex_account.rb Normal file
View File

@@ -0,0 +1,204 @@
# frozen_string_literal: true
class BrexAccount < ApplicationRecord
include CurrencyNormalizable, Encryptable
CARD_PRIMARY_ACCOUNT_ID = "card_primary"
if encryption_ready?
encrypts :raw_payload
encrypts :raw_transactions_payload
end
belongs_to :brex_item
has_one :account_provider, as: :provider, dependent: :destroy
has_one :account, through: :account_provider, source: :account
validates :name, :currency, presence: true
validates :account_id, uniqueness: { scope: :brex_item_id }
validates :account_kind, inclusion: { in: %w[cash card] }
def self.card_account_id
CARD_PRIMARY_ACCOUNT_ID
end
def self.kind_for(account_data)
return account_data.account_kind if account_data.respond_to?(:account_kind)
data = account_data.with_indifferent_access
kind = data[:account_kind].presence || data[:kind].presence || "cash"
kind.to_s == "credit_card" ? "card" : kind.to_s
end
def self.name_for(account_data)
data = account_data.with_indifferent_access
kind = kind_for(data)
if kind == "card"
data[:name].presence || I18n.t("brex_items.default_card_name", default: "Brex Card")
else
data[:name].presence || data[:display_name].presence || I18n.t("brex_items.default_cash_name", id: data[:id], default: "Brex Cash #{data[:id]}")
end
end
def self.currency_for(account_data)
data = account_data.with_indifferent_access
currency_code_from_money(data[:current_balance] || data[:available_balance] || data[:account_limit])
end
def self.default_account_type_for(account_data)
kind_for(account_data) == "card" ? "CreditCard" : "Depository"
end
def self.default_accountable_attributes(accountable_type)
case accountable_type
when "CreditCard"
{ subtype: CreditCard::DEFAULT_SUBTYPE }
when "Depository"
{ subtype: Depository::DEFAULT_SUBTYPE }
else
{}
end
end
def self.money_to_decimal(money_payload)
return nil if money_payload.blank?
payload = money_payload.is_a?(Hash) ? money_payload.with_indifferent_access : { amount: money_payload, currency: "USD" }
amount = payload[:amount]
return nil if amount.nil?
currency = currency_code_from_money(payload)
divisor = Money::Currency.new(currency).minor_unit_conversion
BigDecimal(amount.to_s) / BigDecimal(divisor.to_s)
rescue Money::Currency::UnknownCurrencyError, ArgumentError
Rails.logger.warn("Invalid Brex money payload #{money_payload.inspect}, defaulting conversion to USD")
begin
safe_amount = BigDecimal(payload[:amount].to_s)
safe_amount / BigDecimal(Money::Currency.new("USD").minor_unit_conversion.to_s)
rescue ArgumentError, TypeError
BigDecimal("0")
end
end
def self.currency_code_from_money(money_payload)
payload = money_payload.is_a?(Hash) ? money_payload.with_indifferent_access : {}
currency = payload[:currency].presence || "USD"
Money::Currency.new(currency).iso_code
rescue Money::Currency::UnknownCurrencyError
"USD"
end
def self.sanitize_payload(payload)
case payload
when Array
payload.map { |value| sanitize_payload(value) }
when Hash
payload.each_with_object({}) do |(key, value), sanitized|
key_string = key.to_s
normalized_key = key_string.downcase
if sensitive_number_key?(normalized_key)
sanitized["#{key_string}_last4"] = last_four(value)
elsif normalized_key == "card_metadata"
sanitized[key_string] = sanitize_card_metadata(value)
elsif sensitive_secret_key?(normalized_key)
sanitized[key_string] = "[FILTERED]"
else
sanitized[key_string] = sanitize_payload(value)
end
end
else
payload
end
end
def self.last_four(value)
digits = value.to_s.gsub(/\D/, "")
digits.last(4) if digits.present?
end
def self.sanitize_card_metadata(value)
return nil unless value.is_a?(Hash)
metadata = value.with_indifferent_access
{
"card_id" => metadata[:card_id].presence || metadata[:id].presence,
"card_name" => metadata[:card_name].presence || metadata[:name].presence,
"card_type" => metadata[:card_type].presence || metadata[:type].presence,
"last_four" => last_four(metadata[:last_four].presence || metadata[:last4].presence || metadata[:card_last_four].presence)
}.compact
end
def current_account
account
end
def linked_account
account
end
def cash?
account_kind == "cash"
end
def card?
account_kind == "card"
end
def upsert_brex_snapshot!(account_snapshot)
snapshot = account_snapshot.with_indifferent_access
kind = snapshot[:account_kind].presence || snapshot[:kind].presence || "cash"
kind = "card" if kind.to_s == "credit_card"
update!(
current_balance: self.class.money_to_decimal(snapshot[:current_balance]),
available_balance: self.class.money_to_decimal(snapshot[:available_balance]),
account_limit: self.class.money_to_decimal(snapshot[:account_limit]),
currency: self.class.currency_code_from_money(snapshot[:current_balance] || snapshot[:available_balance] || snapshot[:account_limit]),
name: self.class.name_for(snapshot.merge(account_kind: kind)),
account_id: snapshot[:id]&.to_s,
account_kind: kind,
account_status: snapshot[:status],
account_type: snapshot[:type],
provider: "brex",
institution_metadata: build_institution_metadata(snapshot, kind),
raw_payload: self.class.sanitize_payload(account_snapshot)
)
end
def upsert_brex_transactions_snapshot!(transactions_snapshot)
update!(
raw_transactions_payload: self.class.sanitize_payload(transactions_snapshot)
)
end
private
def self.sensitive_number_key?(normalized_key)
normalized_key.in?(%w[account_number routing_number pan primary_account_number card_number])
end
def self.sensitive_secret_key?(normalized_key)
normalized_key.include?("token") ||
normalized_key.include?("secret") ||
normalized_key.in?(%w[api_key access_key authorization cvc cvv security_code])
end
private_class_method :sensitive_number_key?, :sensitive_secret_key?
def build_institution_metadata(snapshot, kind)
{
name: "Brex",
domain: "brex.com",
url: "https://brex.com",
account_kind: kind,
account_type: snapshot[:type],
primary: snapshot[:primary],
account_number_last4: self.class.last_four(snapshot[:account_number]),
routing_number_last4: self.class.last_four(snapshot[:routing_number]),
status: snapshot[:status],
current_statement_period: self.class.sanitize_payload(snapshot[:current_statement_period])
}.compact
end
end

View File

@@ -0,0 +1,78 @@
# frozen_string_literal: true
class BrexAccount::Processor
include CurrencyNormalizable
attr_reader :brex_account
def initialize(brex_account)
@brex_account = brex_account
end
def process
unless brex_account.current_account.present?
Rails.logger.info "BrexAccount::Processor - No linked account for brex_account #{brex_account.id}, skipping processing"
return
end
process_account!
process_transactions
rescue StandardError => e
Rails.logger.error "BrexAccount::Processor - Failed to process account #{brex_account.id}: #{e.message}"
report_exception(e, "account")
raise
end
private
def process_account!
account = brex_account.current_account
balance = brex_account.current_balance
currency = parse_currency(brex_account.currency)
if balance.nil?
Rails.logger.warn "BrexAccount::Processor - current_balance is nil for brex_account #{brex_account.id}, defaulting to 0"
balance = 0
end
if currency.nil?
Rails.logger.warn "BrexAccount::Processor - currency parse failed for brex_account #{brex_account.id}: #{brex_account.currency.inspect}, defaulting to USD"
Sentry.capture_message("BrexAccount currency parse failed", level: :warning) do |scope|
scope.set_tags(brex_account_id: brex_account.id)
scope.set_context("brex_account", {
id: brex_account.id,
currency: brex_account.currency
})
end
currency = "USD"
end
account.update!(
balance: balance,
cash_balance: balance,
currency: currency
)
if account.accountable_type == "CreditCard" && brex_account.available_balance.present?
account.accountable.update!(available_credit: brex_account.available_balance)
end
end
# Transaction import errors are logged and swallowed so balance sync can continue.
def process_transactions
BrexAccount::Transactions::Processor.new(brex_account).process
rescue StandardError => e
Rails.logger.error "BrexAccount::Processor - Failed to process transactions for brex_account #{brex_account.id}: #{e.message}"
Rails.logger.error Array(e.backtrace).first(10).join("\n")
report_exception(e, "transactions")
end
def report_exception(error, context)
Sentry.capture_exception(error) do |scope|
scope.set_tags(
brex_account_id: brex_account.id,
context: context
)
end
end
end

View File

@@ -0,0 +1,83 @@
class BrexAccount::Transactions::Processor
attr_reader :brex_account
def initialize(brex_account)
@brex_account = brex_account
end
def process
unless brex_account.raw_transactions_payload.present?
Rails.logger.info "BrexAccount::Transactions::Processor - No transactions in raw_transactions_payload for brex_account #{brex_account.id}"
return { success: true, total: 0, imported: 0, skipped: 0, failed: 0, errors: [], skipped_transactions: [] }
end
total_count = brex_account.raw_transactions_payload.count
Rails.logger.info "BrexAccount::Transactions::Processor - Processing #{total_count} transactions for brex_account #{brex_account.id}"
imported_count = 0
failed_count = 0
skipped_count = 0
errors = []
skipped = []
# Each entry is processed inside a transaction, but to avoid locking up the DB when
# there are hundreds or thousands of transactions, we process them individually.
brex_account.raw_transactions_payload.each_with_index do |transaction_data, index|
begin
result = BrexEntry::Processor.new(
transaction_data,
brex_account: brex_account
).process
if result == :skipped
skipped_count += 1
skipped << { index: index, transaction_id: transaction_id_for(transaction_data), reason: "No linked account" }
elsif result.nil?
failed_count += 1
errors << { index: index, transaction_id: transaction_id_for(transaction_data), error: "No transaction imported" }
else
imported_count += 1
end
rescue ArgumentError => e
# Validation error - log and continue
failed_count += 1
transaction_id = transaction_id_for(transaction_data)
error_message = "Validation error: #{e.message}"
Rails.logger.error "BrexAccount::Transactions::Processor - #{error_message} (transaction #{transaction_id})"
errors << { index: index, transaction_id: transaction_id, error: error_message }
rescue => e
# Unexpected error - log with full context and continue
failed_count += 1
transaction_id = transaction_id_for(transaction_data)
error_message = "#{e.class}: #{e.message}"
Rails.logger.error "BrexAccount::Transactions::Processor - Error processing transaction #{transaction_id}: #{error_message}"
Rails.logger.error Array(e.backtrace).first(10).join("\n")
errors << { index: index, transaction_id: transaction_id, error: error_message }
end
end
result = {
success: failed_count == 0,
total: total_count,
imported: imported_count,
skipped: skipped_count,
failed: failed_count,
errors: errors,
skipped_transactions: skipped
}
if failed_count > 0
Rails.logger.warn "BrexAccount::Transactions::Processor - Completed with #{failed_count} failures out of #{total_count} transactions"
else
Rails.logger.info "BrexAccount::Transactions::Processor - Successfully processed #{imported_count} transactions"
end
result
end
private
def transaction_id_for(transaction_data)
transaction_data&.dig(:id) || transaction_data&.dig("id") || "unknown"
end
end

View File

@@ -0,0 +1,180 @@
# frozen_string_literal: true
require "digest/md5"
class BrexEntry::Processor
include CurrencyNormalizable
def initialize(brex_transaction, brex_account:)
@brex_transaction = brex_transaction
@brex_account = brex_account
end
def process
cached_external_id = nil
cached_external_id = external_id
unless account.present?
Rails.logger.warn "BrexEntry::Processor - No linked account for brex_account #{brex_account.id}, skipping transaction #{cached_external_id}"
return :skipped
end
import_adapter.import_transaction(
external_id: cached_external_id,
amount: amount,
currency: currency,
date: date,
name: name,
source: "brex",
merchant: merchant,
notes: notes,
extra: extra
)
rescue ArgumentError => e
Rails.logger.error "BrexEntry::Processor - Validation error for transaction #{cached_external_id || safe_external_id}: #{e.message}"
raise
rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e
Rails.logger.error "BrexEntry::Processor - Failed to save transaction #{cached_external_id || safe_external_id}: #{e.message}"
raise StandardError.new("Failed to import transaction: #{e.message}")
rescue => e
Rails.logger.error "BrexEntry::Processor - Unexpected error processing transaction #{cached_external_id || safe_external_id}: #{e.class} - #{e.message}"
Rails.logger.error Array(e.backtrace).join("\n")
raise StandardError.new("Unexpected error importing transaction: #{e.message}")
end
private
attr_reader :brex_transaction, :brex_account
def import_adapter
@import_adapter ||= Account::ProviderImportAdapter.new(account)
end
def account
@account ||= brex_account.current_account
end
def data
@data ||= brex_transaction.with_indifferent_access
end
def external_id
id = data[:id].presence
raise ArgumentError, "Brex transaction missing required field 'id'" unless id
"brex_#{id}"
end
def safe_external_id
external_id
rescue ArgumentError
"brex_unknown"
end
def name
data[:description].presence ||
merchant_payload[:raw_descriptor].presence ||
merchant_payload[:name].presence ||
I18n.t("brex_items.entries.default_name")
end
def notes
note_parts = []
note_parts << data[:type] if data[:type].present?
note_parts << data[:expense_id] if data[:expense_id].present?
note_parts.any? ? note_parts.join(" - ") : nil
end
def merchant
merchant_name = merchant_payload[:raw_descriptor].presence || merchant_payload[:name].presence
return @merchant if instance_variable_defined?(:@merchant)
return @merchant = nil if merchant_name.blank?
merchant_name = merchant_name.to_s.strip
return @merchant = nil if merchant_name.blank?
merchant_id = Digest::MD5.hexdigest(merchant_name.downcase)
@merchant = import_adapter.find_or_create_merchant(
provider_merchant_id: "brex_merchant_#{merchant_id}",
name: merchant_name,
source: "brex"
)
rescue ActiveRecord::RecordInvalid => e
Rails.logger.error "BrexEntry::Processor - Failed to create merchant '#{merchant_name}': #{e.message}"
@merchant = nil
end
def merchant_payload
@merchant_payload ||= begin
payload = data[:merchant]
payload.is_a?(Hash) ? payload.with_indifferent_access : {}
end
end
def amount
BrexAccount.money_to_decimal(data[:amount]) || BigDecimal("0")
rescue ArgumentError => e
Rails.logger.error "Failed to parse Brex transaction amount: #{data[:amount].inspect} - #{e.message}"
raise
end
def currency
amount_currency = transaction_amount_currency
log_invalid_currency(amount_currency) if amount_currency.blank? && data[:amount].present?
parse_currency(amount_currency) ||
parse_currency(brex_account.currency) ||
"USD"
end
def transaction_amount_currency
amount_payload = data[:amount]
return nil unless amount_payload.is_a?(Hash)
amount_payload.with_indifferent_access[:currency]
end
def log_invalid_currency(currency_value)
Rails.logger.warn(
"Invalid Brex currency #{currency_value.inspect} for transaction #{data[:id].presence || 'unknown'} " \
"on brex_account #{brex_account.id} amount=#{data[:amount].inspect} account_currency=#{brex_account.currency.inspect}; defaulting to fallback"
)
end
def date
date_value = data[:posted_at_date].presence || data[:initiated_at_date].presence
case date_value
when String
Date.parse(date_value)
when Integer, Float
Time.at(date_value).to_date
when Time, DateTime
date_value.to_date
when Date
date_value
else
raise ArgumentError, "Invalid date format: #{date_value.inspect}"
end
rescue ArgumentError, TypeError => e
Rails.logger.error("Failed to parse Brex transaction date '#{date_value}': #{e.message}")
raise ArgumentError, "Unable to parse transaction date: #{date_value.inspect}"
end
def extra
{
brex: {
transaction_id: data[:id],
account_kind: brex_account.account_kind,
type: data[:type],
card_id: data[:card_id],
transfer_id: data[:transfer_id],
expense_id: data[:expense_id],
card_transaction_operation_reference_id: data[:card_transaction_operation_reference_id],
initiated_at_date: data[:initiated_at_date],
posted_at_date: data[:posted_at_date],
merchant: BrexAccount.sanitize_payload(data[:merchant])
}.compact
}
end
end

197
app/models/brex_item.rb Normal file
View File

@@ -0,0 +1,197 @@
class BrexItem < ApplicationRecord
include Syncable, Provided, Unlinking, Encryptable
BLANK_TOKEN_SENTINELS = [ "", " ", " ", " ", "\t", "\n", "\r" ].freeze
enum :status, { good: "good", requires_update: "requires_update" }, default: :good
if encryption_ready?
encrypts :token, deterministic: true
encrypts :raw_payload
end
validates :name, presence: true
validates :token, presence: true, on: :create
validate :base_url_must_be_official_brex_url
validate :token_cannot_be_blank_when_changed
before_validation :normalize_token
before_validation :normalize_base_url
belongs_to :family
has_one_attached :logo, dependent: :purge_later
has_many :brex_accounts, dependent: :destroy
has_many :accounts, through: :brex_accounts
scope :active, -> { where(scheduled_for_deletion: false) }
scope :syncable, -> { active }
scope :ordered, -> { order(created_at: :desc) }
scope :needs_update, -> { where(status: :requires_update) }
scope :with_credentials, -> { where.not(token: [ nil, *BLANK_TOKEN_SENTINELS ]).where("BTRIM(token) <> ''") }
def self.resolve_for(family:, brex_item_id: nil)
normalized_id = brex_item_id.to_s.strip.presence
if normalized_id.present?
return family.brex_items.active.with_credentials.find_by(id: normalized_id)
end
credentialed_items = family.brex_items.active.with_credentials.ordered
credentialed_items.first if credentialed_items.one?
end
def destroy_later
update!(scheduled_for_deletion: true)
DestroyJob.perform_later(self)
end
def import_latest_brex_data(sync_start_date: nil)
provider = brex_provider
unless provider
Rails.logger.error "BrexItem #{id} - Cannot import: provider is not configured"
raise Provider::Brex::BrexError.new("Brex provider is not configured", :not_configured)
end
BrexItem::Importer.new(self, brex_provider: provider, sync_start_date: sync_start_date).import
rescue => e
Rails.logger.error "BrexItem #{id} - Failed to import data: #{e.message}"
raise
end
def process_accounts
return [] if brex_accounts.empty?
results = []
brex_accounts.joins(:account).includes(:account).merge(Account.visible).each do |brex_account|
begin
result = BrexAccount::Processor.new(brex_account).process
results << { brex_account_id: brex_account.id, success: true, result: result }
rescue => e
Rails.logger.error "BrexItem #{id} - Failed to process account #{brex_account.id}: #{e.message}"
results << { brex_account_id: brex_account.id, success: false, error: e.message }
end
end
results
end
def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil)
return [] if accounts.empty?
results = []
accounts.visible.each do |account|
begin
account.sync_later(
parent_sync: parent_sync,
window_start_date: window_start_date,
window_end_date: window_end_date
)
results << { account_id: account.id, success: true }
rescue => e
Rails.logger.error "BrexItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}"
results << { account_id: account.id, success: false, error: e.message }
end
end
results
end
def upsert_brex_snapshot!(accounts_snapshot)
update!(raw_payload: BrexAccount.sanitize_payload(accounts_snapshot))
end
def has_completed_initial_setup?
# Setup is complete if we have any linked accounts
accounts.any?
end
def sync_status_summary
total_accounts = total_accounts_count
linked_count = linked_accounts_count
unlinked_count = unlinked_accounts_count
if total_accounts == 0
I18n.t("brex_items.sync_status.no_accounts")
elsif unlinked_count == 0
I18n.t("brex_items.sync_status.all_synced", count: linked_count)
else
I18n.t("brex_items.sync_status.partial_setup", synced: linked_count, pending: unlinked_count)
end
end
def linked_accounts_count
brex_accounts.joins(:account_provider).count
end
def unlinked_accounts_count
brex_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).count
end
def total_accounts_count
brex_accounts.count
end
def institution_display_name
institution_name.presence || institution_domain.presence || name
end
def connected_institutions
brex_accounts.where.not(institution_metadata: nil)
.pluck(:institution_metadata)
.compact
.uniq { |inst| inst["name"] || inst["institution_name"] }
end
def institution_summary
institutions = connected_institutions
case institutions.count
when 0
I18n.t("brex_items.institution_summary.none")
when 1
name = institutions.first["name"] ||
institutions.first["institution_name"] ||
I18n.t("brex_items.institution_summary.count", count: 1)
I18n.t("brex_items.institution_summary.one", name: name)
else
I18n.t("brex_items.institution_summary.count", count: institutions.count)
end
end
def credentials_configured?
token.to_s.strip.present?
end
def effective_base_url
return Provider::Brex::DEFAULT_BASE_URL if base_url.blank?
Provider::Brex.normalize_base_url(base_url)
end
private
def normalize_token
self.token = token&.strip
end
def token_cannot_be_blank_when_changed
return unless persisted? && will_save_change_to_token? && token.blank?
errors.add(:token, :blank)
end
def normalize_base_url
stripped = base_url.to_s.strip
if stripped.blank?
self.base_url = nil
return
end
normalized = Provider::Brex.normalize_base_url(stripped)
self.base_url = normalized if normalized.present?
end
def base_url_must_be_official_brex_url
return if base_url.blank? || Provider::Brex.allowed_base_url?(base_url)
errors.add(:base_url, :official_hosts_only)
end
end

View File

@@ -0,0 +1,425 @@
# frozen_string_literal: true
class BrexItem::AccountFlow
require_dependency "brex_item/account_flow/setup"
include Setup
CACHE_TTL = 5.minutes
class NoApiTokenError < StandardError; end
class AccountNotFoundError < StandardError; end
class InvalidAccountNameError < StandardError; end
class AccountAlreadyLinkedError < StandardError; end
NavigationResult = Data.define(:target, :flash_type, :message)
SelectionResult = Data.define(:status, :brex_item, :available_accounts, :accountable_type, :message) do
def success? = status == :success
def setup_required? = status == :setup_required
def provider_error? = status.in?([ :api_error, :unexpected_error ])
end
LinkAccountsResult = Data.define(:created_accounts, :already_linked_names, :invalid_account_ids) do
def created_count = created_accounts.count
def already_linked_count = already_linked_names.count
def invalid_count = invalid_account_ids.count
end
SetupResult = Data.define(:created_accounts, :skipped_count, :failed_count) do
def created_count = created_accounts.count
end
SetupCompletion = Data.define(:success, :message) do
def success? = success
end
attr_reader :family, :brex_item_id, :brex_item, :credentialed_items
def initialize(family:, brex_item_id: nil, brex_item: nil)
@family = family
@brex_item_id = brex_item_id.to_s.strip.presence
@credentialed_items = family.brex_items.active.with_credentials.ordered
@brex_item = brex_item || BrexItem.resolve_for(family: family, brex_item_id: @brex_item_id)
end
def self.cache_key(family, brex_item)
"brex_accounts_#{family.id}_#{brex_item.id}"
end
def self.cache_sensitive_update?(permitted_params)
permitted_params.key?(:token) || permitted_params.key?(:base_url)
end
def self.update_item_with_cache_expiration(brex_item, family:, attributes:)
expire_accounts_cache = cache_sensitive_update?(attributes)
updated = brex_item.update(attributes)
Rails.cache.delete(cache_key(family, brex_item)) if updated && expire_accounts_cache
updated
end
def selected?
brex_item.present?
end
def selection_required?
credentialed_items.count > 1 && brex_item_id.blank?
end
def preload_payload
return selection_error_payload if !selected?
return { success: false, error: "no_credentials", has_accounts: false } unless brex_item.credentials_configured?
cached_accounts = Rails.cache.read(cache_key)
cached = !cached_accounts.nil?
available_accounts = cached ? cached_accounts : fetch_and_cache_accounts
{ success: true, has_accounts: available_accounts.any?, cached: cached }
rescue NoApiTokenError
{ success: false, error: "no_api_token", has_accounts: false }
rescue Provider::Brex::BrexError => e
Rails.logger.error("Brex preload error: #{e.message}")
{ success: false, error: "api_error", error_message: e.message, has_accounts: nil }
rescue StandardError => e
Rails.logger.error("Unexpected error preloading Brex accounts: #{e.class}: #{e.message}")
{ success: false, error: "unexpected_error", error_message: I18n.t("brex_items.errors.unexpected_error"), has_accounts: nil }
end
def select_accounts_result(accountable_type:)
selection_result_for(
scope: "brex_items.select_accounts",
accountable_type: accountable_type,
empty_message_key: "no_accounts_found",
log_context: "select_accounts"
)
end
def select_existing_account_result(account:)
return linked_account_result if account.account_providers.exists?
selection_result_for(
scope: "brex_items.select_existing_account",
accountable_type: account.accountable_type,
empty_message_key: "all_accounts_already_linked",
log_context: "select_existing_account"
)
end
def link_new_accounts_result(account_ids:, accountable_type:)
return navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.no_accounts_selected")) if account_ids.blank?
return navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.invalid_account_type")) unless supported_account_type?(accountable_type)
return navigation(:settings_providers, :alert, I18n.t("brex_items.link_accounts.select_connection")) unless selected?
link_navigation_result(link_new_accounts!(account_ids: account_ids, accountable_type: accountable_type))
rescue NoApiTokenError
navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.no_api_token"))
rescue Provider::Brex::BrexError => e
navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.api_error", message: e.message))
rescue StandardError => e
Rails.logger.error("Brex account linking failed: #{e.class} - #{e.message}")
Rails.logger.error(Array(e.backtrace).first(10).join("\n"))
navigation(:new_account, :alert, I18n.t("brex_items.errors.unexpected_error"))
end
def link_existing_account_result(account:, brex_account_id:)
return navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.missing_parameters")) if account.blank? || brex_account_id.blank?
return navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.account_already_linked")) if account.account_providers.exists?
return navigation(:settings_providers, :alert, I18n.t("brex_items.link_existing_account.select_connection")) unless selected?
link_existing_account!(account: account, brex_account_id: brex_account_id)
navigation(:return_to_or_accounts, :notice, I18n.t("brex_items.link_existing_account.success", account_name: account.name))
rescue NoApiTokenError
navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.no_api_token"))
rescue AccountNotFoundError
navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.provider_account_not_found"))
rescue InvalidAccountNameError
navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.invalid_account_name"))
rescue AccountAlreadyLinkedError
navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.provider_account_already_linked"))
rescue Provider::Brex::BrexError => e
navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.api_error", message: e.message))
rescue StandardError => e
Rails.logger.error("Brex existing account linking failed: #{e.class} - #{e.message}")
Rails.logger.error(Array(e.backtrace).first(10).join("\n"))
navigation(:accounts, :alert, I18n.t("brex_items.errors.unexpected_error"))
end
def link_new_accounts!(account_ids:, accountable_type:)
raise ArgumentError, "Unsupported Brex account type: #{accountable_type}" unless supported_account_type?(accountable_type)
created_accounts = []
already_linked_names = []
invalid_account_ids = []
accounts_by_id = indexed_accounts
ActiveRecord::Base.transaction do
account_ids.each do |account_id|
account_data = accounts_by_id[account_id.to_s]
next unless account_data
account_name = BrexAccount.name_for(account_data)
if account_name.blank?
invalid_account_ids << account_id
Rails.logger.warn "BrexItem::AccountFlow - Skipping account #{account_id} with blank name"
next
end
brex_account = upsert_brex_account!(account_id, account_data)
if brex_account.account_provider.present?
already_linked_names << account_name
next
end
account = Account.create_and_sync(
{
family: family,
name: account_name,
balance: 0,
currency: BrexAccount.currency_for(account_data),
accountable_type: accountable_type,
accountable_attributes: BrexAccount.default_accountable_attributes(accountable_type)
},
skip_initial_sync: true
)
AccountProvider.create!(account: account, provider: brex_account)
created_accounts << account
end
end
brex_item.sync_later if created_accounts.any?
LinkAccountsResult.new(
created_accounts: created_accounts,
already_linked_names: already_linked_names,
invalid_account_ids: invalid_account_ids
)
end
def link_existing_account!(account:, brex_account_id:)
account_data = indexed_accounts[brex_account_id.to_s]
raise AccountNotFoundError unless account_data
account_name = BrexAccount.name_for(account_data)
raise InvalidAccountNameError if account_name.blank?
brex_account = nil
ActiveRecord::Base.transaction do
brex_account = upsert_brex_account!(brex_account_id, account_data)
raise AccountAlreadyLinkedError if brex_account.account_provider.present?
AccountProvider.create!(account: account, provider: brex_account)
end
brex_item.sync_later
brex_account
end
private
def selection_error_payload
if brex_item_id.present?
return {
success: false,
error: "select_connection",
error_message: I18n.t("brex_items.select_accounts.select_connection"),
has_accounts: nil
}
end
return { success: false, error: "no_credentials", has_accounts: false } unless selection_required?
{
success: false,
error: "select_connection",
error_message: I18n.t("brex_items.select_accounts.select_connection"),
has_accounts: nil
}
end
def selection_failure_result(scope, accountable_type: nil)
if selection_required?
SelectionResult.new(
status: :select_connection,
brex_item: nil,
available_accounts: [],
accountable_type: accountable_type,
message: I18n.t("#{scope}.select_connection")
)
else
SelectionResult.new(
status: :setup_required,
brex_item: nil,
available_accounts: [],
accountable_type: accountable_type,
message: I18n.t("#{scope}.no_credentials_configured")
)
end
end
def selection_result_for(scope:, accountable_type:, empty_message_key:, log_context:)
return selection_failure_result(scope, accountable_type: accountable_type) unless selected?
available_accounts = filter_accounts(unlinked_available_accounts, accountable_type)
if available_accounts.empty?
return selection_result(
status: :empty,
accountable_type: accountable_type,
message: I18n.t("#{scope}.#{empty_message_key}")
)
end
selection_result(status: :success, accountable_type: accountable_type, available_accounts: available_accounts)
rescue NoApiTokenError
selection_result(
status: :no_api_token,
accountable_type: accountable_type,
message: I18n.t("#{scope}.no_api_token")
)
rescue Provider::Brex::BrexError => e
Rails.logger.error("Brex API error in #{log_context}: #{e.message}")
selection_result(status: :api_error, accountable_type: accountable_type, message: e.message)
rescue StandardError => e
Rails.logger.error("Unexpected error in #{log_context}: #{e.class}: #{e.message}")
selection_result(
status: :unexpected_error,
accountable_type: accountable_type,
message: I18n.t("#{scope}.unexpected_error")
)
end
def selection_result(status:, accountable_type:, available_accounts: [], message: nil)
SelectionResult.new(
status: status,
brex_item: brex_item,
available_accounts: available_accounts,
accountable_type: accountable_type,
message: message
)
end
def linked_account_result
SelectionResult.new(
status: :account_already_linked,
brex_item: brex_item,
available_accounts: [],
accountable_type: nil,
message: I18n.t("brex_items.select_existing_account.account_already_linked")
)
end
def link_navigation_result(result)
if result.invalid_count.positive? && result.created_count.zero? && result.already_linked_count.zero?
navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.invalid_account_names", count: result.invalid_count))
elsif result.invalid_count.positive? && (result.created_count.positive? || result.already_linked_count.positive?)
navigation(
:return_to_or_accounts,
:alert,
I18n.t(
"brex_items.link_accounts.partial_invalid",
created_count: result.created_count,
already_linked_count: result.already_linked_count,
invalid_count: result.invalid_count
)
)
elsif result.created_count.positive? && result.already_linked_count.positive?
navigation(
:return_to_or_accounts,
:notice,
I18n.t(
"brex_items.link_accounts.partial_success",
created_count: result.created_count,
already_linked_count: result.already_linked_count,
already_linked_names: result.already_linked_names.join(", ")
)
)
elsif result.created_count.positive?
navigation(:return_to_or_accounts, :notice, I18n.t("brex_items.link_accounts.success", count: result.created_count))
elsif result.already_linked_count.positive?
navigation(
:return_to_or_accounts,
:alert,
I18n.t(
"brex_items.link_accounts.all_already_linked",
count: result.already_linked_count,
names: result.already_linked_names.join(", ")
)
)
else
navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.link_failed"))
end
end
def navigation(target, flash_type, message)
NavigationResult.new(target: target, flash_type: flash_type, message: message)
end
def cache_key
self.class.cache_key(family, brex_item)
end
def fetch_accounts
provider = brex_item&.brex_provider
raise NoApiTokenError unless provider.present?
accounts_data = provider.get_accounts
accounts_data[:accounts] || []
end
def accounts
cached_accounts = Rails.cache.read(cache_key)
return cached_accounts unless cached_accounts.nil?
fetch_and_cache_accounts
end
def fetch_and_cache_accounts
available_accounts = fetch_accounts
Rails.cache.write(cache_key, available_accounts, expires_in: CACHE_TTL)
available_accounts
end
def unlinked_available_accounts
linked_account_ids = brex_item.brex_accounts
.joins(:account_provider)
.pluck("#{BrexAccount.table_name}.account_id")
.map(&:to_s)
accounts.reject { |account| linked_account_ids.include?(account.with_indifferent_access[:id].to_s) }
end
def filter_accounts(accounts, accountable_type)
return [] unless Provider::BrexAdapter.supported_account_types.include?(accountable_type)
accounts.select do |account|
case accountable_type
when "CreditCard"
BrexAccount.kind_for(account) == "card"
when "Depository"
BrexAccount.kind_for(account) == "cash"
else
true
end
end
end
def indexed_accounts
accounts.index_by { |account| account.with_indifferent_access[:id].to_s }
end
def upsert_brex_account!(account_id, account_data)
brex_account = brex_item.brex_accounts.find_or_initialize_by(account_id: account_id.to_s)
brex_account.upsert_brex_snapshot!(account_data)
brex_account
end
def supported_account_type?(accountable_type)
Provider::BrexAdapter.supported_account_types.include?(accountable_type)
end
end

View File

@@ -0,0 +1,242 @@
# frozen_string_literal: true
class BrexItem::AccountFlow
module Setup
def import_accounts_from_api_if_needed
raise NoApiTokenError unless brex_item&.credentials_configured?
available_accounts = fetch_accounts
return nil if available_accounts.empty?
existing_accounts = brex_item.brex_accounts.index_by(&:account_id)
available_accounts.each do |account_data|
account_id = account_data.with_indifferent_access[:id].to_s
account_name = BrexAccount.name_for(account_data)
next if account_id.blank? || account_name.blank?
brex_account = existing_accounts[account_id]
next if brex_account.present? && !brex_account_snapshot_changed?(brex_account, account_data)
upsert_brex_account!(account_id, account_data)
end
nil
end
def unlinked_brex_accounts
brex_item.brex_accounts
.left_joins(:account_provider)
.where(account_providers: { id: nil })
end
def account_type_options
supported_types = Provider::BrexAdapter.supported_account_types
account_type_keys = {
"depository" => "Depository",
"credit_card" => "CreditCard",
"investment" => "Investment",
"loan" => "Loan",
"other_asset" => "OtherAsset"
}
options = account_type_keys.filter_map do |key, type|
next unless supported_types.include?(type)
[ I18n.t("brex_items.setup_accounts.account_types.#{key}"), type ]
end
[ [ I18n.t("brex_items.setup_accounts.account_types.skip"), "skip" ] ] + options
end
def displayable_account_type_options
account_type_options.reject { |_, type| type == "skip" }
end
def subtype_options
supported_types = Provider::BrexAdapter.supported_account_types
all_subtype_options = {
"Depository" => {
label: I18n.t("brex_items.setup_accounts.subtype_labels.depository"),
options: translate_subtypes("depository", Depository::SUBTYPES)
},
"CreditCard" => {
label: I18n.t("brex_items.setup_accounts.subtype_labels.credit_card"),
options: [],
message: I18n.t("brex_items.setup_accounts.subtype_messages.credit_card")
},
"Investment" => {
label: I18n.t("brex_items.setup_accounts.subtype_labels.investment"),
options: translate_subtypes("investment", Investment::SUBTYPES)
},
"Loan" => {
label: I18n.t("brex_items.setup_accounts.subtype_labels.loan"),
options: translate_subtypes("loan", Loan::SUBTYPES)
},
"OtherAsset" => {
label: I18n.t("brex_items.setup_accounts.subtype_labels.other_asset", default: "Other asset"),
options: [],
message: I18n.t("brex_items.setup_accounts.subtype_messages.other_asset")
}
}
all_subtype_options.slice(*supported_types)
end
def complete_setup!(account_types:, account_subtypes:)
created_accounts = []
skipped_count = 0
valid_types = Provider::BrexAdapter.supported_account_types
failed_count = 0
submitted_brex_accounts = brex_item.brex_accounts
.where(id: account_types.keys)
.includes(:account_provider)
.index_by { |brex_account| brex_account.id.to_s }
account_types.each do |brex_account_id, selected_type|
if selected_type == "skip" || selected_type.blank?
skipped_count += 1
next
end
unless valid_types.include?(selected_type)
Rails.logger.warn("Invalid account type '#{selected_type}' submitted for Brex account #{brex_account_id}")
skipped_count += 1
next
end
brex_account = submitted_brex_accounts[brex_account_id.to_s]
unless brex_account
Rails.logger.warn("Brex account #{brex_account_id} not found for item #{brex_item.id}")
next
end
if brex_account.account_provider.present?
Rails.logger.info("Brex account #{brex_account_id} already linked, skipping")
next
end
selected_subtype = selected_subtype_for(
selected_type: selected_type,
submitted_subtype: account_subtypes[brex_account_id]
)
begin
ActiveRecord::Base.transaction do
account = Account.create_and_sync(
{
family: family,
name: brex_account.name,
balance: brex_account.current_balance || 0,
currency: brex_account.currency.presence || family.currency,
accountable_type: selected_type,
accountable_attributes: selected_subtype.present? ? { subtype: selected_subtype } : {}
},
skip_initial_sync: true
)
AccountProvider.create!(account: account, provider: brex_account)
created_accounts << account
end
rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e
failed_count += 1
Rails.logger.error("Brex account setup failed for #{brex_account_id}: #{e.class} - #{e.message}")
Rails.logger.error(Array(e.backtrace).first(10).join("\n"))
end
end
brex_item.sync_later if created_accounts.any?
SetupResult.new(created_accounts: created_accounts, skipped_count: skipped_count, failed_count: failed_count)
end
def import_accounts_with_user_facing_error
import_accounts_from_api_if_needed
rescue NoApiTokenError
I18n.t("brex_items.setup_accounts.no_api_token")
rescue Provider::Brex::BrexError => e
Rails.logger.error("Brex API error: #{e.message}")
I18n.t("brex_items.setup_accounts.api_error", message: e.message)
rescue StandardError => e
Rails.logger.error("Unexpected error fetching Brex accounts: #{e.class}: #{e.message}")
I18n.t("brex_items.setup_accounts.api_error", message: I18n.t("brex_items.errors.unexpected_error"))
end
def complete_setup_result(account_types:, account_subtypes:)
result = complete_setup!(account_types: account_types, account_subtypes: account_subtypes)
SetupCompletion.new(success: result.failed_count.zero? && result.created_count.positive?, message: setup_notice(result))
rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e
Rails.logger.error("Brex account setup failed: #{e.class} - #{e.message}")
Rails.logger.error(Array(e.backtrace).first(10).join("\n"))
SetupCompletion.new(
success: false,
message: I18n.t("brex_items.complete_account_setup.creation_failed", error: e.message)
)
rescue StandardError => e
Rails.logger.error("Brex account setup failed unexpectedly: #{e.class} - #{e.message}")
Rails.logger.error(Array(e.backtrace).first(10).join("\n"))
SetupCompletion.new(
success: false,
message: I18n.t(
"brex_items.complete_account_setup.creation_failed",
error: I18n.t("brex_items.complete_account_setup.unexpected_error")
)
)
end
private
def setup_notice(result)
if result.failed_count.positive? && result.created_count.positive?
I18n.t("brex_items.complete_account_setup.partial_success", created_count: result.created_count, failed_count: result.failed_count)
elsif result.skipped_count.positive? && result.created_count.positive?
I18n.t("brex_items.complete_account_setup.partial_skipped", created_count: result.created_count, skipped_count: result.skipped_count)
elsif result.failed_count.positive?
I18n.t("brex_items.complete_account_setup.creation_failed_count", count: result.failed_count)
elsif result.created_count.positive?
I18n.t("brex_items.complete_account_setup.success", count: result.created_count)
elsif result.skipped_count.positive?
I18n.t("brex_items.complete_account_setup.all_skipped")
else
I18n.t("brex_items.complete_account_setup.no_accounts")
end
end
def brex_account_snapshot_changed?(brex_account, account_data)
snapshot = account_data.with_indifferent_access
balances = snapshot.slice(:current_balance, :available_balance, :account_limit)
expected = {
account_kind: BrexAccount.kind_for(snapshot),
account_status: snapshot[:status],
account_type: snapshot[:type],
available_balance: BrexAccount.money_to_decimal(balances[:available_balance]),
current_balance: BrexAccount.money_to_decimal(balances[:current_balance]),
account_limit: BrexAccount.money_to_decimal(balances[:account_limit]),
currency: BrexAccount.currency_code_from_money(balances[:current_balance] || balances[:available_balance] || balances[:account_limit]),
name: BrexAccount.name_for(snapshot),
raw_payload: BrexAccount.sanitize_payload(account_data)
}
expected.any? { |attribute, value| brex_account.public_send(attribute) != value }
end
def translate_subtypes(type_key, subtypes_hash)
subtypes_hash.map do |key, value|
[
I18n.t("brex_items.setup_accounts.subtypes.#{type_key}.#{key}", default: value[:long] || key.to_s.humanize),
key
]
end
end
def selected_subtype_for(selected_type:, submitted_subtype:)
return CreditCard::DEFAULT_SUBTYPE if selected_type == "CreditCard" && submitted_subtype.blank?
return Depository::DEFAULT_SUBTYPE if selected_type == "Depository" && submitted_subtype.blank?
submitted_subtype
end
end
end

View File

@@ -0,0 +1,245 @@
# frozen_string_literal: true
class BrexItem::Importer
attr_reader :brex_item, :brex_provider, :sync_start_date
def initialize(brex_item, brex_provider:, sync_start_date: nil)
@brex_item = brex_item
@brex_provider = brex_provider
@sync_start_date = sync_start_date
end
def import
Rails.logger.info "BrexItem::Importer - Starting import for item #{brex_item.id}"
accounts_data = fetch_accounts_data
return failed_result("Failed to fetch accounts data") unless accounts_data
store_item_snapshot(accounts_data)
account_result = import_accounts(accounts_data[:accounts].to_a)
transaction_result = import_transactions
brex_item.update!(status: :good) if account_result[:accounts_failed].zero? && transaction_result[:transactions_failed].zero?
{
success: account_result[:accounts_failed].zero? && transaction_result[:transactions_failed].zero?,
**account_result,
**transaction_result
}
end
private
def fetch_accounts_data
accounts_data = brex_provider.get_accounts
unless accounts_data.is_a?(Hash)
Rails.logger.error "BrexItem::Importer - Invalid accounts_data format: expected Hash, got #{accounts_data.class}"
return nil
end
accounts_data
rescue Provider::Brex::BrexError => e
mark_requires_update_if_credentials_error(e)
Rails.logger.error "BrexItem::Importer - Brex API error: #{e.message} trace_id=#{e.trace_id}"
nil
rescue JSON::ParserError => e
Rails.logger.error "BrexItem::Importer - Failed to parse Brex API response: #{e.message}"
nil
rescue => e
Rails.logger.error "BrexItem::Importer - Unexpected error fetching accounts: #{e.class} - #{e.message}"
Rails.logger.error Array(e.backtrace).join("\n")
nil
end
def store_item_snapshot(accounts_data)
brex_item.upsert_brex_snapshot!(accounts_data)
rescue => e
Rails.logger.error "BrexItem::Importer - Failed to store accounts snapshot: #{e.message}"
Sentry.capture_exception(e) do |scope|
scope.set_tags(brex_item_id: brex_item.id)
scope.set_context("brex_item_snapshot", {
brex_item_id: brex_item.id,
accounts_data: BrexAccount.sanitize_payload(accounts_data)
})
end
raise
end
def import_accounts(accounts)
accounts_updated = 0
accounts_created = 0
accounts_failed = 0
all_existing_ids = brex_item.brex_accounts.pluck("#{BrexAccount.table_name}.account_id").map(&:to_s)
accounts.each do |account_data|
snapshot = account_data.with_indifferent_access
account_id = snapshot[:id].to_s
account_name = BrexAccount.name_for(snapshot)
next if account_id.blank? || account_name.blank?
if all_existing_ids.include?(account_id)
import_account(snapshot)
accounts_updated += 1
else
import_account(snapshot)
accounts_created += 1
all_existing_ids << account_id
end
rescue => e
accounts_failed += 1
Rails.logger.error "BrexItem::Importer - Failed to import account #{account_id.presence || 'unknown'}: #{e.message}"
end
{
accounts_updated: accounts_updated,
accounts_created: accounts_created,
accounts_failed: accounts_failed
}
end
def import_account(account_data)
account_id = account_data[:id].to_s
raise ArgumentError, "Account ID is required" if account_id.blank?
brex_account = brex_item.brex_accounts.find_or_initialize_by(account_id: account_id)
brex_account.name ||= BrexAccount.name_for(account_data)
brex_account.currency ||= BrexAccount.currency_code_from_money(account_data[:current_balance] || account_data[:available_balance] || account_data[:account_limit])
brex_account.upsert_brex_snapshot!(account_data)
brex_account
end
def import_transactions
transactions_imported = 0
transactions_failed = 0
brex_item.brex_accounts.joins(:account).merge(Account.visible).find_each do |brex_account|
result = fetch_and_store_transactions(brex_account)
if result[:success]
transactions_imported += result[:transactions_count]
else
transactions_failed += 1
end
rescue => e
transactions_failed += 1
Rails.logger.error "BrexItem::Importer - Failed to fetch/store transactions for account #{brex_account.account_id}: #{e.message}"
end
{
transactions_imported: transactions_imported,
transactions_failed: transactions_failed
}
end
def fetch_and_store_transactions(brex_account)
start_date = determine_sync_start_date(brex_account)
Rails.logger.info "BrexItem::Importer - Fetching #{brex_account.account_kind} transactions for account #{brex_account.account_id} from #{start_date}"
transactions_data = if brex_account.card?
brex_provider.get_primary_card_transactions(start_date: start_date)
else
brex_provider.get_cash_transactions(brex_account.account_id, start_date: start_date)
end
unless transactions_data.is_a?(Hash)
Rails.logger.error "BrexItem::Importer - Invalid transactions_data format for account #{brex_account.account_id}"
return { success: false, transactions_count: 0, error: "Invalid response format" }
end
transactions = transactions_data[:transactions].to_a
created_count = store_new_transactions(brex_account, transactions, window_start_date: start_date)
{ success: true, transactions_count: created_count }
rescue Provider::Brex::BrexError => e
mark_requires_update_if_credentials_error(e)
Rails.logger.error "BrexItem::Importer - Brex API error for account #{brex_account.account_id}: #{e.message} trace_id=#{e.trace_id}"
{ success: false, transactions_count: 0, error: e.message }
rescue JSON::ParserError => e
Rails.logger.error "BrexItem::Importer - Failed to parse transaction response for account #{brex_account.account_id}: #{e.message}"
{ success: false, transactions_count: 0, error: "Failed to parse response" }
rescue => e
Rails.logger.error "BrexItem::Importer - Unexpected error fetching transactions for account #{brex_account.account_id}: #{e.class} - #{e.message}"
Rails.logger.error Array(e.backtrace).join("\n")
{ success: false, transactions_count: 0, error: "Unexpected error: #{e.message}" }
end
def store_new_transactions(brex_account, transactions, window_start_date:)
existing_payload = brex_account.raw_transactions_payload.to_a
existing_transactions = transactions_in_window(existing_payload, window_start_date)
existing_ids = existing_transactions.map { |tx| tx.with_indifferent_access[:id] }.to_set
new_transactions = transactions.select do |tx|
tx_id = tx.with_indifferent_access[:id]
tx_id.present? && !existing_ids.include?(tx_id) && transaction_in_window?(tx, window_start_date)
end
return 0 if new_transactions.empty? && existing_transactions.count == existing_payload.count
brex_account.upsert_brex_transactions_snapshot!(existing_transactions + new_transactions)
new_transactions.count
end
def transactions_in_window(transactions, window_start_date)
transactions.select { |transaction| transaction_in_window?(transaction, window_start_date) }
end
def transaction_in_window?(transaction, window_start_date)
return true if window_start_date.blank?
transaction_date = transaction_date_for(transaction)
return true if transaction_date.blank?
transaction_date >= window_start_date.to_date
end
def transaction_date_for(transaction)
data = transaction.with_indifferent_access
date_value = data[:posted_at_date].presence || data[:initiated_at_date].presence || data[:posted_at].presence || data[:created_at].presence
case date_value
when Date
date_value
when Time, DateTime
date_value.to_date
when String
Date.parse(date_value)
else
nil
end
rescue ArgumentError, TypeError
nil
end
def determine_sync_start_date(brex_account)
return sync_start_date if sync_start_date.present?
if brex_account.raw_transactions_payload.to_a.any?
brex_item.last_synced_at ? brex_item.last_synced_at - 7.days : 90.days.ago
else
account_baseline = brex_account.created_at || Time.current
[ account_baseline - 7.days, 90.days.ago ].max
end
end
def mark_requires_update_if_credentials_error(error)
return unless error.error_type.in?([ :unauthorized, :access_forbidden ])
brex_item.update!(status: :requires_update)
rescue => update_error
Rails.logger.error "BrexItem::Importer - Failed to update item status: #{update_error.message}"
end
def failed_result(error)
{
success: false,
error: error,
accounts_updated: 0,
accounts_created: 0,
accounts_failed: 0,
transactions_imported: 0,
transactions_failed: 0
}
end
end

View File

@@ -0,0 +1,16 @@
module BrexItem::Provided
extend ActiveSupport::Concern
def brex_provider
return nil unless credentials_configured?
base_url = effective_base_url
return nil unless base_url.present?
Provider::Brex.new(token.to_s.strip, base_url: base_url)
end
def syncer
BrexItem::Syncer.new(self)
end
end

View File

@@ -0,0 +1,148 @@
class BrexItem::Syncer
include SyncStats::Collector
SafeSyncError = Class.new(StandardError)
attr_reader :brex_item
def initialize(brex_item)
@brex_item = brex_item
end
def perform_sync(sync)
sync_errors = []
# Phase 1: Import data from Brex API
update_status(sync, :importing_accounts)
import_result = brex_item.import_latest_brex_data(sync_start_date: sync.window_start_date)
sync_errors.concat(import_result_errors(import_result))
# Phase 2: Collect setup statistics
update_status(sync, :checking_account_configuration)
linked_count = brex_item.brex_accounts.joins(:account_provider).count
unlinked_count = brex_item.brex_accounts
.left_joins(:account_provider)
.where(account_providers: { id: nil })
.count
total_count = linked_count + unlinked_count
collect_brex_setup_stats(
sync,
total_count: total_count,
linked_count: linked_count,
unlinked_count: unlinked_count
)
# Set pending_account_setup if there are unlinked accounts
if unlinked_count.positive?
brex_item.update!(pending_account_setup: true)
update_status(sync, :accounts_need_setup, count: unlinked_count)
else
brex_item.update!(pending_account_setup: false)
end
# Phase 3: Process transactions for linked accounts only
if linked_count.positive?
linked_accounts = brex_item.brex_accounts.joins(:account_provider)
update_status(sync, :processing_transactions)
mark_import_started(sync)
Rails.logger.info "BrexItem::Syncer - Processing #{linked_count} linked accounts"
process_results = brex_item.process_accounts
sync_errors.concat(result_failure_errors(process_results, category: :account_processing_error, message_key: :account_processing_failed))
Rails.logger.info "BrexItem::Syncer - Finished processing accounts"
# Phase 4: Schedule balance calculations for linked accounts
update_status(sync, :calculating_balances)
schedule_results = brex_item.schedule_account_syncs(
parent_sync: sync,
window_start_date: sync.window_start_date,
window_end_date: sync.window_end_date
)
sync_errors.concat(result_failure_errors(schedule_results, category: :account_sync_error, message_key: :account_sync_failed))
# Phase 5: Collect transaction statistics
account_ids = linked_accounts
.includes(account_provider: :account)
.filter_map { |ma| ma.current_account&.id }
collect_transaction_stats(sync, account_ids: account_ids, source: "brex")
else
Rails.logger.info "BrexItem::Syncer - No linked accounts to process"
end
# Mark sync health
collect_health_stats(sync, errors: sync_errors.presence)
rescue => e
safe_message = user_safe_error_message(e)
Rails.logger.error "BrexItem::Syncer - sync failed for Brex item #{brex_item.id}: #{e.class} - #{e.message}"
Rails.logger.error Array(e.backtrace).first(10).join("\n")
Sentry.capture_exception(e) do |scope|
scope.set_tags(brex_item_id: brex_item.id)
end
collect_health_stats(sync, errors: [ { message: safe_message, category: "sync_error" } ])
raise SafeSyncError, safe_message
end
def perform_post_sync
# no-op
end
private
def update_status(sync, key, **options)
return unless sync.respond_to?(:status_text)
sync.update!(status_text: I18n.t("brex_items.syncer.#{key}", **options))
end
def collect_brex_setup_stats(sync, total_count:, linked_count:, unlinked_count:)
return {} unless sync.respond_to?(:sync_stats)
setup_stats = {
"total_accounts" => total_count,
"linked_accounts" => linked_count,
"unlinked_accounts" => unlinked_count
}
merge_sync_stats(sync, setup_stats)
setup_stats
end
def import_result_errors(result)
return [] if result.is_a?(Hash) && result[:success]
unless result.is_a?(Hash)
return [ sync_error(:import_error, :import_failed) ]
end
errors = []
accounts_failed = result[:accounts_failed].to_i
transactions_failed = result[:transactions_failed].to_i
errors << sync_error(:account_import_error, :accounts_failed, count: accounts_failed) if accounts_failed.positive?
errors << sync_error(:transaction_import_error, :transactions_failed, count: transactions_failed) if transactions_failed.positive?
errors << sync_error(:import_error, :import_failed) if errors.empty?
errors
end
def result_failure_errors(results, category:, message_key:)
failed_count = Array(results).count { |result| result.is_a?(Hash) && result[:success] == false }
return [] unless failed_count.positive?
[ sync_error(category, message_key, count: failed_count) ]
end
def sync_error(category, message_key, **options)
{
message: I18n.t("brex_items.syncer.#{message_key}", **options),
category: category.to_s
}
end
def user_safe_error_message(error)
if error.is_a?(Provider::Brex::BrexError) && error.error_type.in?([ :unauthorized, :access_forbidden ])
I18n.t("brex_items.syncer.credentials_invalid")
else
I18n.t("brex_items.syncer.failed")
end
end
end

View File

@@ -0,0 +1,56 @@
# frozen_string_literal: true
module BrexItem::Unlinking
# Concern that encapsulates unlinking logic for a Brex item.
extend ActiveSupport::Concern
# Idempotently remove all connections between this Brex item and local accounts.
# - Detaches any AccountProvider links for each BrexAccount
# - Detaches Holdings that point at the AccountProvider links
# Returns a per-account result payload for observability
def unlink_all!(dry_run: false)
results = []
brex_accounts.find_each do |provider_account|
result = {
provider_account_id: provider_account.id,
name: provider_account.name,
provider_link_ids: []
}
results << result
if dry_run
result[:provider_link_ids] = AccountProvider.where(provider_type: "BrexAccount", provider_id: provider_account.id).ids
next
end
link_ids = []
begin
ActiveRecord::Base.transaction do
links = AccountProvider.where(provider_type: "BrexAccount", provider_id: provider_account.id).to_a
link_ids = links.map(&:id)
result[:provider_link_ids] = link_ids
# Detach holdings for any provider links found
if link_ids.any?
Holding.where(account_provider_id: link_ids).update_all(account_provider_id: nil)
end
# Destroy all provider links
links.each do |ap|
ap.destroy!
end
end
rescue StandardError => e
Rails.logger.warn(
"BrexItem Unlinker: failed to fully unlink provider account ##{provider_account.id} (links=#{link_ids.inspect}): #{e.class} - #{e.message}"
)
# Record error for observability; continue with other accounts
result[:error] = e.message
end
end
results
end
end

View File

@@ -6,11 +6,7 @@ module Encryptable
# This allows encryption to be optional - if not configured, sensitive fields
# are stored in plaintext (useful for development or legacy deployments).
def encryption_ready?
creds_ready = Rails.application.credentials.active_record_encryption.present?
env_ready = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"].present? &&
ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"].present? &&
ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"].present?
creds_ready || env_ready
ActiveRecordEncryptionConfig.explicitly_configured?
end
end
end

View File

@@ -1,6 +1,8 @@
class CreditCard < ApplicationRecord
include Accountable
DEFAULT_SUBTYPE = "credit_card"
SUBTYPES = {
"credit_card" => { short: "Credit Card", long: "Credit Card" }
}.freeze

View File

@@ -1,5 +1,19 @@
class DataEnrichment < ApplicationRecord
belongs_to :enrichable, polymorphic: true
enum :source, { rule: "rule", plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats", mercury: "mercury", indexa_capital: "indexa_capital", sophtron: "sophtron" }
enum :source, {
rule: "rule",
plaid: "plaid",
simplefin: "simplefin",
lunchflow: "lunchflow",
synth: "synth",
ai: "ai",
enable_banking: "enable_banking",
coinstats: "coinstats",
mercury: "mercury",
brex: "brex",
indexa_capital: "indexa_capital",
sophtron: "sophtron",
ibkr: "ibkr"
}
end

View File

@@ -1,6 +1,8 @@
class Depository < ApplicationRecord
include Accountable
DEFAULT_SUBTYPE = "checking"
SUBTYPES = {
"checking" => { short: "Checking", long: "Checking" },
"savings" => { short: "Savings", long: "Savings" },

View File

@@ -23,29 +23,51 @@ class EnableBankingAccount::Transactions::Processor
Account::ProviderImportAdapter.new(enable_banking_account.current_account)
end
# Pre-fetch external_ids that were manually merged and must not be re-imported.
# One query per sync; O(1) Set lookup per transaction — avoids N+1.
# Uses a lateral jsonb_array_elements join to extract only the ID strings in SQL,
# avoiding loading full extra blobs into Ruby. Handles both Array (current) and
# Hash (legacy) formats via jsonb_typeof.
# Pre-fetch external_ids that must not be re-imported.
# One query per category per sync; O(1) Set lookup per transaction — avoids N+1.
excluded_ids = if enable_banking_account.current_account
Transaction.joins(:entry)
.where(entries: { account_id: enable_banking_account.current_account.id })
.where("transactions.extra ? 'manual_merge'")
.joins(
Arel.sql(<<~SQL.squish)
CROSS JOIN LATERAL jsonb_array_elements(
CASE jsonb_typeof(transactions.extra->'manual_merge')
WHEN 'array' THEN transactions.extra->'manual_merge'
WHEN 'object' THEN jsonb_build_array(transactions.extra->'manual_merge')
ELSE '[]'::jsonb
END
) AS merge_elem
SQL
)
.pluck(Arel.sql("merge_elem->>'merged_from_external_id'"))
.compact
.to_set
account_id = enable_banking_account.current_account.id
# 1. Manually merged: pending entries the user explicitly merged into a posted transaction.
# Uses a lateral join to extract merged_from_external_id from the manual_merge JSON
# (handles both Array current format and legacy Hash format via jsonb_typeof).
manually_merged_ids = Transaction.joins(:entry)
.where(entries: { account_id: account_id })
.where("transactions.extra ? 'manual_merge'")
.joins(
Arel.sql(<<~SQL.squish)
CROSS JOIN LATERAL jsonb_array_elements(
CASE jsonb_typeof(transactions.extra->'manual_merge')
WHEN 'array' THEN transactions.extra->'manual_merge'
WHEN 'object' THEN jsonb_build_array(transactions.extra->'manual_merge')
ELSE '[]'::jsonb
END
) AS merge_elem
SQL
)
.pluck(Arel.sql("merge_elem->>'merged_from_external_id'"))
.compact
.to_set
# 2. Auto-claimed: pending entries that were automatically matched to a booked transaction
# by the amount/date heuristic. Their old external_ids are stored in
# extra["auto_claimed_pending_ids"] so they are not re-imported as new pending entries
# on subsequent syncs (the stored raw payload still contains the old pending data).
auto_claimed_ids = Transaction.joins(:entry)
.where(entries: { account_id: account_id })
.where("transactions.extra ? 'auto_claimed_pending_ids'")
.joins(
Arel.sql(<<~SQL.squish)
CROSS JOIN LATERAL jsonb_array_elements_text(
transactions.extra->'auto_claimed_pending_ids'
) AS claimed_id
SQL
)
.pluck(Arel.sql("claimed_id"))
.compact
.to_set
manually_merged_ids | auto_claimed_ids
else
Set.new
end

View File

@@ -13,7 +13,25 @@ class EnableBankingEntry::Processor
def self.compute_external_id(raw_transaction_data)
data = raw_transaction_data.with_indifferent_access
id = data[:transaction_id].presence || data[:entry_reference].presence
id ? "enable_banking_#{id}" : nil
return "enable_banking_#{id}" if id
# Some ASPSPs omit both transaction_id and entry_reference (both are optional
# in PSD2). Generate a deterministic content-based ID so these transactions
# can still be imported idempotently. Uses the same fields as the importer's
# dedup key so the two strategies stay in sync.
date = data[:booking_date].presence || data[:value_date].presence || data[:transaction_date]
amount = data.dig(:transaction_amount, :amount).presence || data[:amount]
currency = data.dig(:transaction_amount, :currency).presence || data[:currency]
direction = data[:credit_debit_indicator]
creditor = data.dig(:creditor, :name).presence || data[:creditor_name]
debtor = data.dig(:debtor, :name).presence || data[:debtor_name]
remittance = data[:remittance_information]
remittance_key = remittance.is_a?(Array) ? remittance.compact.map(&:to_s).sort.join("|") : remittance.to_s
content = [ date, amount, currency, direction, creditor, debtor, remittance_key ].map(&:to_s).join("\x1F")
return nil if content.gsub("\x1F", "").blank?
"enable_banking_content_#{Digest::MD5.hexdigest(content)}"
end
def initialize(enable_banking_transaction, enable_banking_account:, import_adapter: nil)
@@ -75,7 +93,7 @@ class EnableBankingEntry::Processor
def external_id
id = self.class.compute_external_id(data)
raise ArgumentError, "Enable Banking transaction missing required field 'transaction_id'" unless id
raise ArgumentError, "Enable Banking transaction missing required identifier (transaction_id, entry_reference, or identifiable content)" unless id
id
end

View File

@@ -243,26 +243,36 @@ class EnableBankingItem::Importer
pending_transactions = []
if include_pending
# Also fetch pending transactions (visible for 1-3 days before they become BOOK) if setting is enabled
pending_transactions = fetch_paginated_transactions(
enable_banking_account,
start_date: start_date,
transaction_status: "PDNG",
psu_headers: enable_banking_item.build_psu_headers
)
begin
pending_transactions = fetch_paginated_transactions(
enable_banking_account,
start_date: start_date,
transaction_status: "PDNG",
psu_headers: enable_banking_item.build_psu_headers
)
rescue Provider::EnableBanking::EnableBankingError => e
raise unless e.error_type == :validation_error && e.message.include?("transactionStatus")
Rails.logger.warn "EnableBankingItem::Importer - ASPSP does not support PDNG transaction status for account #{enable_banking_account.uid}, skipping pending transactions. API error: #{e.message}"
end
end
book_ids = all_transactions
.map { |tx| tx.with_indifferent_access[:transaction_id].presence }
book_fingerprints = all_transactions
.map { |tx| EnableBankingEntry::Processor.compute_external_id(tx) }
.compact.to_set
# Also index all booked entry_references so a pending row that lacks
# transaction_id can still be matched when the settled BOOK row adds one
# (fingerprints differ; entry_reference stays the same across settlement).
book_entry_refs = all_transactions
.select { |tx| tx.with_indifferent_access[:transaction_id].blank? }
.map { |tx| tx.with_indifferent_access[:entry_reference].presence }
.compact.to_set
pending_transactions.reject! do |tx|
tx = tx.with_indifferent_access
tx[:transaction_id].present? ? book_ids.include?(tx[:transaction_id]) : book_entry_refs.include?(tx[:entry_reference].presence)
tx_ia = tx.with_indifferent_access
fp = EnableBankingEntry::Processor.compute_external_id(tx_ia)
entry_ref = tx_ia[:entry_reference].presence
(fp.present? && book_fingerprints.include?(fp)) ||
(entry_ref.present? && book_entry_refs.include?(entry_ref))
end
all_transactions = all_transactions + tag_as_pending(pending_transactions)
@@ -291,14 +301,17 @@ class EnableBankingItem::Importer
if all_transactions.any?
# C4: Remove stored PDNG entries that have now settled as BOOK.
# When a BOOK transaction arrives with the same transaction_id as a stored
# PDNG entry, the pending entry is stale — drop it to avoid duplicates.
book_ids = all_transactions
# Two match strategies run in parallel:
# 1. Fingerprint: covers same-ID rows and ID-less rows matched by content.
# 2. Entry-reference cross-match: covers the case where a pending row had
# no transaction_id but the settled BOOK row gained one — fingerprints
# diverge (enable_banking_<ref> vs enable_banking_<txn_id>) but the
# shared entry_reference is a reliable settlement signal.
book_fingerprints = all_transactions
.reject { |tx| tx.with_indifferent_access[:_pending] }
.map { |tx| tx.with_indifferent_access[:transaction_id].presence }
.map { |tx| EnableBankingEntry::Processor.compute_external_id(tx) }
.compact.to_set
# Fallback: collect entry_references for BOOK rows that have no transaction_id
book_entry_refs = all_transactions
.reject { |tx| tx.with_indifferent_access[:_pending] }
.map { |tx| tx.with_indifferent_access[:entry_reference].presence }
@@ -310,21 +323,20 @@ class EnableBankingItem::Importer
pending_flag = tx.dig(:extra, :enable_banking, :pending) || tx[:_pending]
next false unless pending_flag
tx[:transaction_id].present? ?
book_ids.include?(tx[:transaction_id]) :
book_entry_refs.include?(tx[:entry_reference].presence)
fp = EnableBankingEntry::Processor.compute_external_id(tx)
entry_ref = tx[:entry_reference].presence
(fp.present? && book_fingerprints.include?(fp)) ||
(entry_ref.present? && book_entry_refs.include?(entry_ref))
end
end
existing_ids = existing_transactions.map { |tx|
tx = tx.with_indifferent_access
tx[:transaction_id].presence || tx[:entry_reference].presence
EnableBankingEntry::Processor.compute_external_id(tx)
}.compact.to_set
new_transactions = all_transactions.select do |tx|
# Use transaction_id if present, otherwise fall back to entry_reference
tx_id = tx[:transaction_id].presence || tx[:entry_reference].presence
tx_id.present? && !existing_ids.include?(tx_id)
ext_id = EnableBankingEntry::Processor.compute_external_id(tx)
ext_id.present? && !existing_ids.include?(ext_id)
end
if new_transactions.any? || removed_pending
@@ -398,7 +410,7 @@ class EnableBankingItem::Importer
# omit transaction_id rarely produce such exact duplicates in the same
# API response; timestamps or remittance info usually differ. (Issue #954)
def build_transaction_content_key(tx)
date = tx[:booking_date].presence || tx[:value_date]
date = tx[:booking_date].presence || tx[:value_date].presence || tx[:transaction_date]
amount = tx.dig(:transaction_amount, :amount).presence || tx[:amount]
currency = tx.dig(:transaction_amount, :currency).presence || tx[:currency]
creditor = tx.dig(:creditor, :name).presence || tx[:creditor_name]

View File

@@ -1,8 +1,8 @@
class Family < ApplicationRecord
include Syncable, AutoTransferMatchable, Subscribeable, VectorSearchable
include PlaidConnectable, SimplefinConnectable, LunchflowConnectable, EnableBankingConnectable
include CoinbaseConnectable, BinanceConnectable, CoinstatsConnectable, SnaptradeConnectable, MercuryConnectable, SophtronConnectable
include IndexaCapitalConnectable
include CoinbaseConnectable, BinanceConnectable, KrakenConnectable, CoinstatsConnectable, SnaptradeConnectable, MercuryConnectable, BrexConnectable, SophtronConnectable
include IndexaCapitalConnectable, IbkrConnectable
DATE_FORMATS = [
[ "MM-DD-YYYY", "%m-%d-%Y" ],

View File

@@ -0,0 +1,29 @@
# frozen_string_literal: true
module Family::BrexConnectable
extend ActiveSupport::Concern
included do
has_many :brex_items, dependent: :destroy
end
def can_connect_brex?
true
end
def create_brex_item!(token:, base_url: nil, item_name: nil)
brex_item = brex_items.create!(
name: item_name.presence || I18n.t("brex_items.default_connection_name"),
token: token,
base_url: base_url
)
brex_item.sync_later
brex_item
end
def has_brex_credentials?
brex_items.active.with_credentials.exists?
end
end

View File

@@ -29,6 +29,10 @@ class Family::DataExporter
zipfile.put_next_entry("rules.csv")
zipfile.write generate_rules_csv
# Add attachment manifest metadata. Binary file payloads are not included.
zipfile.put_next_entry("attachments.json")
zipfile.write generate_attachments_manifest
# Add all.ndjson
zipfile.put_next_entry("all.ndjson")
zipfile.write generate_ndjson
@@ -138,6 +142,69 @@ class Family::DataExporter
end
end
def generate_attachments_manifest
{
version: 1,
binary_included: false,
attachments: attachment_manifest_items
}.to_json
end
def attachment_manifest_items
(transaction_attachment_manifest_items + family_document_attachment_manifest_items)
.sort_by { |item| [ item[:record_type], item[:record_id].to_s, item[:filename].to_s, item[:id].to_s ] }
end
def transaction_attachment_manifest_items
@family.transactions
.with_attached_attachments
.includes(:attachments_attachments, entry: :account)
.flat_map do |transaction|
transaction.attachments.map do |attachment|
attachment_manifest_item(
attachment,
record_type: "Transaction",
record_id: transaction.id,
extra: {
entry_id: transaction.entry.id,
account_id: transaction.entry.account_id
}
)
end
end
end
def family_document_attachment_manifest_items
@family.family_documents.with_attached_file.filter_map do |document|
next unless document.file.attached?
attachment_manifest_item(
document.file.attachment,
record_type: "FamilyDocument",
record_id: document.id,
extra: {
status: document.status
}
)
end
end
def attachment_manifest_item(attachment, record_type:, record_id:, extra: {})
blob = attachment.blob
{
id: attachment.id,
record_type: record_type,
record_id: record_id,
name: attachment.name,
filename: blob.filename.to_s,
content_type: blob.content_type,
byte_size: blob.byte_size,
checksum: blob.checksum,
binary_included: false,
created_at: attachment.created_at
}.merge(extra)
end
def generate_ndjson
lines = []
@@ -426,11 +493,14 @@ class Family::DataExporter
end
def serialize_condition(condition)
operand = resolve_condition_operand(condition)
data = {
condition_type: condition.condition_type,
operator: condition.operator,
value: resolve_condition_value(condition)
value: operand[:value]
}
value_ref = operand[:value_ref]
data[:value_ref] = value_ref if value_ref.present?
if condition.compound? && condition.sub_conditions.any?
data[:sub_conditions] = condition.sub_conditions.map { |sub| serialize_condition(sub) }
@@ -440,52 +510,79 @@ class Family::DataExporter
end
def serialize_action(action)
{
operand = resolve_action_operand(action)
data = {
action_type: action.action_type,
value: resolve_action_value(action)
value: operand[:value]
}
value_ref = operand[:value_ref]
data[:value_ref] = value_ref if value_ref.present?
data
end
def resolve_condition_value(condition)
return condition.value unless condition.value.present?
def resolve_condition_operand(condition)
return rule_operand(condition.value) unless condition.value.present?
# Map category UUIDs to names for portability
if condition.condition_type == "transaction_category" && condition.value.present?
category = @family.categories.find_by(id: condition.value)
return category&.name || condition.value
if condition.condition_type == "transaction_category"
return rule_operand(condition.value, type: "Category", relation: @family.categories)
end
# Map merchant UUIDs to names for portability
if condition.condition_type == "transaction_merchant" && condition.value.present?
merchant = @family.merchants.find_by(id: condition.value)
return merchant&.name || condition.value
if condition.condition_type == "transaction_merchant"
return rule_operand(condition.value, type: "Merchant", relation: @family.merchants)
end
condition.value
rule_operand(condition.value)
end
def resolve_action_value(action)
return action.value unless action.value.present?
def resolve_action_operand(action)
return rule_operand(action.value) unless action.value.present?
# Map category UUIDs to names for portability
if action.action_type == "set_transaction_category" && action.value.present?
category = @family.categories.find_by(id: action.value) || @family.categories.find_by(name: action.value)
return category&.name || action.value
if action.action_type == "set_transaction_category"
return rule_operand(action.value, type: "Category", relation: @family.categories, fallback_to_name: true)
end
# Map merchant UUIDs to names for portability
if action.action_type == "set_transaction_merchant" && action.value.present?
merchant = @family.merchants.find_by(id: action.value) || @family.merchants.find_by(name: action.value)
return merchant&.name || action.value
if action.action_type == "set_transaction_merchant"
return rule_operand(action.value, type: "Merchant", relation: @family.merchants, fallback_to_name: true)
end
# Map tag UUIDs to names for portability
if action.action_type == "set_transaction_tags" && action.value.present?
tag = @family.tags.find_by(id: action.value) || @family.tags.find_by(name: action.value)
return tag&.name || action.value
if action.action_type == "set_transaction_tags"
return rule_operand(action.value, type: "Tag", relation: @family.tags, fallback_to_name: true)
end
action.value
rule_operand(action.value)
end
def rule_operand(value, type: nil, relation: nil, fallback_to_name: false)
record = relation && resolve_rule_operand_record(relation, value, fallback_to_name: fallback_to_name)
{
value: record&.name || value,
value_ref: record ? rule_value_ref(type, record) : nil
}
end
def resolve_rule_operand_record(relation, value, fallback_to_name:)
return relation.find_by(id: value) if uuid_like?(value)
relation.find_by(name: value) if fallback_to_name
end
def rule_value_ref(type, record)
{
type: type,
id: record.id,
name: record.name
}
end
def uuid_like?(value)
UuidFormat.valid?(value)
end
def serialize_conditions_for_csv(conditions)

View File

@@ -1,7 +1,7 @@
require "set"
class Family::DataImporter
SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Transfer RejectedTransfer Trade Holding Valuation Budget BudgetCategory Rule].freeze
SUPPORTED_TYPES = %w[Account Balance Category Tag Merchant RecurringTransaction Transaction Transfer RejectedTransfer Trade Holding Valuation Budget BudgetCategory Rule].freeze
ACCOUNTABLE_TYPES = Accountable::TYPES.freeze
def initialize(family, ndjson_content)
@@ -30,6 +30,7 @@ class Family::DataImporter
Import.transaction do
# Import in dependency order
import_accounts(records["Account"] || [])
import_balances(records["Balance"] || [])
import_categories(records["Category"] || [])
import_tags(records["Tag"] || [])
import_merchants(records["Merchant"] || [])
@@ -128,6 +129,49 @@ class Family::DataImporter
status.to_s.in?(%w[active disabled draft]) ? status.to_s : "active"
end
def import_balances(records)
records.each do |record|
data = record["data"] || {}
new_account_id = @id_mappings[:accounts][data["account_id"]]
balance_date = parse_import_date(data["date"])
next if new_account_id.blank? || balance_date.blank? || data["balance"].blank?
account = @family.accounts.find(new_account_id)
currency = data["currency"].presence || account.currency
balance = account.balances.find_or_initialize_by(date: balance_date, currency: currency)
balance.assign_attributes(imported_balance_attributes(data))
balance.save!
end
end
def imported_balance_attributes(data)
attributes = {
balance: data["balance"].to_d,
cash_balance: optional_decimal(data["cash_balance"]),
start_cash_balance: optional_decimal(data["start_cash_balance"]),
start_non_cash_balance: optional_decimal(data["start_non_cash_balance"]),
cash_inflows: optional_decimal(data["cash_inflows"]),
cash_outflows: optional_decimal(data["cash_outflows"]),
non_cash_inflows: optional_decimal(data["non_cash_inflows"]),
non_cash_outflows: optional_decimal(data["non_cash_outflows"]),
net_market_flows: optional_decimal(data["net_market_flows"]),
cash_adjustments: optional_decimal(data["cash_adjustments"]),
non_cash_adjustments: optional_decimal(data["non_cash_adjustments"])
}.compact
attributes[:flows_factor] = balance_flows_factor_for(data["flows_factor"]) if data["flows_factor"].present?
attributes
end
def optional_decimal(value)
value.presence&.to_d
end
def balance_flows_factor_for(value)
value.to_i.in?([ -1, 1 ]) ? value.to_i : 1
end
def import_categories(records)
# First pass: create all categories without parent relationships
parent_mappings = {}
@@ -472,7 +516,7 @@ class Family::DataImporter
# Account-level opening balances must precede every imported account
# activity, including standalone valuation snapshots.
%w[Transaction Trade Holding Valuation].each do |type|
%w[Balance Transaction Trade Holding Valuation].each do |type|
records[type].to_a.each do |record|
data = record["data"] || {}
account_id = data["account_id"]
@@ -627,7 +671,7 @@ class Family::DataImporter
def resolve_rule_condition_value(condition_data)
condition_type = condition_data["condition_type"]
value = condition_data["value"]
value = rule_operand_value(condition_data)
return value unless value.present?
@@ -655,7 +699,7 @@ class Family::DataImporter
def resolve_rule_action_value(action_data)
action_type = action_data["action_type"]
value = action_data["value"]
value = rule_operand_value(action_data)
return value unless value.present?
@@ -688,6 +732,21 @@ class Family::DataImporter
value
end
def rule_operand_value(data)
raw_value = data["value"]
value = raw_value.is_a?(String) ? raw_value.presence : raw_value
value_ref_name = data.dig("value_ref", "name")
return value_ref_name if value.is_a?(String) && uuid_like?(value) && value_ref_name.present?
return value unless value.nil?
value_ref_name
end
def uuid_like?(value)
UuidFormat.valid?(value)
end
def importable_cost_basis_source(value)
source = value.to_s
Holding::COST_BASIS_SOURCES.include?(source) ? source : nil

View File

@@ -0,0 +1,22 @@
module Family::IbkrConnectable
extend ActiveSupport::Concern
included do
has_many :ibkr_items, dependent: :destroy
end
def can_connect_ibkr?
true
end
def create_ibkr_item!(query_id:, token:, item_name: nil)
ibkr_item = ibkr_items.create!(
name: item_name.presence || "Interactive Brokers",
query_id: query_id,
token: token
)
ibkr_item.sync_later
ibkr_item
end
end

View File

@@ -0,0 +1,29 @@
# frozen_string_literal: true
module Family::KrakenConnectable
extend ActiveSupport::Concern
included do
has_many :kraken_items, dependent: :destroy
end
def can_connect_kraken?
true
end
def create_kraken_item!(api_key:, api_secret:, item_name: nil)
item = kraken_items.create!(
name: item_name || "Kraken",
api_key: api_key,
api_secret: api_secret
)
item.set_kraken_institution_defaults!
item.sync_later
item
end
def has_kraken_credentials?
kraken_items.active.any?(&:credentials_configured?)
end
end

View File

@@ -17,6 +17,7 @@ class Family::Syncer
coinbase_items
coinstats_items
mercury_items
brex_items
binance_items
snaptrade_items
sophtron_items

View File

@@ -38,7 +38,7 @@ class Holding < ApplicationRecord
return nil unless amount
return 0 if amount.zero?
account.balance.zero? ? 1 : amount / account.balance * 100
account.balance.zero? ? 1 : amount_in_account_currency / account.balance * 100
end
# Returns average cost per share, or nil if unknown.
@@ -256,6 +256,14 @@ class Holding < ApplicationRecord
end
private
def amount_in_account_currency
return amount if currency == account.currency
Money.new(amount, currency).exchange_to(account.currency, date: date).amount
rescue Money::ConversionError
amount
end
def calculate_trend
return nil unless amount_money
return nil if avg_cost.nil? # Can't calculate trend without cost basis (0 is valid for airdrops)

View File

@@ -22,10 +22,10 @@ class Holding::Materializer
# securities are still needed to derive sane balance charts between sync snapshots.
cleanup_shadowed_calculated_holdings
# Also remove calculated rows on the provider's latest snapshot date when those
# securities are no longer present in the provider payload. This keeps "current"
# holdings/balance composition aligned with the provider snapshot while preserving
# older calculated history.
# Also remove non-provider rows on the provider's latest snapshot date for securities
# that appear in the provider snapshot. The provider snapshot is authoritative for
# those securities on that day, even when it is denominated in a different currency
# than the account or the reverse-calculated holdings.
cleanup_stale_calculated_rows_on_latest_provider_snapshot
# Reload holdings association to clear any cached stale data
@@ -152,17 +152,12 @@ class Holding::Materializer
.where(date: provider_snapshot_date)
.distinct
.pluck(:security_id)
return if provider_security_ids.empty?
scope = account.holdings
.where(account_provider_id: nil, date: provider_snapshot_date)
deleted_count = account.holdings
.where(account_provider_id: nil, date: provider_snapshot_date, security_id: provider_security_ids)
.delete_all
scope = if provider_security_ids.any?
scope.where.not(security_id: provider_security_ids)
else
scope
end
deleted_count = scope.delete_all
Rails.logger.info("Cleaned up #{deleted_count} stale calculated holdings on latest provider snapshot date") if deleted_count > 0
end

View File

@@ -40,7 +40,7 @@ class Holding::PortfolioCache
price_money = Money.new(price.price, price.currency)
begin
converted_amount = price_money.exchange_to(account.currency).amount
converted_amount = price_money.exchange_to(account.currency, date: date).amount
rescue Money::ConversionError
converted_amount = price.price
end

View File

@@ -0,0 +1,78 @@
class IbkrAccount < ApplicationRecord
include CurrencyNormalizable, Encryptable
include IbkrAccount::DataHelpers
if encryption_ready?
encrypts :raw_holdings_payload
encrypts :raw_activities_payload
encrypts :raw_cash_report_payload
encrypts :raw_equity_summary_payload
end
belongs_to :ibkr_item
has_one :account_provider, as: :provider, dependent: :destroy
has_one :account, through: :account_provider, source: :account
has_one :linked_account, through: :account_provider, source: :account
validates :name, :currency, presence: true
validates :ibkr_account_id, uniqueness: { scope: :ibkr_item_id, allow_nil: true }
def current_account
account || linked_account
end
def ensure_account_provider!(account = nil)
if account_provider.present?
account_provider.update!(account: account) if account && account_provider.account_id != account.id
return account_provider
end
acct = account || current_account
return nil unless acct
provider = AccountProvider
.find_or_initialize_by(provider_type: "IbkrAccount", provider_id: id)
.tap do |record|
record.account = acct
record.save!
end
reload_account_provider
provider
rescue => e
Rails.logger.warn("IbkrAccount##{id}: failed to ensure AccountProvider link: #{e.class} - #{e.message}")
nil
end
def upsert_from_ibkr_statement!(account_data)
data = account_data.with_indifferent_access
update!(
ibkr_account_id: data[:ibkr_account_id],
name: data[:name],
currency: parse_currency(data[:currency]) || "USD",
current_balance: data[:current_balance],
cash_balance: data[:cash_balance],
institution_metadata: {
provider_name: "Interactive Brokers",
statement_from_date: data.dig(:statement, :from_date),
statement_to_date: data.dig(:statement, :to_date)
}.compact,
report_date: data[:report_date],
raw_holdings_payload: data[:open_positions] || [],
raw_activities_payload: {
trades: data[:trades] || [],
cash_transactions: data[:cash_transactions] || []
},
raw_cash_report_payload: data[:cash_report] || [],
raw_equity_summary_payload: data[:equity_summary_in_base] || [],
last_holdings_sync: Time.current,
last_activities_sync: Time.current
)
end
def ibkr_provider
ibkr_item.ibkr_provider
end
end

View File

@@ -0,0 +1,221 @@
class IbkrAccount::ActivitiesProcessor
include IbkrAccount::DataHelpers
SUPPORTED_CASH_TRANSACTION_TYPES = [ "DEPOSITS/WITHDRAWALS", "DIVIDENDS" ].freeze
def initialize(ibkr_account)
@ibkr_account = ibkr_account
end
def process
return { trades: 0, transactions: 0 } unless account.present?
activities = (@ibkr_account.raw_activities_payload || {}).with_indifferent_access
trades = Array(activities[:trades])
cash_transactions = Array(activities[:cash_transactions])
@fee_transactions_count = 0
trades_count = trades.sum { |trade| process_trade(trade.with_indifferent_access) ? 1 : 0 }
cash_transactions_count = cash_transactions.sum { |cash_transaction| process_cash_transaction(cash_transaction.with_indifferent_access) ? 1 : 0 }
{
trades: trades_count,
transactions: cash_transactions_count + @fee_transactions_count
}
end
private
def account
@ibkr_account.current_account
end
def import_adapter
@import_adapter ||= Account::ProviderImportAdapter.new(account)
end
def process_trade(row)
return false unless supported_trade?(row)
security = resolve_security(row)
return false unless security
quantity = parse_decimal(row[:quantity])
native_price = parse_decimal(row[:trade_price])
return false if quantity.nil? || native_price.nil?
buy_sell = row[:buy_sell].to_s.upcase
signed_quantity = buy_sell == "SELL" ? -quantity.abs : quantity.abs
native_amount = buy_sell == "SELL" ? -(native_price * quantity.abs) : (native_price * quantity.abs)
currency = extract_currency(row, fallback: @ibkr_account.currency)
date = trade_date_for(row)
external_id = "ibkr_trade_#{row[:trade_id]}"
import_adapter.import_trade(
external_id: external_id,
security: security,
quantity: signed_quantity,
price: native_price,
amount: native_amount,
currency: currency,
date: date,
name: build_trade_name(security.ticker, signed_quantity),
source: "ibkr",
activity_label: buy_sell == "SELL" ? "Sell" : "Buy",
exchange_rate: parse_decimal(row[:fx_rate_to_base])&.to_f
)
import_commission_transaction(row, security, date)
true
rescue => e
Rails.logger.error("IbkrAccount::ActivitiesProcessor - Failed to process trade #{row[:trade_id]}: #{e.message}")
false
end
def process_cash_transaction(row)
return false unless supported_cash_transaction?(row)
amount = parse_decimal(row[:amount])
return false if amount.nil? || amount.zero?
label, signed_amount = classify_cash_transaction(row, amount)
return false unless label
currency = extract_currency(row, fallback: @ibkr_account.currency)
security = resolve_security_for_cash_transaction(row)
import_adapter.import_transaction(
external_id: "ibkr_cash_#{row[:transaction_id]}",
amount: signed_amount,
currency: currency,
date: parse_date(row[:report_date]),
name: build_cash_transaction_name(row, label, security),
source: "ibkr",
investment_activity_label: label,
extra: {
exchange_rate: parse_decimal(row[:fx_rate_to_base])&.to_f,
security_id: security&.id,
ibkr: {
transaction_id: row[:transaction_id],
type: row[:type],
conid: row[:conid],
amount: row[:amount],
currency: row[:currency],
fx_rate_to_base: row[:fx_rate_to_base],
report_date: row[:report_date]
}.compact
}
)
true
rescue => e
Rails.logger.error("IbkrAccount::ActivitiesProcessor - Failed to process cash transaction #{row[:transaction_id]}: #{e.message}")
false
end
def import_commission_transaction(row, security, date)
commission = parse_decimal(row[:ib_commission])
return if commission.nil? || commission.zero?
currency = row.with_indifferent_access[:ib_commission_currency].to_s.upcase.presence || @ibkr_account.currency
ticker = security&.ticker || row.with_indifferent_access[:symbol]
result = import_adapter.import_transaction(
external_id: "ibkr_trade_fee_#{row[:trade_id]}",
amount: commission.abs,
currency: currency,
date: date,
name: "Trade Commission for #{ticker}",
source: "ibkr",
investment_activity_label: "Fee",
extra: {
exchange_rate: parse_decimal(row[:fx_rate_to_base])&.to_f,
security_id: security&.id,
ibkr: {
trade_id: row[:trade_id],
transaction_id: row[:transaction_id],
ib_commission: row[:ib_commission],
ib_commission_currency: row[:ib_commission_currency],
fx_rate_to_base: row[:fx_rate_to_base]
}.compact
}
)
@fee_transactions_count += 1 if result
end
def build_trade_name(ticker, signed_quantity)
action = signed_quantity.negative? ? "Sell" : "Buy"
"#{action} #{signed_quantity.abs} shares of #{ticker}"
end
def supported_trade?(row)
row[:asset_category].to_s == "STK" &&
row[:buy_sell].present? &&
row[:conid].present? &&
row[:currency].present? &&
row[:quantity].present? &&
row[:symbol].present? &&
row[:trade_date].present? &&
row[:trade_id].present? &&
row[:trade_price].present? &&
row[:transaction_id].present? &&
fx_rate_available?(row)
end
def supported_cash_transaction?(row)
type = row[:type].to_s.upcase.strip
return false unless SUPPORTED_CASH_TRANSACTION_TYPES.include?(type)
return false unless row[:transaction_id].present? && row[:amount].present? && row[:currency].present? && row[:report_date].present?
return false unless fx_rate_available?(row)
type != "DIVIDENDS" || row[:conid].present?
end
def classify_cash_transaction(row, amount)
type = row[:type].to_s.upcase.strip
case type
when "DEPOSITS/WITHDRAWALS"
amount.positive? ? [ "Contribution", -amount.abs ] : [ "Withdrawal", amount.abs ]
when "DIVIDENDS"
[ "Dividend", -amount.abs ]
else
[ nil, nil ]
end
end
def build_cash_transaction_name(row, label, security = nil)
return label unless label == "Dividend"
ticker = security&.ticker || security_symbol_for_conid(row[:conid]) || row[:conid]
"Dividend from #{ticker}"
end
def resolve_security_for_cash_transaction(row)
symbol = security_symbol_for_conid(row[:conid])
return nil if symbol.blank?
resolve_security({ symbol: symbol })
end
def security_symbol_for_conid(conid)
return nil if conid.blank?
holding_symbol = Array(@ibkr_account.raw_holdings_payload).find do |holding|
holding.with_indifferent_access[:conid].to_s == conid.to_s
end&.with_indifferent_access&.dig(:symbol)
return holding_symbol if holding_symbol.present?
Array(@ibkr_account.raw_activities_payload&.dig("trades") || @ibkr_account.raw_activities_payload&.dig(:trades)).find do |trade|
trade.with_indifferent_access[:conid].to_s == conid.to_s
end&.with_indifferent_access&.dig(:symbol)
end
def fx_rate_available?(row)
source_currency = extract_currency(row, fallback: nil)
return false if source_currency.blank?
return true if source_currency == @ibkr_account.currency
row[:fx_rate_to_base].present?
end
end

View File

@@ -0,0 +1,78 @@
module IbkrAccount::DataHelpers
extend ActiveSupport::Concern
private
def parse_decimal(value)
return nil if value.nil?
normalized = value.is_a?(String) ? value.delete(",").strip : value.to_s
return nil if normalized.blank? || normalized == "-"
BigDecimal(normalized)
rescue ArgumentError
nil
end
def parse_date(value)
return nil if value.blank?
case value
when Date
value
when Time, DateTime, ActiveSupport::TimeWithZone
value.to_date
else
normalized = value.to_s.tr(";", " ")
Time.zone.parse(normalized)&.to_date || Date.parse(normalized)
end
rescue ArgumentError, TypeError
nil
end
def parse_datetime(value)
return nil if value.blank?
case value
when Time, DateTime, ActiveSupport::TimeWithZone
value.in_time_zone
when Date
value.in_time_zone
else
Time.zone.parse(value.to_s.tr(";", " "))
end
rescue ArgumentError, TypeError
nil
end
def resolve_security(row)
data = row.with_indifferent_access
ticker = data[:symbol].to_s.strip.upcase
return nil if ticker.blank?
Security.find_by(ticker: ticker) || create_security_from_row(ticker)
rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
Security.find_by(ticker: ticker)
end
def trade_date_for(row)
data = row.with_indifferent_access
parsed_trade_date = parse_date(data[:trade_date])
return parsed_trade_date if parsed_trade_date
Rails.logger.warn(
"IbkrAccount::DataHelpers - Missing or invalid trade_date, falling back to Date.current. " \
"trade_id=#{data[:trade_id].inspect}"
)
Date.current
end
def extract_currency(row, fallback: nil)
value = row.with_indifferent_access[:currency]
value.present? ? value.to_s.upcase : fallback
end
def create_security_from_row(ticker)
Security.create!(ticker: ticker, name: ticker)
end
end

View File

@@ -0,0 +1,79 @@
class IbkrAccount::HistoricalBalancesSync
include IbkrAccount::DataHelpers
attr_reader :ibkr_account
def initialize(ibkr_account)
@ibkr_account = ibkr_account
end
def sync!
return unless account.present?
return if normalized_rows.empty?
account.balances.upsert_all(
balance_rows,
unique_by: %i[account_id date currency]
)
end
private
def account
ibkr_account.current_account
end
def normalized_rows
@normalized_rows ||= Array(ibkr_account.raw_equity_summary_payload)
.filter_map do |row|
next unless row.is_a?(Hash)
data = row.with_indifferent_access
currency = data[:currency].presence&.upcase
account_currency = ibkr_account.currency.to_s.upcase
next if currency.present? && currency != account_currency
date = parse_date(data[:report_date])
total = parse_decimal(data[:total])
cash = parse_decimal(data[:cash]) || BigDecimal("0")
next unless date && total
{
date: date,
total: total,
cash: cash,
non_cash: total - cash
}
end
.sort_by { |row| row[:date] }
end
def balance_rows
current_time = Time.current
normalized_rows.each_with_index.map do |row, index|
previous_row = index.zero? ? nil : normalized_rows[index - 1]
start_cash_balance = previous_row ? previous_row[:cash] : row[:cash]
start_non_cash_balance = previous_row ? previous_row[:non_cash] : row[:non_cash]
{
account_id: account.id,
date: row[:date],
balance: row[:total],
cash_balance: row[:cash],
currency: account.currency,
start_cash_balance: start_cash_balance,
start_non_cash_balance: start_non_cash_balance,
cash_inflows: 0,
cash_outflows: 0,
non_cash_inflows: 0,
non_cash_outflows: 0,
net_market_flows: 0,
cash_adjustments: row[:cash] - start_cash_balance,
non_cash_adjustments: row[:non_cash] - start_non_cash_balance,
flows_factor: 1,
created_at: current_time,
updated_at: current_time
}
end
end
end

View File

@@ -0,0 +1,105 @@
class IbkrAccount::HoldingsProcessor
include IbkrAccount::DataHelpers
def initialize(ibkr_account)
@ibkr_account = ibkr_account
end
def process
return unless account.present?
grouped_positions.each_value do |group|
process_group(group)
end
end
private
def account
@ibkr_account.current_account
end
def import_adapter
@import_adapter ||= Account::ProviderImportAdapter.new(account)
end
def grouped_positions
Array(@ibkr_account.raw_holdings_payload).each_with_object({}) do |position, groups|
data = position.with_indifferent_access
next unless supported_position?(data)
symbol_key = data[:conid].presence || data[:symbol].presence || data[:security_id].presence
currency = extract_currency(data, fallback: @ibkr_account.currency)
report_date = parse_date(data[:report_date]) || @ibkr_account.report_date || Date.current
key = [ symbol_key, currency, report_date ]
groups[key] ||= []
groups[key] << data
end
end
def process_group(rows)
sample = rows.first
security = resolve_security(sample)
return unless security
quantity = rows.sum { |row| parse_decimal(row[:position]) || BigDecimal("0") }
return if quantity.zero?
price = parse_decimal(sample[:mark_price])
cost_basis = weighted_cost_basis_for(rows)
return unless price && cost_basis
amount = quantity.abs * price
currency = extract_currency(sample, fallback: @ibkr_account.currency)
report_date = parse_date(sample[:report_date]) || @ibkr_account.report_date || Date.current
external_id = [ "ibkr", @ibkr_account.ibkr_account_id, sample[:conid].presence || security.ticker, report_date, currency ].join("_")
import_adapter.import_holding(
security: security,
quantity: quantity,
amount: amount,
currency: currency,
date: report_date,
price: price || BigDecimal("0"),
cost_basis: cost_basis,
external_id: external_id,
source: "ibkr",
account_provider_id: @ibkr_account.account_provider&.id,
delete_future_holdings: false
)
end
def weighted_cost_basis_for(rows)
total_quantity = BigDecimal("0")
total_cost = BigDecimal("0")
rows.each do |row|
row_quantity = parse_decimal(row[:position])
row_cost_basis = parse_decimal(row[:cost_basis_price])
return nil unless row_quantity && row_cost_basis
total_quantity += row_quantity.abs
total_cost += row_quantity.abs * row_cost_basis
end
return nil if total_quantity.zero?
total_cost / total_quantity
end
def supported_position?(row)
row[:asset_category].to_s == "STK" &&
row[:side].to_s == "Long" &&
row[:conid].present? &&
row[:security_id].present? &&
row[:security_id_type].present? &&
row[:symbol].present? &&
row[:currency].present? &&
row[:fx_rate_to_base].present? &&
row[:position].present? &&
row[:mark_price].present? &&
row[:cost_basis_price].present? &&
row[:report_date].present?
end
end

View File

@@ -0,0 +1,56 @@
class IbkrAccount::Processor
attr_reader :ibkr_account
def initialize(ibkr_account)
@ibkr_account = ibkr_account
end
def process
return unless ibkr_account.current_account.present?
update_account_balance!
IbkrAccount::HoldingsProcessor.new(ibkr_account).process
IbkrAccount::ActivitiesProcessor.new(ibkr_account).process
repair_default_opening_anchor!
ibkr_account.current_account.broadcast_sync_complete
end
private
def update_account_balance!
account = ibkr_account.current_account
total_balance = ibkr_account.current_balance || ibkr_account.cash_balance || 0
cash_balance = ibkr_account.cash_balance || 0
account.assign_attributes(
balance: total_balance,
cash_balance: cash_balance,
currency: ibkr_account.currency
)
account.save!
account.set_current_balance(total_balance)
end
def repair_default_opening_anchor!
account = ibkr_account.current_account
return unless account&.linked_to?("IbkrAccount")
return unless account.has_opening_anchor?
opening_anchor_entry = account.valuations.opening_anchor.includes(:entry).first&.entry
return unless opening_anchor_entry
return unless opening_anchor_entry.created_at.to_date == account.created_at.to_date
return unless account.entries.where.not(entryable_type: "Valuation").exists?
imported_current_balance = (ibkr_account.current_balance || ibkr_account.cash_balance || 0).to_d
return unless opening_anchor_entry.amount.to_d == imported_current_balance
result = Account::OpeningBalanceManager.new(account).set_opening_balance(
balance: 0,
date: opening_anchor_entry.date
)
raise result.error if result.error
end
end

124
app/models/ibkr_item.rb Normal file
View File

@@ -0,0 +1,124 @@
class IbkrItem < ApplicationRecord
include Syncable, Provided, Unlinking, Encryptable
enum :status, { good: "good", requires_update: "requires_update" }, default: :good
if encryption_ready?
encrypts :query_id, deterministic: true
encrypts :token
encrypts :raw_payload
end
belongs_to :family
has_one_attached :logo, dependent: :purge_later
has_many :ibkr_accounts, dependent: :destroy
validates :name, presence: true
validates :query_id, presence: true, on: :create
validates :token, presence: true, on: :create
scope :active, -> { where(scheduled_for_deletion: false) }
scope :syncable, -> { active.where.not(query_id: [ nil, "" ]).where.not(token: nil) }
scope :ordered, -> { order(created_at: :desc) }
scope :needs_update, -> { where(status: :requires_update) }
def destroy_later
update!(scheduled_for_deletion: true)
DestroyJob.perform_later(self)
end
def credentials_configured?
query_id.present? && token.present?
end
def import_latest_ibkr_data
provider = ibkr_provider
raise StandardError, "IBKR provider is not configured" unless provider
IbkrItem::Importer.new(self, ibkr_provider: provider).import
rescue => e
Rails.logger.error("IbkrItem #{id} - Failed to import data: #{e.message}")
raise
end
def process_accounts
return [] if ibkr_accounts.empty?
linked_ibkr_accounts.includes(account_provider: :account).each_with_object([]) do |ibkr_account, results|
account = ibkr_account.current_account
next unless account
next if account.pending_deletion? || account.disabled?
begin
result = IbkrAccount::Processor.new(ibkr_account).process
results << { ibkr_account_id: ibkr_account.id, success: true, result: result }
rescue => e
Rails.logger.error("IbkrItem #{id} - Failed to process account #{ibkr_account.id}: #{e.message}")
results << { ibkr_account_id: ibkr_account.id, success: false, error: e.message }
end
end
end
def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil)
accounts.reject { |account| account.pending_deletion? || account.disabled? }.each_with_object([]) do |account, results|
begin
account.sync_later(
parent_sync: parent_sync,
window_start_date: window_start_date,
window_end_date: window_end_date
)
results << { account_id: account.id, success: true }
rescue => e
Rails.logger.error("IbkrItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}")
results << { account_id: account.id, success: false, error: e.message }
end
end
end
def upsert_ibkr_snapshot!(payload)
update!(raw_payload: payload, status: :good)
end
def accounts
ibkr_accounts.includes(account_provider: :account).filter_map(&:current_account).uniq
end
def linked_ibkr_accounts
ibkr_accounts.joins(:account_provider)
end
def linked_accounts_count
ibkr_accounts.joins(:account_provider).count
end
def unlinked_accounts_count
ibkr_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).count
end
def total_accounts_count
ibkr_accounts.count
end
def has_completed_initial_setup?
accounts.any?
end
def sync_status_summary
total_accounts = total_accounts_count
linked_count = linked_accounts_count
unlinked_count = unlinked_accounts_count
if total_accounts.zero?
I18n.t("ibkr_items.sync_status.no_accounts")
elsif unlinked_count.zero?
I18n.t("ibkr_items.sync_status.all_linked", count: linked_count)
else
I18n.t("ibkr_items.sync_status.partial", linked: linked_count, unlinked: unlinked_count)
end
end
def institution_display_name
I18n.t("ibkr_items.defaults.name")
end
end

View File

@@ -0,0 +1,33 @@
class IbkrItem::Importer
attr_reader :ibkr_item, :ibkr_provider
def initialize(ibkr_item, ibkr_provider:)
@ibkr_item = ibkr_item
@ibkr_provider = ibkr_provider
end
def import
xml_body = ibkr_provider.download_statement
parsed_report = IbkrItem::ReportParser.new(xml_body).parse
accounts_imported = 0
ibkr_item.transaction do
ibkr_item.upsert_ibkr_snapshot!(parsed_report[:metadata].merge("fetched_at" => Time.current.iso8601))
parsed_report[:accounts].each do |account_data|
next if account_data[:ibkr_account_id].blank?
ibkr_account = ibkr_item.ibkr_accounts.find_or_initialize_by(ibkr_account_id: account_data[:ibkr_account_id])
ibkr_account.upsert_from_ibkr_statement!(account_data)
accounts_imported += 1
end
ibkr_item.update!(status: :good)
end
{
success: true,
accounts_imported: accounts_imported
}
end
end

View File

@@ -0,0 +1,9 @@
module IbkrItem::Provided
extend ActiveSupport::Concern
def ibkr_provider
return nil unless credentials_configured?
Provider::IbkrFlex.new(query_id: query_id, token: token)
end
end

View File

@@ -0,0 +1,143 @@
class IbkrItem::ReportParser
include IbkrAccount::DataHelpers
class ParseError < StandardError; end
POSITION_VALUE_CONTAINER_NAMES = %w[ChangeInPositionValues].freeze
POSITION_VALUE_ROW_NAMES = %w[ChangeInPositionValue].freeze
CASH_REPORT_CONTAINER_NAMES = %w[CashReport CashReports].freeze
CASH_REPORT_ROW_NAMES = %w[CashReport CashReportCurrency CashReportRow].freeze
EQUITY_SUMMARY_CONTAINER_NAMES = %w[EquitySummaryInBase].freeze
EQUITY_SUMMARY_ROW_NAMES = %w[EquitySummaryByReportDateInBase].freeze
OPEN_POSITION_CONTAINER_NAMES = %w[OpenPositions].freeze
OPEN_POSITION_ROW_NAMES = %w[OpenPosition].freeze
TRADES_CONTAINER_NAMES = %w[Trades].freeze
TRADE_ROW_NAMES = %w[Trade].freeze
CASH_TRANSACTION_CONTAINER_NAMES = %w[CashTransactions].freeze
CASH_TRANSACTION_ROW_NAMES = %w[CashTransaction].freeze
def initialize(xml_body)
@document = Nokogiri::XML(xml_body.to_s) { |config| config.strict.noblanks }
rescue Nokogiri::XML::SyntaxError => e
raise ParseError, "Invalid IBKR Flex XML: #{e.message}"
end
def parse
validate_document!
{
metadata: root_metadata,
accounts: flex_statements.map { |statement| parse_statement(statement) }
}
end
private
def validate_document!
raise ParseError, "Invalid IBKR Flex XML: missing FlexQueryResponse root." unless @document.at_xpath("//FlexQueryResponse")
raise ParseError, "Invalid IBKR Flex XML: no FlexStatement nodes found." if flex_statements.empty?
end
def flex_statements
@document.xpath("//FlexStatement")
end
def root_metadata
node_attributes(@document.at_xpath("//FlexQueryResponse"))
end
def parse_statement(statement)
statement_data = node_attributes(statement)
account_information = node_attributes(statement.at_xpath("./AccountInformation"))
position_values = section_rows(statement, POSITION_VALUE_CONTAINER_NAMES, POSITION_VALUE_ROW_NAMES)
cash_report = section_rows(statement, CASH_REPORT_CONTAINER_NAMES, CASH_REPORT_ROW_NAMES)
equity_summary_in_base = section_rows(statement, EQUITY_SUMMARY_CONTAINER_NAMES, EQUITY_SUMMARY_ROW_NAMES)
open_positions = section_rows(statement, OPEN_POSITION_CONTAINER_NAMES, OPEN_POSITION_ROW_NAMES)
trades = section_rows(statement, TRADES_CONTAINER_NAMES, TRADE_ROW_NAMES)
cash_transactions = section_rows(statement, CASH_TRANSACTION_CONTAINER_NAMES, CASH_TRANSACTION_ROW_NAMES)
account_id = account_information["account_id"].presence || statement_data["account_id"]
raise ParseError, "Invalid IBKR Flex XML: missing account identifier in FlexStatement." if account_id.blank?
currency = account_information["currency"].presence&.upcase || "USD"
report_date = open_positions.filter_map { |row| parse_date(row["report_date"]) }.max ||
equity_summary_in_base.filter_map { |row| parse_date(row["report_date"]) }.max ||
parse_date(statement_data["to_date"]) ||
Date.current
{
ibkr_account_id: account_id,
name: account_id,
currency: currency,
cash_balance: extract_cash_balance(cash_report, currency),
current_balance: extract_total_balance(position_values, cash_report, currency),
report_date: report_date,
statement: statement_data,
cash_report: cash_report,
equity_summary_in_base: equity_summary_in_base,
open_positions: open_positions,
trades: trades,
cash_transactions: cash_transactions,
raw_payload: {
statement: statement_data,
cash_report: cash_report,
equity_summary_in_base: equity_summary_in_base,
open_positions: open_positions,
trades: trades,
cash_transactions: cash_transactions
}
}
end
def section_rows(statement, container_names, row_names)
rows = []
container_names.each do |container_name|
statement.xpath("./#{container_name}").each do |container|
children = container.element_children
if children.any?
rows.concat(children.select { |child| row_names.include?(child.name) })
elsif row_names.include?(container.name)
rows << container
end
end
end
if rows.empty?
row_names.each do |row_name|
rows.concat(statement.xpath("./#{row_name}"))
end
end
rows.map { |row| node_attributes(row) }.reject(&:blank?)
end
def node_attributes(node)
return {} unless node
node.attribute_nodes.each_with_object({}) do |attribute, result|
result[attribute.name.underscore] = attribute.value
end
end
def extract_cash_balance(cash_rows, account_currency)
base_summary = cash_rows.find { |row| row["currency"] == "BASE_SUMMARY" }
account_row = cash_rows.find { |row| row["currency"] == account_currency }
row = base_summary || account_row
parse_decimal(row&.fetch("ending_cash", nil)) || BigDecimal("0")
end
def extract_current_balance(position_values, account_currency)
base_summary = position_values.find { |row| row["currency"] == "BASE_SUMMARY" }
account_row = position_values.find { |row| row["currency"] == account_currency }
row = base_summary || account_row
parse_decimal(row&.fetch("end_of_period_value", nil)) || BigDecimal("0")
end
def extract_total_balance(position_values, cash_rows, account_currency)
extract_current_balance(position_values, account_currency) + extract_cash_balance(cash_rows, account_currency)
end
end

View File

@@ -0,0 +1,22 @@
class IbkrItem::SyncCompleteEvent
attr_reader :ibkr_item
def initialize(ibkr_item)
@ibkr_item = ibkr_item
end
def broadcast
ibkr_item.accounts.each do |account|
account.broadcast_sync_complete
end
ibkr_item.broadcast_replace_to(
ibkr_item.family,
target: "ibkr_item_#{ibkr_item.id}",
partial: "ibkr_items/ibkr_item",
locals: { ibkr_item: ibkr_item }
)
ibkr_item.family.broadcast_sync_complete
end
end

View File

@@ -0,0 +1,68 @@
class IbkrItem::Syncer
include SyncStats::Collector
attr_reader :ibkr_item
def initialize(ibkr_item)
@ibkr_item = ibkr_item
end
def perform_sync(sync)
sync.update!(status_text: "Checking IBKR credentials...") if sync.respond_to?(:status_text)
unless ibkr_item.credentials_configured?
ibkr_item.update!(status: :requires_update)
raise Provider::IbkrFlex::ConfigurationError, "IBKR credentials are missing."
end
sync.update!(status_text: "Importing IBKR accounts...") if sync.respond_to?(:status_text)
ibkr_item.import_latest_ibkr_data
sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text)
collect_setup_stats(sync, provider_accounts: ibkr_item.ibkr_accounts.to_a)
unlinked_accounts = ibkr_item.ibkr_accounts.left_joins(:account_provider).where(account_providers: { id: nil })
linked_accounts = ibkr_item.ibkr_accounts.joins(:account).merge(Account.visible)
if unlinked_accounts.any?
ibkr_item.update!(pending_account_setup: true)
sync.update!(status_text: "#{unlinked_accounts.count} IBKR account(s) need setup...") if sync.respond_to?(:status_text)
else
ibkr_item.update!(pending_account_setup: false)
end
if linked_accounts.any?
sync.update!(status_text: "Processing holdings and activity...") if sync.respond_to?(:status_text)
ibkr_item.process_accounts
sync.update!(status_text: "Calculating balances...") if sync.respond_to?(:status_text)
ibkr_item.schedule_account_syncs(
parent_sync: sync,
window_start_date: sync.window_start_date,
window_end_date: sync.window_end_date
)
account_ids = linked_accounts.includes(:account).filter_map { |provider_account| provider_account.account&.id }
collect_transaction_stats(sync, account_ids: account_ids, source: "ibkr") if account_ids.any?
collect_trades_stats(sync, account_ids: account_ids, source: "ibkr") if account_ids.any?
collect_holdings_stats(sync, holdings_count: count_holdings, label: "processed")
end
collect_health_stats(sync, errors: nil)
rescue Provider::IbkrFlex::AuthenticationError, Provider::IbkrFlex::ConfigurationError => e
ibkr_item.update!(status: :requires_update)
collect_health_stats(sync, errors: [ { message: e.message, category: "auth_error" } ])
raise
rescue => e
collect_health_stats(sync, errors: [ { message: e.message, category: "sync_error" } ])
raise
end
def perform_post_sync
end
private
def count_holdings
ibkr_item.ibkr_accounts.sum { |account| Array(account.raw_holdings_payload).size }
end
end

View File

@@ -0,0 +1,37 @@
# frozen_string_literal: true
module IbkrItem::Unlinking
extend ActiveSupport::Concern
def unlink_all!(dry_run: false)
results = []
ibkr_accounts.find_each do |provider_account|
links = AccountProvider.where(provider_type: "IbkrAccount", provider_id: provider_account.id).to_a
link_ids = links.map(&:id)
result = {
provider_account_id: provider_account.id,
name: provider_account.name,
provider_link_ids: link_ids
}
results << result
next if dry_run
begin
ActiveRecord::Base.transaction do
Holding.where(account_provider_id: link_ids).update_all(account_provider_id: nil) if link_ids.any?
links.each(&:destroy!)
end
rescue => e
Rails.logger.warn(
"IbkrItem Unlinker: failed to fully unlink provider account ##{provider_account.id} " \
"(links=#{link_ids.inspect}): #{e.class} - #{e.message}"
)
result[:error] = e.message
end
end
results
end
end

View File

@@ -2,6 +2,7 @@ class Import < ApplicationRecord
MaxRowCountExceededError = Class.new(StandardError)
MappingError = Class.new(StandardError)
# Shared CSV upload/content limit for web and API imports, including preflight.
MAX_CSV_SIZE = 10.megabytes
MAX_PDF_SIZE = 25.megabytes
ALLOWED_CSV_MIME_TYPES = %w[text/csv text/plain application/vnd.ms-excel application/csv].freeze
@@ -24,6 +25,10 @@ class Import < ApplicationRecord
Date.new(1970, 1, 1)..Date.today.next_year(5)
end
def self.max_csv_size
MAX_CSV_SIZE
end
AMOUNT_TYPE_STRATEGIES = %w[signed_amount custom_column].freeze
belongs_to :family

View File

@@ -0,0 +1,454 @@
# frozen_string_literal: true
class Import::Preflight
Response = Struct.new(:status, :payload, keyword_init: true)
class PreflightError < StandardError
attr_reader :status, :payload
def initialize(response)
@status = response.status
@payload = response.payload
super(response.payload[:message])
end
end
CONFIG_PARAM_KEYS = %i[
date_col_label
amount_col_label
name_col_label
category_col_label
tags_col_label
notes_col_label
account_col_label
qty_col_label
ticker_col_label
price_col_label
entity_type_col_label
currency_col_label
exchange_operating_mic_col_label
date_format
number_format
signage_convention
col_sep
amount_type_strategy
amount_type_inflow_value
rows_to_skip
].freeze
PARAM_KEYS = ([
:type,
:account_id,
:file,
:raw_file_content
] + CONFIG_PARAM_KEYS).freeze
UNSUPPORTED_PREFLIGHT_IMPORT_TYPES = %w[PdfImport QifImport].freeze
IMPORT_TYPES = (Import::TYPES - UNSUPPORTED_PREFLIGHT_IMPORT_TYPES).freeze
def initialize(family:, params:)
@family = family
@params = params.to_h.symbolize_keys
end
def call
type = preflight_import_type
return invalid_import_type_response unless type
type == "SureImport" ? sure_import_response : csv_import_response(type)
rescue PreflightError => e
Response.new(status: e.status, payload: e.payload)
end
private
attr_reader :family, :params
def preflight_import_type
type = params[:type].to_s
return "TransactionImport" if type.blank?
type if IMPORT_TYPES.include?(type)
end
def invalid_import_type_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "invalid_import_type",
message: "type must be one of: #{IMPORT_TYPES.join(', ')}"
}
)
end
def sure_import_response
upload_attributes = sure_import_upload_attributes
return missing_sure_content_response unless upload_attributes
content, filename, content_type = upload_attributes
Response.new(
status: :ok,
payload: {
data: sure_import_preflight_payload(content, filename, content_type)
}
)
end
def csv_import_response(type)
upload_attributes = csv_upload_attributes
return missing_csv_content_response unless upload_attributes
content, filename, content_type = upload_attributes
import = family.imports.build(import_config_params.merge(type: type, raw_file_str: content))
import.account = preflight_account if params[:account_id].present?
apply_import_defaults(import)
return unsupported_import_type_response unless import.requires_csv_workflow?
unless import.valid?
return Response.new(
status: :ok,
payload: {
data: csv_preflight_payload(
import: import,
type: type,
filename: filename,
content_type: content_type,
content: content,
parsed_rows_count: 0,
csv_headers: [],
missing_required_headers: [],
errors: validation_errors(import),
warnings: []
)
}
)
end
csv_content = csv_content_for(import, content)
csv = Import.parse_csv_str(csv_content, col_sep: import.col_sep)
parsed_rows_count = csv.length
csv_headers = Array(csv.headers).compact
missing_required_headers = missing_required_headers(import, csv_headers)
errors = validation_errors(import)
if missing_required_headers.any?
errors << {
code: "missing_required_headers",
message: "Missing required columns: #{missing_required_headers.join(', ')}"
}
end
if parsed_rows_count.zero?
errors << {
code: "no_data_rows",
message: "No data rows were found."
}
end
warnings = []
warnings << "Row count exceeds this import type's publish limit." if parsed_rows_count > import.max_row_count
Response.new(
status: :ok,
payload: {
data: csv_preflight_payload(
import: import,
type: type,
filename: filename,
content_type: content_type,
content: content,
parsed_rows_count: parsed_rows_count,
csv_headers: csv_headers,
missing_required_headers: missing_required_headers,
errors: errors,
warnings: warnings
)
}
)
end
def import_config_params
params.slice(*CONFIG_PARAM_KEYS)
end
def preflight_account
raise ActiveRecord::RecordNotFound unless Api::V1::BaseController.valid_uuid?(params[:account_id])
family.accounts.find(params[:account_id])
end
def csv_upload_attributes
if params[:file].present?
csv_file_upload_attributes(params[:file])
elsif params[:raw_file_content].present?
csv_raw_content_attributes(params[:raw_file_content].to_s)
end
end
def csv_file_upload_attributes(file)
raise_response csv_file_too_large_response if file.size > Import.max_csv_size
raise_response invalid_csv_file_type_response unless Import::ALLOWED_CSV_MIME_TYPES.include?(file.content_type)
[
file.read,
file.original_filename.presence || "import.csv",
file.content_type.presence || "text/csv"
]
end
def csv_raw_content_attributes(content)
raise_response csv_content_too_large_response if content.bytesize > Import.max_csv_size
[ content, "import.csv", "text/csv" ]
end
def sure_import_upload_attributes
if params[:file].present?
sure_import_file_upload_attributes(params[:file])
elsif params[:raw_file_content].present?
sure_import_raw_content_attributes(params[:raw_file_content].to_s)
end
end
def sure_import_file_upload_attributes(file)
raise_response sure_file_too_large_response if file.size > SureImport.max_ndjson_size
extension = File.extname(file.original_filename.to_s).downcase
unless SureImport::ALLOWED_NDJSON_CONTENT_TYPES.include?(file.content_type) || extension.in?(%w[.ndjson .json])
raise_response invalid_sure_file_type_response
end
[
file.read,
file.original_filename.presence || "sure-import.ndjson",
file.content_type.presence || "application/x-ndjson"
]
end
def sure_import_raw_content_attributes(content)
raise_response sure_content_too_large_response if content.bytesize > SureImport.max_ndjson_size
[ content, "sure-import.ndjson", "application/x-ndjson" ]
end
def sure_import_preflight_payload(content, filename, content_type)
line_counts = Hash.new(0)
errors = []
valid_rows_count = 0
nonblank_rows_count = 0
content.each_line.with_index(1) do |line, line_number|
next if line.strip.blank?
nonblank_rows_count += 1
record = JSON.parse(line)
unless record.is_a?(Hash)
errors << {
code: "invalid_ndjson_record",
message: "Line #{line_number} must be a JSON object."
}
next
end
if record["type"].blank? || !record.key?("data")
errors << {
code: "invalid_ndjson_record",
message: "Line #{line_number} must include type and data."
}
next
end
valid_rows_count += 1
line_counts[record["type"]] += 1
rescue JSON::ParserError => e
errors << {
code: "invalid_json",
message: "Line #{line_number} is not valid JSON: #{e.message}"
}
end
if nonblank_rows_count.zero?
errors << {
code: "no_data_rows",
message: "No data rows were found."
}
end
entity_counts = SureImport.dry_run_totals_from_line_type_counts(line_counts)
unsupported_types = line_counts.keys - SureImport.importable_ndjson_types
warnings = []
warnings << "No importable records were found." if nonblank_rows_count.positive? && entity_counts.values.sum.zero?
warnings << "Some records use unsupported types: #{unsupported_types.join(', ')}" if unsupported_types.any?
warnings << "Row count exceeds this import type's publish limit." if nonblank_rows_count > SureImport.max_row_count
{
type: "SureImport",
valid: errors.empty?,
content: content_payload(filename, content_type, content),
stats: {
rows_count: nonblank_rows_count,
valid_rows_count: valid_rows_count,
invalid_rows_count: nonblank_rows_count - valid_rows_count,
entity_counts: entity_counts,
record_type_counts: line_counts
},
errors: errors,
warnings: warnings
}
end
def content_payload(filename, content_type, content)
{
filename: filename,
content_type: content_type,
byte_size: content.bytesize
}
end
def csv_content_for(import, content)
return content unless import.rows_to_skip.to_i.positive?
content.lines.drop(import.rows_to_skip.to_i).join
end
def apply_import_defaults(import)
return unless import.is_a?(MintImport)
MintImport.default_column_mappings.each do |attribute, value|
import.public_send("#{attribute}=", value) if import.public_send(attribute).blank?
end
end
def validation_errors(import)
import.errors.full_messages.map { |message| { code: "validation_failed", message: message } }
end
def csv_preflight_payload(import:, type:, filename:, content_type:, content:, parsed_rows_count:, csv_headers:, missing_required_headers:, errors:, warnings:)
{
type: type,
valid: errors.empty?,
content: content_payload(filename, content_type, content),
stats: {
rows_count: parsed_rows_count
},
headers: csv_headers,
required_headers: required_header_labels(import),
missing_required_headers: missing_required_headers,
errors: errors,
warnings: warnings
}
end
def required_header_labels(import)
import.required_column_keys.filter_map do |key|
import.respond_to?("#{key}_col_label") ? import.public_send("#{key}_col_label").presence || key.to_s : key.to_s
end
end
def missing_required_headers(import, headers)
normalized_headers = Array(headers).compact.to_h { |header| [ normalized_header(header), header ] }
required_header_labels(import).reject do |header|
normalized_headers.key?(normalized_header(header))
end
end
def normalized_header(header)
header.to_s.strip.downcase.gsub(/\*/, "").gsub(/[\s-]+/, "_")
end
def missing_csv_content_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "missing_content",
message: "Provide a CSV file or raw_file_content."
}
)
end
def missing_sure_content_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "missing_content",
message: "Provide a Sure NDJSON file or raw_file_content."
}
)
end
def csv_file_too_large_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "file_too_large",
message: "File is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB."
}
)
end
def csv_content_too_large_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "content_too_large",
message: "Content is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB."
}
)
end
def invalid_csv_file_type_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "invalid_file_type",
message: "Invalid file type. Please upload a CSV file."
}
)
end
def sure_file_too_large_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "file_too_large",
message: "File is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB."
}
)
end
def sure_content_too_large_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "content_too_large",
message: "Content is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB."
}
)
end
def invalid_sure_file_type_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "invalid_file_type",
message: "Invalid file type. Please upload a Sure NDJSON file."
}
)
end
def raise_response(response)
raise PreflightError, response
end
def unsupported_import_type_response
Response.new(
status: :unprocessable_entity,
payload: {
error: "unsupported_import_type",
message: "Preflight supports CSV import types and SureImport."
}
)
end
end

View File

@@ -0,0 +1,39 @@
# frozen_string_literal: true
class KrakenAccount < ApplicationRecord
include Encryptable
STABLECOINS = %w[USDT USDC DAI PYUSD USDP TUSD USDG].freeze
FIAT_CURRENCIES = %w[USD EUR GBP CAD AUD CHF JPY AED].freeze
if encryption_ready?
encrypts :raw_payload
encrypts :raw_transactions_payload
end
belongs_to :kraken_item
has_one :account_provider, as: :provider, dependent: :destroy
has_one :account, through: :account_provider, source: :account
validates :name, :account_id, :account_type, :currency, presence: true
def current_account
account
end
def ensure_account_provider!(target_account = nil)
acct = target_account || current_account
return nil unless acct
AccountProvider
.find_or_initialize_by(provider_type: "KrakenAccount", provider_id: id)
.tap do |ap|
ap.account = acct
ap.save!
end
rescue StandardError => e
Rails.logger.warn("KrakenAccount #{id}: failed to link account provider - #{e.class}: #{e.message}")
nil
end
end

View File

@@ -0,0 +1,67 @@
# frozen_string_literal: true
class KrakenAccount::AssetNormalizer
SUFFIX_PATTERN = /(\.[A-Z])\z/
FIAT_PREFIXES = {
"ZUSD" => "USD",
"ZEUR" => "EUR",
"ZGBP" => "GBP",
"ZCAD" => "CAD",
"ZAUD" => "AUD",
"ZCHF" => "CHF",
"ZJPY" => "JPY"
}.freeze
SYMBOL_FALLBACKS = {
"XBT" => "BTC",
"XXBT" => "BTC",
"XETH" => "ETH",
"ZUSD" => "USD"
}.freeze
def initialize(asset_metadata = {})
@asset_metadata = asset_metadata || {}
end
def normalize(raw_asset)
raw = raw_asset.to_s.upcase
suffix = raw[SUFFIX_PATTERN, 1]
raw_base = suffix ? raw.delete_suffix(suffix) : raw
metadata = metadata_for(raw, raw_base)
base_symbol = metadata_symbol(metadata, raw_base)
normalized_base = normalize_base_symbol(base_symbol)
symbol = suffix.present? ? "#{normalized_base}#{suffix}" : normalized_base
{
raw_asset: raw,
raw_base: raw_base,
symbol: symbol,
price_symbol: normalized_base,
suffix: suffix,
metadata: metadata
}
end
private
attr_reader :asset_metadata
def metadata_for(raw, raw_base)
asset_metadata[raw] || asset_metadata[raw_base] || asset_metadata.values.find do |metadata|
candidate = metadata_symbol(metadata, raw_base)
[ raw, raw_base ].include?(candidate.to_s.upcase)
end
end
def metadata_symbol(metadata, fallback)
return fallback unless metadata.is_a?(Hash)
metadata["altname"].presence || metadata["display_name"].presence || fallback
end
def normalize_base_symbol(symbol)
value = symbol.to_s.upcase
value = FIAT_PREFIXES[value] if FIAT_PREFIXES.key?(value)
SYMBOL_FALLBACKS[value] || value
end
end

View File

@@ -0,0 +1,84 @@
# frozen_string_literal: true
class KrakenAccount::HoldingsProcessor
include KrakenAccount::UsdConverter
def initialize(kraken_account)
@kraken_account = kraken_account
end
def process
return unless account&.accountable_type == "Crypto"
raw_assets.each { |asset| process_asset(asset) }
rescue StandardError => e
Rails.logger.error "KrakenAccount::HoldingsProcessor - error: #{e.message}"
nil
end
private
attr_reader :kraken_account
def target_currency
kraken_account.kraken_item&.family&.currency
end
def account
kraken_account.current_account
end
def raw_assets
kraken_account.raw_payload&.dig("assets") || []
end
def process_asset(asset)
symbol = asset["symbol"] || asset[:symbol]
price_symbol = asset["price_symbol"] || asset[:price_symbol] || symbol
total = (asset["balance"] || asset[:balance] || 0).to_d
price_usd = asset["price_usd"] || asset[:price_usd]
source = asset["source"] || asset[:source] || "spot"
return if symbol.blank? || total.zero? || price_usd.blank?
security = resolve_security(symbol)
return unless security
amount_usd = total * price_usd.to_d
amount, amount_stale, amount_rate_date = convert_from_usd(amount_usd, date: Date.current)
price, price_stale, price_rate_date = convert_from_usd(price_usd.to_d, date: Date.current)
log_stale_rate(symbol, "amount", amount_rate_date) if amount_stale
log_stale_rate(symbol, "price", price_rate_date) if price_stale
import_adapter.import_holding(
security: security,
quantity: total,
amount: amount,
currency: target_currency,
date: Date.current,
price: price,
cost_basis: nil,
external_id: "kraken_#{symbol}_#{source}_#{Date.current}",
account_provider_id: kraken_account.account_provider&.id,
source: "kraken",
delete_future_holdings: false
)
rescue StandardError => e
Rails.logger.error "KrakenAccount::HoldingsProcessor - failed asset symbol=#{symbol.presence || "unknown"}: #{e.message}"
end
def import_adapter
@import_adapter ||= Account::ProviderImportAdapter.new(account)
end
def resolve_security(symbol)
ticker = symbol.to_s.include?(":") ? symbol.to_s : "CRYPTO:#{symbol}"
KrakenAccount::SecurityResolver.resolve(ticker, symbol)
end
def log_stale_rate(symbol, field, rate_date)
Rails.logger.warn(
"KrakenAccount::HoldingsProcessor - stale FX rate for #{field} symbol=#{symbol} rate_date=#{rate_date || "unknown"}"
)
end
end

View File

@@ -0,0 +1,122 @@
# frozen_string_literal: true
class KrakenAccount::Processor
include KrakenAccount::UsdConverter
attr_reader :kraken_account
def initialize(kraken_account)
@kraken_account = kraken_account
end
def process
return unless kraken_account.current_account.present?
KrakenAccount::HoldingsProcessor.new(kraken_account).process
process_account!
process_trades
end
private
def target_currency
kraken_account.kraken_item&.family&.currency
end
def process_account!
account = kraken_account.current_account
amount, stale, rate_date = convert_from_usd((kraken_account.current_balance || 0).to_d, date: Date.current)
account.update!(
balance: amount,
cash_balance: 0,
currency: target_currency
)
kraken_account.update!(extra: kraken_account.extra.to_h.deep_merge(build_stale_extra(stale, rate_date, Date.current)))
end
def process_trades
raw_trades.each do |txid, trade|
process_trade(txid, trade)
end
rescue StandardError => e
Rails.logger.error "KrakenAccount::Processor - trade processing failed: #{e.message}"
end
def raw_trades
kraken_account.raw_transactions_payload&.dig("trades") || {}
end
def process_trade(txid, trade)
account = kraken_account.current_account
return unless account
external_id = "kraken_trade_#{txid}"
return if account.entries.exists?(external_id: external_id, source: "kraken")
type = trade["type"].to_s.downcase
return unless %w[buy sell].include?(type)
pair = trade["pair"].to_s
base_symbol, quote_symbol = infer_pair_symbols(pair, trade)
return if base_symbol.blank?
qty = trade["vol"].to_d
return if qty.zero?
price = trade["price"].to_d
cost = trade["cost"].presence&.to_d
cost ||= (qty * price).round(8)
fee = trade["fee"].presence&.to_d || 0
currency = quote_symbol.presence || "USD"
date = Time.zone.at(trade["time"].to_d).to_date
security = KrakenAccount::SecurityResolver.resolve("CRYPTO:#{base_symbol}", base_symbol)
return unless security
entry_amount = type == "buy" ? -cost : cost
trade_qty = type == "buy" ? qty : -qty
label = type == "buy" ? "Buy" : "Sell"
account.entries.create!(
date: date,
name: "#{label} #{qty.round(8)} #{base_symbol}",
amount: entry_amount,
currency: currency,
external_id: external_id,
source: "kraken",
notes: trade["ordertxid"].presence,
entryable: Trade.new(
security: security,
qty: trade_qty,
price: price,
currency: currency,
fee: fee,
investment_activity_label: label
)
)
rescue StandardError => e
Rails.logger.error "KrakenAccount::Processor - failed to process trade #{txid}: #{e.message}"
end
def infer_pair_symbols(pair, trade)
pair_metadata = kraken_account.raw_payload&.dig("pair_metadata") || {}
metadata = pair_metadata[pair] || pair_metadata.values.find { |candidate| candidate["altname"].to_s == pair }
normalizer = KrakenAccount::AssetNormalizer.new(kraken_account.raw_payload&.dig("asset_metadata") || {})
if metadata
base = normalizer.normalize(metadata["base"])[:symbol]
quote = normalizer.normalize(metadata["quote"])[:symbol]
return [ base, quote ]
end
altname = trade["pair"].to_s
%w[USDT USDC USD EUR GBP BTC ETH].each do |quote|
next unless altname.end_with?(quote)
return [ normalizer.normalize(altname.delete_suffix(quote))[:symbol], quote ]
end
[ altname, "USD" ]
end
end

View File

@@ -0,0 +1,16 @@
# frozen_string_literal: true
class KrakenAccount::SecurityResolver
EXCHANGE_MIC = "XKRA"
def self.resolve(ticker, symbol)
Security::Resolver.new(ticker).resolve
rescue StandardError => e
Rails.logger.warn "KrakenAccount::SecurityResolver - resolver failed for #{ticker}: #{e.message}"
Security.find_or_initialize_by(ticker: ticker, exchange_operating_mic: EXCHANGE_MIC).tap do |security|
security.name = symbol if security.name.blank?
security.offline = true unless security.offline
security.save! if security.changed?
end
end
end

View File

@@ -0,0 +1,32 @@
# frozen_string_literal: true
module KrakenAccount::UsdConverter
private
def convert_from_usd(amount, date: Date.current)
return [ amount.to_d, false, nil ] if target_currency == "USD"
rate = ExchangeRate.find_or_fetch_rate(from: "USD", to: target_currency, date: date)
return [ amount.to_d, true, nil ] if rate.nil?
converted = Money.new(amount, "USD").exchange_to(target_currency, custom_rate: rate.rate).amount
stale = rate.date != date
rate_date = stale ? rate.date : nil
[ converted, stale, rate_date ]
end
def build_stale_extra(stale, rate_date, target_date)
kraken_meta = if stale
{
"stale_rate" => true,
"rate_date_used" => rate_date&.to_s,
"rate_target_date" => target_date&.to_s
}
else
{ "stale_rate" => false }
end
{ "kraken" => kraken_meta }
end
end

153
app/models/kraken_item.rb Normal file
View File

@@ -0,0 +1,153 @@
# frozen_string_literal: true
class KrakenItem < ApplicationRecord
include Syncable, Provided, Unlinking, Encryptable
enum :status, { good: "good", requires_update: "requires_update" }, default: :good
if encryption_ready?
encrypts :api_key, deterministic: true
encrypts :api_secret
encrypts :raw_payload
end
validates :name, presence: true
validates :api_key, presence: true
validates :api_secret, presence: true
belongs_to :family
has_one_attached :logo, dependent: :purge_later
has_many :kraken_accounts, dependent: :destroy
has_many :accounts, through: :kraken_accounts
scope :active, -> { where(scheduled_for_deletion: false) }
scope :syncable, -> { active }
scope :ordered, -> { order(created_at: :desc) }
scope :needs_update, -> { where(status: :requires_update) }
scope :credentials_configured, -> { where.not(api_key: [ nil, "" ]).where.not(api_secret: nil) }
before_validation :strip_credentials
def destroy_later
update!(scheduled_for_deletion: true)
DestroyJob.perform_later(self)
end
def import_latest_kraken_data
provider = kraken_provider
raise StandardError, "Kraken credentials not configured" unless provider
KrakenItem::Importer.new(self, kraken_provider: provider).import
rescue StandardError => e
Rails.logger.error "KrakenItem #{id} - Failed to import: #{e.full_message}"
raise
end
def process_accounts
return [] if kraken_accounts.empty?
results = []
kraken_accounts.joins(:account).merge(Account.visible).each do |kraken_account|
begin
result = KrakenAccount::Processor.new(kraken_account).process
results << { kraken_account_id: kraken_account.id, success: true, result: result }
rescue StandardError => e
Rails.logger.error "KrakenItem #{id} - Failed to process account #{kraken_account.id}: #{e.full_message}"
results << { kraken_account_id: kraken_account.id, success: false, error: e.message }
end
end
results
end
def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil)
return [] if accounts.empty?
accounts.visible.map do |account|
account.sync_later(
parent_sync: parent_sync,
window_start_date: window_start_date,
window_end_date: window_end_date
)
{ account_id: account.id, success: true }
rescue StandardError => e
Rails.logger.error "KrakenItem #{id} - Failed to schedule sync for account #{account.id}: #{e.full_message}"
{ account_id: account.id, success: false, error: e.message }
end
end
def upsert_kraken_snapshot!(payload)
update!(raw_payload: payload)
end
def has_completed_initial_setup?
accounts.any?
end
def sync_status_summary
total = total_accounts_count
linked = linked_accounts_count
unlinked = unlinked_accounts_count
if total.zero?
I18n.t("kraken_items.kraken_item.sync_status.no_accounts")
elsif unlinked.zero?
I18n.t("kraken_items.kraken_item.sync_status.all_synced", count: linked)
else
I18n.t("kraken_items.kraken_item.sync_status.partial_sync", linked_count: linked, unlinked_count: unlinked)
end
end
def linked_accounts_count
kraken_accounts.joins(:account_provider).count
end
def unlinked_accounts_count
kraken_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).count
end
def total_accounts_count
kraken_accounts.count
end
def stale_rate_accounts
kraken_accounts
.joins(:account)
.where(accounts: { status: "active" })
.where("kraken_accounts.extra -> 'kraken' ->> 'stale_rate' = 'true'")
end
def institution_display_name
institution_name.presence || institution_domain.presence || name
end
def credentials_configured?
api_key.to_s.strip.present? && api_secret.to_s.strip.present?
end
def next_nonce!
with_lock do
candidate = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
candidate = last_nonce.to_i + 1 if candidate <= last_nonce.to_i
update!(last_nonce: candidate)
candidate.to_s
end
end
def set_kraken_institution_defaults!
update!(
institution_name: "Kraken",
institution_domain: "kraken.com",
institution_url: "https://www.kraken.com",
institution_color: "#5841D8"
)
end
private
def strip_credentials
self.api_key = api_key.to_s.strip if api_key_changed? && !api_key.nil?
self.api_secret = api_secret.to_s.strip if api_secret_changed? && !api_secret.nil?
end
end

View File

@@ -0,0 +1,187 @@
# frozen_string_literal: true
class KrakenItem::Importer
MAX_TRADE_PAGES = 200
TRADE_PAGE_SIZE = 50
attr_reader :kraken_item, :kraken_provider
def initialize(kraken_item, kraken_provider:)
@kraken_item = kraken_item
@kraken_provider = kraken_provider
end
def import
api_key_info = kraken_provider.get_api_key_info
asset_metadata = kraken_provider.get_asset_info || {}
pair_metadata = kraken_provider.get_asset_pairs || {}
balances = kraken_provider.get_extended_balance || {}
assets = parse_assets(balances, asset_metadata)
trades = fetch_trades
total_usd = assets.sum { |asset| asset[:amount_usd].to_d }.round(2)
kraken_account = upsert_kraken_account(
assets: assets,
balances: balances,
trades: trades,
asset_metadata: asset_metadata,
pair_metadata: pair_metadata,
api_key_info: api_key_info,
total_usd: total_usd
)
kraken_item.upsert_kraken_snapshot!({
"api_key_info" => api_key_info,
"balances" => balances,
"asset_metadata" => asset_metadata,
"pair_metadata" => pair_metadata,
"imported_at" => Time.current.iso8601
})
{ success: true, account_id: kraken_account.id, assets_imported: assets.size, trades_imported: trades.size, total_usd: total_usd }
rescue Provider::Kraken::PermissionError => e
kraken_item.update!(status: :requires_update)
raise e
end
private
def parse_assets(balances, asset_metadata)
normalizer = KrakenAccount::AssetNormalizer.new(asset_metadata)
balances.filter_map do |raw_asset, balance_data|
parsed = normalizer.normalize(raw_asset)
balance = balance_data.fetch("balance", "0").to_d
credit = balance_data.fetch("credit", "0").to_d
credit_used = balance_data.fetch("credit_used", "0").to_d
hold_trade = balance_data.fetch("hold_trade", "0").to_d
available = balance + credit - credit_used - hold_trade
next if balance.zero? && hold_trade.zero?
price_usd, price_status = price_for(parsed[:price_symbol])
amount_usd = price_usd ? (balance * price_usd).round(2) : 0.to_d
parsed.merge(
balance: balance.to_s("F"),
available: available.to_s("F"),
hold_trade: hold_trade.to_s("F"),
price_usd: price_usd&.to_s("F"),
amount_usd: amount_usd.to_s("F"),
price_status: price_status,
source: "spot"
)
end
end
def price_for(symbol)
return [ 1.to_d, "exact" ] if symbol == "USD" || KrakenAccount::STABLECOINS.include?(symbol)
if KrakenAccount::FIAT_CURRENCIES.include?(symbol)
rate = ExchangeRate.find_or_fetch_rate(from: symbol, to: "USD", date: Date.current)
return [ rate.rate.to_d, rate.date == Date.current ? "exact" : "stale" ] if rate
return [ nil, "missing" ]
end
ticker_price = ticker_price_for(symbol)
return [ ticker_price, "exact" ] if ticker_price
[ nil, "missing" ]
rescue StandardError => e
Rails.logger.warn "KrakenItem::Importer - could not price #{symbol}: #{e.message}"
[ nil, "missing" ]
end
def ticker_price_for(symbol)
pair_candidates_for(symbol).each do |pair|
response = kraken_provider.get_ticker(pair)
ticker_payload = response&.values&.first
price = ticker_payload&.dig("c", 0)
return price.to_d if price.present?
rescue Provider::Kraken::ApiError
next
end
nil
end
def pair_candidates_for(symbol)
kraken_symbol = symbol == "BTC" ? "XBT" : symbol
[
"#{kraken_symbol}USD",
"#{symbol}USD",
"X#{kraken_symbol}ZUSD",
"#{kraken_symbol}USDT",
"#{symbol}USDT"
].uniq
end
def fetch_trades
start_time = kraken_item.sync_start_date&.to_i
offset = 0
all_trades = {}
MAX_TRADE_PAGES.times do
result = kraken_provider.get_trades_history(start: start_time, offset: offset)
trades = result.to_h.fetch("trades", {})
duplicate_trade_ids = all_trades.keys & trades.keys
if duplicate_trade_ids.any?
Rails.logger.warn("KrakenItem::Importer - #{duplicate_trade_ids.size} duplicate trade ids from Kraken page ignored")
end
all_trades.merge!(trades.except(*duplicate_trade_ids))
count = result.to_h["count"].to_i
break if trades.size < TRADE_PAGE_SIZE
offset += trades.size
break if count.positive? && offset >= count
end
all_trades
end
def upsert_kraken_account(assets:, balances:, trades:, asset_metadata:, pair_metadata:, api_key_info:, total_usd:)
kraken_item.kraken_accounts.find_or_initialize_by(account_id: "combined").tap do |account|
account.assign_attributes(
name: kraken_item.institution_name.presence || "Kraken",
account_type: "combined",
currency: "USD",
current_balance: total_usd,
institution_metadata: institution_metadata(assets),
raw_payload: {
"balances" => balances,
"assets" => assets.map(&:stringify_keys),
"asset_metadata" => asset_metadata,
"pair_metadata" => pair_metadata,
"api_key_info" => api_key_info,
"fetched_at" => Time.current.iso8601
},
raw_transactions_payload: {
"trades" => trades,
"fetched_at" => Time.current.iso8601
},
extra: account.extra.to_h.deep_merge(price_metadata(assets))
)
account.save!
end
end
def institution_metadata(assets)
{
"name" => "Kraken",
"domain" => "kraken.com",
"url" => "https://www.kraken.com",
"color" => "#5841D8",
"asset_count" => assets.size,
"assets" => assets.map { |asset| asset[:symbol] }
}
end
def price_metadata(assets)
missing = assets.select { |asset| asset[:price_status] == "missing" }.map { |asset| asset[:symbol] }
stale = assets.select { |asset| asset[:price_status] == "stale" }.map { |asset| asset[:symbol] }
{ "kraken" => { "missing_prices" => missing, "stale_prices" => stale } }
end
end

View File

@@ -0,0 +1,15 @@
# frozen_string_literal: true
module KrakenItem::Provided
extend ActiveSupport::Concern
def kraken_provider
return nil unless credentials_configured?
Provider::Kraken.new(
api_key: api_key.to_s.strip,
api_secret: api_secret.to_s.strip,
nonce_generator: -> { next_nonce! }
)
end
end

View File

@@ -0,0 +1,20 @@
# frozen_string_literal: true
class KrakenItem::SyncCompleteEvent
def initialize(kraken_item)
raise ArgumentError, "kraken_item is required" unless kraken_item.respond_to?(:family) && kraken_item.respond_to?(:id)
@kraken_item = kraken_item
end
def broadcast
Turbo::StreamsChannel.broadcast_replace_to(
@kraken_item.family,
target: ActionView::RecordIdentifier.dom_id(@kraken_item),
partial: "kraken_items/kraken_item",
locals: { kraken_item: @kraken_item }
)
rescue StandardError => e
Rails.logger.warn("KrakenItem::SyncCompleteEvent failed for #{@kraken_item.id}: #{e.class}")
end
end

View File

@@ -0,0 +1,81 @@
# frozen_string_literal: true
class KrakenItem::Syncer
include SyncStats::Collector
attr_reader :kraken_item
def initialize(kraken_item)
@kraken_item = kraken_item
end
def perform_sync(sync)
sync.update!(status_text: I18n.t("kraken_item.syncer.checking_credentials")) if sync.respond_to?(:status_text)
unless kraken_item.credentials_configured?
kraken_item.update!(status: :requires_update)
mark_failed(sync, I18n.t("kraken_item.syncer.credentials_invalid"))
return
end
sync.update!(status_text: I18n.t("kraken_item.syncer.importing_accounts")) if sync.respond_to?(:status_text)
kraken_item.import_latest_kraken_data
kraken_item.update!(status: :good) if kraken_item.requires_update?
sync.update!(status_text: I18n.t("kraken_item.syncer.checking_configuration")) if sync.respond_to?(:status_text)
collect_setup_stats(sync, provider_accounts: kraken_item.kraken_accounts.to_a)
unlinked = kraken_item.kraken_accounts.left_joins(:account_provider).where(account_providers: { id: nil })
linked = kraken_item.kraken_accounts.joins(:account_provider).joins(:account).merge(Account.visible)
if unlinked.any?
kraken_item.update!(pending_account_setup: true)
sync.update!(status_text: I18n.t("kraken_item.syncer.accounts_need_setup", count: unlinked.count)) if sync.respond_to?(:status_text)
else
kraken_item.update!(pending_account_setup: false)
end
return unless linked.any?
sync.update!(status_text: I18n.t("kraken_item.syncer.processing_accounts")) if sync.respond_to?(:status_text)
kraken_item.process_accounts
sync.update!(status_text: I18n.t("kraken_item.syncer.calculating_balances")) if sync.respond_to?(:status_text)
kraken_item.schedule_account_syncs(
parent_sync: sync,
window_start_date: sync.window_start_date,
window_end_date: sync.window_end_date
)
account_ids = linked.map { |kraken_account| kraken_account.current_account&.id }.compact
if account_ids.any?
collect_transaction_stats(sync, account_ids: account_ids, source: "kraken")
collect_trades_stats(sync, account_ids: account_ids, source: "kraken")
end
rescue Provider::Kraken::AuthenticationError, Provider::Kraken::PermissionError, Provider::Kraken::OTPRequiredError => e
kraken_item.update!(status: :requires_update)
mark_failed(sync, e.message)
raise
rescue StandardError => e
Rails.logger.error "KrakenItem::Syncer - unexpected error during sync: #{e.message}\n#{e.backtrace&.first(5)&.join("\n")}"
mark_failed(sync, e.message)
raise
end
def perform_post_sync
end
private
def mark_failed(sync, error_message)
sync.start! if sync.respond_to?(:may_start?) && sync.may_start?
if sync.respond_to?(:may_fail?) && sync.may_fail?
sync.fail!
elsif sync.respond_to?(:status)
sync.update!(status: :failed)
end
sync.update!(error: error_message) if sync.respond_to?(:error)
sync.update!(status_text: error_message) if sync.respond_to?(:status_text)
end
end

View File

@@ -0,0 +1,37 @@
# frozen_string_literal: true
module KrakenItem::Unlinking
extend ActiveSupport::Concern
def unlink_all!(dry_run: false)
results = []
links_by_provider_id = AccountProvider
.where(provider_type: KrakenAccount.name, provider_id: kraken_accounts.select(:id))
.group_by { |link| link.provider_id.to_s }
kraken_accounts.find_each do |provider_account|
links = links_by_provider_id[provider_account.id.to_s] || []
link_ids = links.map(&:id)
result = {
provider_account_id: provider_account.id,
name: provider_account.name,
provider_link_ids: link_ids
}
results << result
next if dry_run
begin
ActiveRecord::Base.transaction do
Holding.where(account_provider_id: link_ids).update_all(account_provider_id: nil) if link_ids.any?
links.each(&:destroy!)
end
rescue StandardError => e
Rails.logger.warn("KrakenItem Unlinker: failed to unlink ##{provider_account.id}: #{e.class} - #{e.message}")
result[:error] = e.message
end
end
results
end
end

View File

@@ -1,6 +1,8 @@
class LunchflowItem < ApplicationRecord
include Syncable, Provided, Unlinking, Encryptable
DEFAULT_BASE_URL = "https://lunchflow.app/api/v1".freeze
enum :status, { good: "good", requires_update: "requires_update" }, default: :good
# Encrypt sensitive credentials and raw payloads if ActiveRecord encryption is configured
@@ -154,6 +156,17 @@ class LunchflowItem < ApplicationRecord
end
def effective_base_url
base_url.presence || "https://lunchflow.app/api/v1"
return DEFAULT_BASE_URL if base_url.blank?
uri = URI.parse(base_url)
return DEFAULT_BASE_URL unless uri.is_a?(URI::HTTPS)
return DEFAULT_BASE_URL unless uri.host == "lunchflow.app"
return DEFAULT_BASE_URL unless [ "", "/", "/api/v1", "/api/v1/" ].include?(uri.path)
return DEFAULT_BASE_URL unless uri.query.blank?
return DEFAULT_BASE_URL unless uri.fragment.blank?
DEFAULT_BASE_URL
rescue URI::InvalidURIError
DEFAULT_BASE_URL
end
end

View File

@@ -1,6 +1,24 @@
class MintImport < Import
after_create :set_mappings
DEFAULT_COLUMN_MAPPINGS = {
signage_convention: "inflows_positive",
date_col_label: "Date",
date_format: "%m/%d/%Y",
name_col_label: "Description",
amount_col_label: "Amount",
currency_col_label: "Currency",
account_col_label: "Account Name",
category_col_label: "Category",
tags_col_label: "Labels",
notes_col_label: "Notes",
entity_type_col_label: "Transaction Type"
}.freeze
def self.default_column_mappings
DEFAULT_COLUMN_MAPPINGS
end
def generate_rows_from_csv
rows.destroy_all
@@ -83,18 +101,7 @@ class MintImport < Import
private
def set_mappings
self.signage_convention = "inflows_positive"
self.date_col_label = "Date"
self.date_format = "%m/%d/%Y"
self.name_col_label = "Description"
self.amount_col_label = "Amount"
self.currency_col_label = "Currency"
self.account_col_label = "Account Name"
self.category_col_label = "Category"
self.tags_col_label = "Labels"
self.notes_col_label = "Notes"
self.entity_type_col_label = "Transaction Type"
assign_attributes(self.class.default_column_mappings)
save!
end
end

View File

@@ -25,10 +25,13 @@ class OidcIdentity < ApplicationRecord
groups: groups
})
# Sync name to user if provided (keep existing if IdP doesn't provide)
# Sync name to user only when Sure has nothing on file (first link, or an
# admin blanked the field). Edits made inside Sure must survive subsequent
# SSO logins — previously the IdP value won unconditionally and clobbered
# any manually-edited name on every login (#1103).
user.update!(
first_name: auth.info&.first_name.presence || user.first_name,
last_name: auth.info&.last_name.presence || user.last_name
first_name: user.first_name.presence || auth.info&.first_name.presence,
last_name: user.last_name.presence || auth.info&.last_name.presence
)
# Apply role mapping based on group membership

View File

@@ -35,6 +35,16 @@ class Provider::BinancePublic < Provider
MS_PER_DAY = 24 * 60 * 60 * 1000
SEARCH_LIMIT = 25
# USD-pegged stablecoins. Binance has no self-pair (USDTUSDT is invalid) and
# the few stablecoin/USDT pairs that do exist (USDCUSDT, etc.) hover at ~1.0
# with sub-cent noise — synthesizing a flat 1.0 USD price is both accurate
# enough and avoids surfacing transient depeg ticks from market data.
USD_STABLECOINS = %w[USDT USDC BUSD DAI FDUSD TUSD USDP PYUSD].freeze
# Symbol prefix applied by holdings processors (CoinStats, Coinbase, Kraken,
# Binance, SimpleFIN, Lunchflow) to distinguish crypto from stock tickers.
CRYPTO_PREFIX = "CRYPTO:".freeze
def initialize
# No API key required — public market data only.
end
@@ -58,9 +68,13 @@ class Provider::BinancePublic < Provider
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
with_provider_response do
query = symbol.to_s.strip.upcase
query = symbol.to_s.strip.upcase.delete_prefix(CRYPTO_PREFIX)
next [] if query.empty?
if USD_STABLECOINS.include?(query)
next [ stablecoin_search_result(query) ]
end
symbols = exchange_info_symbols
matches = symbols.select do |s|
@@ -128,10 +142,12 @@ class Provider::BinancePublic < Provider
# logo_url is intentionally nil — crypto logos are set at save time by
# Security#generate_logo_url_from_brandfetch via the /crypto/{base}
# route, not returned from this provider.
links = parsed[:binance_pair] ? "https://www.binance.com/en/trade/#{parsed[:binance_pair]}" : nil
SecurityInfo.new(
symbol: symbol,
name: parsed[:base],
links: "https://www.binance.com/en/trade/#{parsed[:binance_pair]}",
links: links,
logo_url: nil,
description: nil,
kind: "crypto",
@@ -161,6 +177,10 @@ class Provider::BinancePublic < Provider
parsed = parse_ticker(symbol)
raise InvalidSecurityPriceError, "Unsupported Binance ticker: #{symbol}" if parsed.nil?
if parsed[:stablecoin]
next stablecoin_prices(symbol, parsed, start_date, end_date, exchange_operating_mic)
end
binance_pair = parsed[:binance_pair]
display_currency = parsed[:display_currency]
prices = []
@@ -220,6 +240,36 @@ class Provider::BinancePublic < Provider
end
private
# Synthetic search hit for a USD-pegged stablecoin. Binance has no self-pair
# (USDTUSDT etc. don't exist), so we manufacture a result instead of letting
# the resolver fall back to an offline CRYPTO:* row. The downstream price
# path short-circuits via parse_ticker -> stablecoin_prices.
def stablecoin_search_result(base)
Security.new(
symbol: "#{base}USD",
name: base,
logo_url: ::Security.brandfetch_crypto_url(base),
exchange_operating_mic: BINANCE_MIC,
country_code: nil,
currency: "USD"
)
end
# Synthesize flat 1.0 USD prices for USD-pegged stablecoins across the
# requested range. Avoids a Binance round-trip (there is no self-pair like
# USDTUSDT) and produces stable values for portfolio aggregation.
def stablecoin_prices(symbol, parsed, start_date, end_date, exchange_operating_mic)
(start_date..end_date).map do |date|
Price.new(
symbol: symbol,
date: date,
price: 1.0,
currency: parsed[:display_currency],
exchange_operating_mic: exchange_operating_mic
)
end
end
def base_url
ENV["BINANCE_PUBLIC_URL"] || "https://data-api.binance.vision"
end
@@ -247,11 +297,24 @@ class Provider::BinancePublic < Provider
end
end
# Maps a user-visible ticker (e.g. "BTCUSD", "ETHEUR") to the Binance pair
# symbol, base asset, and display currency. Returns nil if the ticker does
# not end with a supported quote currency.
# Maps a user-visible ticker to the Binance pair symbol, base asset, and
# display currency. Accepts:
# - "BTCUSD"/"ETHEUR" — fiat suffix from search_securities output
# - "CRYPTO:BTCUSD" — prefixed form stored by holdings processors
# - "CRYPTO:SOL"/"SOL" — bare base asset; defaults to the USDT pair (USD)
# - "CRYPTO:USDT"/"USDT" — USD-pegged stablecoin; binance_pair is nil and
# callers short-circuit to a synthetic 1.0 USD price
# Returns nil only when the input is empty after stripping the prefix.
def parse_ticker(ticker)
ticker_up = ticker.to_s.upcase
raw = ticker.to_s.upcase
prefixed = raw.start_with?(CRYPTO_PREFIX)
ticker_up = raw.delete_prefix(CRYPTO_PREFIX)
return nil if ticker_up.empty?
if USD_STABLECOINS.include?(ticker_up)
return { binance_pair: nil, base: ticker_up, display_currency: "USD", stablecoin: true }
end
SUPPORTED_QUOTES.each do |quote|
display_currency = QUOTE_TO_CURRENCY[quote]
next unless ticker_up.end_with?(display_currency)
@@ -259,9 +322,22 @@ class Provider::BinancePublic < Provider
base = ticker_up.delete_suffix(display_currency)
next if base.empty?
# "{stablecoin}USD" form (e.g. "USDTUSD" produced by search_securities)
# routes to synthetic 1.0 USD pricing — there is no Binance self-pair.
if display_currency == "USD" && USD_STABLECOINS.include?(base)
return { binance_pair: nil, base: base, display_currency: "USD", stablecoin: true }
end
return { binance_pair: "#{base}#{quote}", base: base, display_currency: display_currency }
end
nil
# No fiat suffix matched. Only treat the input as a bare base asset when
# it arrived with the CRYPTO: prefix from a holdings processor — that
# tells us it really is a single coin symbol (SOL, TRUMP, KAITO), not a
# malformed pair like "BTCBNB" or "BTCGBP" that we want to reject.
return nil unless prefixed
{ binance_pair: "#{ticker_up}USDT", base: ticker_up, display_currency: "USD" }
end
# Cached for 24h — exchangeInfo returns the full symbol universe (thousands

271
app/models/provider/brex.rb Normal file
View File

@@ -0,0 +1,271 @@
# frozen_string_literal: true
class Provider::Brex
include HTTParty
extend SslConfigurable
DEFAULT_BASE_URL = "https://api.brex.com"
STAGING_BASE_URL = "https://api-staging.brex.com"
ALLOWED_BASE_URLS = [ DEFAULT_BASE_URL, STAGING_BASE_URL ].freeze
DEFAULT_LIMIT = 1000
# Transaction syncs are date-window bounded; this is only a runaway cursor guard.
MAX_PAGES = 25
headers "User-Agent" => "Sure Finance Brex Client"
default_options.merge!({ timeout: 120 }.merge(httparty_ssl_options))
attr_reader :token, :base_url
def initialize(token, base_url: DEFAULT_BASE_URL)
@token = token.to_s.strip
@base_url = self.class.normalize_base_url(base_url)
raise ArgumentError, "Brex base URL must be blank or one of: #{ALLOWED_BASE_URLS.join(', ')}" unless @base_url.present?
end
def self.normalize_base_url(value)
stripped = value.to_s.strip
return DEFAULT_BASE_URL if stripped.blank?
uri = URI.parse(stripped)
return nil unless uri.is_a?(URI::HTTPS)
return nil if uri.userinfo.present?
return nil if uri.query.present? || uri.fragment.present?
return nil unless uri.path.blank? || uri.path == "/"
return nil unless uri.port == 443
# This exact allowlist is the SSRF boundary; arbitrary Brex-like hosts are never accepted.
normalized = "#{uri.scheme.downcase}://#{uri.host.to_s.downcase}"
ALLOWED_BASE_URLS.include?(normalized) ? normalized : nil
rescue URI::InvalidURIError
nil
end
def self.allowed_base_url?(value)
normalize_base_url(value).present?
end
def get_accounts
cash_accounts = get_cash_accounts
card_accounts = get_card_accounts
accounts = cash_accounts.dup
accounts << aggregate_card_account(card_accounts) if card_accounts.any?
{
accounts: accounts,
cash_accounts: cash_accounts,
card_accounts: card_accounts
}
end
def get_cash_accounts
get_paginated("/v2/accounts/cash").map { |account| account.with_indifferent_access.merge(account_kind: "cash") }
end
def get_card_accounts
get_paginated("/v2/accounts/card").map { |account| account.with_indifferent_access.merge(account_kind: "card") }
end
def get_cash_transactions(account_id, start_date: nil)
path = "/v2/transactions/cash/#{ERB::Util.url_encode(account_id.to_s)}"
{
transactions: get_paginated(path, params: posted_at_start_params(start_date))
}
end
def get_primary_card_transactions(start_date: nil)
{
transactions: get_paginated("/v2/transactions/card/primary", params: posted_at_start_params(start_date))
}
end
private
def aggregate_card_account(card_accounts)
totals = %i[current_balance available_balance account_limit].index_with do |field|
sum_money(card_accounts.filter_map { |account| account.with_indifferent_access[field] })
end
{
id: BrexAccount.card_account_id,
name: "Brex Card",
account_kind: "card",
status: card_accounts.map { |account| account.with_indifferent_access[:status] }.compact.first,
card_accounts_count: card_accounts.count,
current_balance: totals[:current_balance],
available_balance: totals[:available_balance],
account_limit: totals[:account_limit],
raw_card_accounts: BrexAccount.sanitize_payload(card_accounts)
}.compact
end
def sum_money(money_values)
normalized = money_values.compact
return nil if normalized.empty?
currencies = normalized.map { |money| BrexAccount.currency_code_from_money(money) }.uniq
if currencies.many?
Rails.logger.warn "Brex API: Cannot aggregate card balances with mixed currencies: #{currencies.join(', ')}"
return nil
end
currency = currencies.first
total = normalized.sum do |money|
money.with_indifferent_access[:amount].to_i
end
{ amount: total, currency: currency }
end
def posted_at_start_params(start_date)
return {} if start_date.blank?
{ posted_at_start: rfc3339_start_date(start_date) }
end
def get_paginated(path, params: {})
records = []
cursor = nil
seen_cursors = Set.new
page_count = 0
loop do
page_count += 1
raise BrexError.new("Brex pagination exceeded #{MAX_PAGES} pages", :pagination_error) if page_count > MAX_PAGES
page_params = params.compact.merge(limit: DEFAULT_LIMIT)
page_params[:cursor] = cursor if cursor.present?
response_payload = get_json(path, params: page_params)
if response_payload.is_a?(Array)
records.concat(response_payload)
break
end
page_records = extract_records(response_payload)
records.concat(page_records)
next_cursor = response_payload.with_indifferent_access[:next_cursor]
break if next_cursor.blank?
if seen_cursors.include?(next_cursor)
raise BrexError.new("Brex pagination returned a repeated cursor", :pagination_error)
end
seen_cursors.add(next_cursor)
cursor = next_cursor
end
records
end
def get_json(path, params: {})
query = params.present? ? "?#{URI.encode_www_form(params)}" : ""
request_path = "#{path}#{query}"
response = self.class.get(
"#{base_url}#{request_path}",
headers: auth_headers
)
handle_response(response, path: path)
rescue BrexError
raise
rescue SocketError, Net::OpenTimeout, Net::ReadTimeout => e
Rails.logger.error "Brex API: GET #{path} failed: #{e.class}: #{e.message}"
raise BrexError.new("Exception during GET request: #{e.message}", :request_failed)
rescue JSON::ParserError => e
Rails.logger.error "Brex API: invalid JSON for GET #{path}: #{e.message}"
raise BrexError.new("Invalid response from Brex API", :invalid_response)
rescue => e
Rails.logger.error "Brex API: Unexpected error during GET #{path}: #{e.class}: #{e.message}"
raise BrexError.new("Exception during GET request: #{e.message}", :request_failed)
end
def extract_records(response_payload)
return response_payload if response_payload.is_a?(Array)
payload = response_payload.with_indifferent_access
payload[:items] ||
payload[:data] ||
payload[:accounts] ||
payload[:transactions] ||
[]
end
def auth_headers
{
"Authorization" => "Bearer #{token}",
"Content-Type" => "application/json",
"Accept" => "application/json"
}
end
def handle_response(response, path:)
trace_id = brex_trace_id(response)
case response.code
when 200
parse_json(response.body)
when 400
Rails.logger.error "Brex API: bad request for #{path} trace_id=#{trace_id}"
raise BrexError.new("Bad request to Brex API", :bad_request, http_status: 400, trace_id: trace_id)
when 401
Rails.logger.warn "Brex API: unauthorized for #{path} trace_id=#{trace_id}"
raise BrexError.new("Invalid Brex API token or account permissions", :unauthorized, http_status: 401, trace_id: trace_id)
when 403
Rails.logger.warn "Brex API: access forbidden for #{path} trace_id=#{trace_id}"
raise BrexError.new("Access forbidden - check Brex API token scopes", :access_forbidden, http_status: 403, trace_id: trace_id)
when 404
Rails.logger.warn "Brex API: resource not found for #{path} trace_id=#{trace_id}"
raise BrexError.new("Brex resource not found", :not_found, http_status: 404, trace_id: trace_id)
when 429
Rails.logger.warn "Brex API: rate limited for #{path} trace_id=#{trace_id}"
raise BrexError.new("Brex rate limit exceeded. Please try again later.", :rate_limited, http_status: 429, trace_id: trace_id)
else
Rails.logger.error "Brex API: unexpected response code=#{response.code} path=#{path} trace_id=#{trace_id}"
raise BrexError.new("Failed to fetch data from Brex API: HTTP #{response.code}", :fetch_failed, http_status: response.code, trace_id: trace_id)
end
end
def parse_json(body)
return {} if body.blank?
JSON.parse(body, symbolize_names: true)
end
def rfc3339_start_date(start_date)
time =
case start_date
when Time
start_date
when DateTime
start_date.to_time
when Date
start_date.to_time(:utc)
else
Time.zone.parse(start_date.to_s)
end
raise ArgumentError, "Invalid start_date: #{start_date.inspect}" if time.nil?
time.utc.iso8601
end
def brex_trace_id(response)
headers = response.respond_to?(:headers) ? response.headers : {}
headers["X-Brex-Trace-Id"].presence ||
headers["x-brex-trace-id"].presence
end
class BrexError < StandardError
attr_reader :error_type, :http_status, :trace_id
def initialize(message, error_type = :unknown, http_status: nil, trace_id: nil)
super(message)
@error_type = error_type
@http_status = http_status
@trace_id = trace_id
end
end
end

View File

@@ -0,0 +1,119 @@
class Provider::BrexAdapter < Provider::Base
include Provider::Syncable
include Provider::InstitutionMetadata
# Register this adapter with the factory
Provider::Factory.register("BrexAccount", self)
def self.supported_account_types
%w[Depository CreditCard]
end
# Returns connection configurations for this provider
def self.connection_configs(family:)
return [] unless family.can_connect_brex?
brex_items = family.brex_items.active.with_credentials.ordered
return [ connection_config_for(nil) ] if brex_items.empty?
brex_items.map { |brex_item| connection_config_for(brex_item) }
end
def provider_name
"brex"
end
# Build a Brex provider instance with family-specific credentials
# @param family [Family] The family to get credentials for (required)
# @return [Provider::Brex, nil] Returns nil if credentials are not configured
def self.build_provider(family: nil, brex_item_id: nil)
return nil unless family.present?
brex_item = BrexItem.resolve_for(family: family, brex_item_id: brex_item_id)
return nil unless brex_item&.credentials_configured?
base_url = brex_item.effective_base_url
return nil unless base_url.present?
Provider::Brex.new(
brex_item.token.to_s.strip,
base_url: base_url
)
end
def self.connection_config_for(brex_item)
path_params = ->(extra = {}) do
brex_item.present? ? extra.merge(brex_item_id: brex_item.id) : extra
end
{
key: brex_item.present? ? "brex_#{brex_item.id}" : "brex",
name: brex_item.present? ? I18n.t("brex_items.provider_connection.name", name: brex_item.name) : I18n.t("brex_items.provider_connection.default_name"),
description: brex_item.present? ? I18n.t("brex_items.provider_connection.description", name: brex_item.name) : I18n.t("brex_items.provider_connection.default_description"),
can_connect: true,
new_account_path: ->(accountable_type, return_to) {
Rails.application.routes.url_helpers.select_accounts_brex_items_path(
path_params.call(accountable_type: accountable_type, return_to: return_to)
)
},
existing_account_path: ->(account_id) {
Rails.application.routes.url_helpers.select_existing_account_brex_items_path(
path_params.call(account_id: account_id)
)
}
}
end
private_class_method :connection_config_for
def sync_path
Rails.application.routes.url_helpers.sync_brex_item_path(item)
end
def item
provider_account.brex_item
end
def can_delete_holdings?
false
end
def institution_domain
metadata = provider_account.institution_metadata
return nil unless metadata.present?
domain = metadata["domain"]
url = metadata["url"]
# Derive domain from URL if missing
if domain.blank? && url.present?
begin
parsed_host = URI.parse(url).host
Rails.logger.warn("Brex account #{provider_account.id} institution URL has no host: #{url}") if parsed_host.nil?
domain = parsed_host&.gsub(/^www\./, "")
rescue URI::InvalidURIError
Rails.logger.warn("Invalid institution URL for Brex account #{provider_account.id}: #{url}")
end
end
domain
end
def institution_name
metadata = provider_account.institution_metadata
metadata&.dig("name") || item&.institution_name
end
def institution_url
metadata = provider_account.institution_metadata
metadata&.dig("url") || item&.institution_url
end
def institution_color
metadata = provider_account.institution_metadata
metadata&.dig("color") || item&.institution_color
end
end

View File

@@ -0,0 +1,59 @@
class Provider::IbkrAdapter < Provider::Base
include Provider::Syncable
include Provider::InstitutionMetadata
Provider::Factory.register("IbkrAccount", self)
def self.supported_account_types
%w[Investment]
end
def self.connection_configs(family:)
return [] unless family.can_connect_ibkr?
[ {
key: "ibkr",
name: I18n.t("providers.ibkr.name"),
description: I18n.t("providers.ibkr.connection_description"),
can_connect: true,
new_account_path: ->(_accountable_type, _return_to) {
Rails.application.routes.url_helpers.select_accounts_ibkr_items_path
},
existing_account_path: ->(account_id) {
Rails.application.routes.url_helpers.select_existing_account_ibkr_items_path(account_id: account_id)
}
} ]
end
def provider_name
"ibkr"
end
def sync_path
Rails.application.routes.url_helpers.sync_ibkr_item_path(item)
end
def item
provider_account.ibkr_item
end
def can_delete_holdings?
false
end
def institution_domain
"interactivebrokers.com"
end
def institution_name
I18n.t("providers.ibkr.institution_name")
end
def institution_url
"https://www.interactivebrokers.com"
end
def institution_color
"#D32F2F"
end
end

View File

@@ -0,0 +1,144 @@
class Provider::IbkrFlex
include HTTParty
extend SslConfigurable
class Error < StandardError; end
class AuthenticationError < Error; end
class ConfigurationError < Error; end
class ApiError < Error
attr_reader :status_code, :response_body, :error_code
def initialize(message, status_code: nil, response_body: nil, error_code: nil)
super(message)
@status_code = status_code
@response_body = response_body
@error_code = error_code
end
end
base_uri "https://ndcdyn.interactivebrokers.com/AccountManagement/FlexWebService"
headers "User-Agent" => "Sure Finance IBKR Flex Client"
default_options.merge!({ timeout: 120 }.merge(httparty_ssl_options))
MAX_RETRIES = 3
INITIAL_RETRY_DELAY = 2
MAX_RETRY_DELAY = 30
POLL_INTERVAL = 3
MAX_POLL_ATTEMPTS = 20
PENDING_ERROR_CODES = %w[1004 1019].freeze
RETRYABLE_ERRORS = [
SocketError,
Net::OpenTimeout,
Net::ReadTimeout,
Errno::ECONNRESET,
Errno::ECONNREFUSED,
Errno::ETIMEDOUT,
EOFError
].freeze
attr_reader :query_id, :token
def initialize(query_id:, token:)
raise ConfigurationError, "query_id is required" if query_id.blank?
raise ConfigurationError, "token is required" if token.blank?
@query_id = query_id.to_s.strip
@token = token.to_s.strip
end
def download_statement
reference_code = request_reference_code
poll_statement(reference_code)
end
private
def request_reference_code
response = with_retries("SendRequest") do
self.class.get("/SendRequest", query: { t: token, q: query_id, v: 3 })
end
xml = parse_xml(response.body)
error = response_error(xml, response)
raise error if error
reference_code = xml.at_xpath("//ReferenceCode")&.text.to_s.strip
raise ApiError.new("IBKR Flex did not return a reference code.", status_code: response.code, response_body: response.body) if reference_code.blank?
reference_code
end
def poll_statement(reference_code)
attempts = 0
loop do
attempts += 1
response = with_retries("GetStatement") do
self.class.get("/GetStatement", query: { t: token, q: reference_code, v: 3 })
end
xml = parse_xml(response.body)
return response.body if xml.at_xpath("//FlexQueryResponse")
error = response_error(xml, response)
if error.is_a?(ApiError) && PENDING_ERROR_CODES.include?(error.error_code.to_s)
raise ApiError.new("IBKR Flex statement is still being generated.", error_code: error.error_code) if attempts >= MAX_POLL_ATTEMPTS
sleep(POLL_INTERVAL)
next
end
raise(error || ApiError.new("IBKR Flex returned an unexpected response.", status_code: response.code, response_body: response.body))
end
end
def response_error(xml, response)
error_code = xml.at_xpath("//ErrorCode")&.text.to_s.strip.presence
error_message = xml.at_xpath("//ErrorMessage")&.text.to_s.strip.presence
return nil if error_code.blank? && response.success?
message = error_message.presence || "IBKR Flex request failed"
case error_code
when "1012", "1015"
AuthenticationError.new(message)
when "1014"
ConfigurationError.new(message)
else
ApiError.new(message, status_code: response.code, response_body: response.body, error_code: error_code)
end
end
def parse_xml(body)
Nokogiri::XML(body.to_s)
end
def with_retries(operation_name, max_retries: MAX_RETRIES)
retries = 0
begin
yield
rescue *RETRYABLE_ERRORS => e
retries += 1
if retries <= max_retries
delay = calculate_retry_delay(retries)
Rails.logger.warn(
"IBKR Flex: #{operation_name} failed (attempt #{retries}/#{max_retries}): #{e.class}: #{e.message}. Retrying in #{delay}s..."
)
sleep(delay)
retry
end
raise ApiError.new("Network error after #{max_retries} retries: #{e.message}")
end
end
def calculate_retry_delay(retry_count)
base_delay = INITIAL_RETRY_DELAY * (2**(retry_count - 1))
jitter = base_delay * rand * 0.25
[ base_delay + jitter, MAX_RETRY_DELAY ].min
end
end

View File

@@ -0,0 +1,153 @@
# frozen_string_literal: true
class Provider::Kraken
include HTTParty
extend SslConfigurable
class Error < StandardError; end
class AuthenticationError < Error; end
class PermissionError < Error; end
class RateLimitError < Error; end
class NonceError < Error; end
class OTPRequiredError < Error; end
class ApiError < Error; end
BASE_URL = "https://api.kraken.com"
PRIVATE_PREFIX = "/0/private"
PUBLIC_PREFIX = "/0/public"
base_uri BASE_URL
default_options.merge!({ timeout: 30 }.merge(httparty_ssl_options))
attr_reader :api_key, :api_secret
def initialize(api_key:, api_secret:, nonce_generator: nil)
@api_key = api_key # pipelock:ignore user-supplied Kraken credential kept in memory for signed requests
@api_secret = api_secret # pipelock:ignore user-supplied Kraken credential kept in memory for signed requests
@nonce_generator = nonce_generator || -> { Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond).to_s }
end
def get_api_key_info
private_post("GetApiKeyInfo")
end
def get_extended_balance
private_post("BalanceEx")
end
def get_trades_history(start: nil, offset: nil)
params = {}
params["start"] = start.to_i.to_s if start.present?
params["ofs"] = offset.to_i.to_s if offset.present?
private_post("TradesHistory", params)
end
def get_asset_info(asset: nil)
params = {}
params["asset"] = asset if asset.present?
public_get("Assets", params)
end
def get_asset_pairs(pair: nil)
params = {}
params["pair"] = pair if pair.present?
public_get("AssetPairs", params)
end
def get_ticker(pair)
public_get("Ticker", "pair" => pair)
end
def get_ohlc(pair, interval: 1440, since: nil)
params = { "pair" => pair, "interval" => interval.to_s }
params["since"] = since.to_i.to_s if since.present?
public_get("OHLC", params)
end
private
attr_reader :nonce_generator
def public_get(method, params = {})
response = self.class.get("#{PUBLIC_PREFIX}/#{method}", query: params)
handle_response(response)
end
def private_post(method, params = {})
path = "#{PRIVATE_PREFIX}/#{method}"
request_params = { "nonce" => nonce_generator.call.to_s }.merge(stringify_params(params))
body = URI.encode_www_form(request_params)
response = self.class.post(
path,
body: body,
headers: auth_headers(path, request_params).merge("Content-Type" => "application/x-www-form-urlencoded")
)
handle_response(response)
end
def stringify_params(params)
params.each_with_object({}) { |(key, value), hash| hash[key.to_s] = value.to_s }
end
def auth_headers(path, params)
{
"API-Key" => api_key,
"API-Sign" => sign(path, params)
}
end
def sign(path, params)
encoded_payload = URI.encode_www_form(params)
nonce = params.fetch("nonce").to_s
digest = OpenSSL::Digest::SHA256.digest(nonce + encoded_payload)
hmac = OpenSSL::HMAC.digest("sha512", Base64.decode64(api_secret), path + digest)
Base64.strict_encode64(hmac)
end
def handle_response(response)
parsed = response.parsed_response
unless response.code.between?(200, 299)
raise ApiError, "Kraken API request failed: #{response.code}"
end
unless parsed.is_a?(Hash)
raise ApiError, "Malformed Kraken API response"
end
unless parsed.key?("error")
raise ApiError, "Malformed Kraken API response: missing error"
end
errors = Array(parsed["error"]).reject(&:blank?)
raise classified_error(errors) if errors.any?
unless parsed.key?("result")
raise ApiError, "Malformed Kraken API response: missing result"
end
parsed["result"]
end
def classified_error(errors)
message = errors.join(", ")
case message
when /Invalid key|Invalid signature|Temporary lockout/i
AuthenticationError.new(message)
when /Invalid nonce/i
NonceError.new(message)
when /Permission denied|Invalid permissions/i
PermissionError.new(message)
when /Rate limit exceeded|Too many requests|limit exceeded|Throttled/i
RateLimitError.new(message)
when /otp|2fa|two.factor/i
OTPRequiredError.new(message)
else
ApiError.new(message)
end
end
end

View File

@@ -0,0 +1,110 @@
# frozen_string_literal: true
class Provider::KrakenAdapter < Provider::Base
include Provider::Syncable
include Provider::InstitutionMetadata
Provider::Factory.register("KrakenAccount", self)
def self.supported_account_types
%w[Crypto]
end
def self.connection_configs(family:)
return [] unless family.can_connect_kraken?
kraken_items = family.kraken_items.active.credentials_configured.ordered.select(&:credentials_configured?)
return [ connection_config_for(nil) ] if kraken_items.empty?
kraken_items.map { |kraken_item| connection_config_for(kraken_item) }
end
def self.build_provider(family: nil, kraken_item_id: nil)
return nil unless family.present?
kraken_item = resolve_kraken_item(family, kraken_item_id)
return nil unless kraken_item&.credentials_configured?
kraken_item.kraken_provider
end
def provider_name
"kraken"
end
def sync_path
return unless item
Rails.application.routes.url_helpers.sync_kraken_item_path(item)
end
def item
provider_account.kraken_item
end
def can_delete_holdings?
false
end
def institution_domain
institution_metadata_value("domain")
end
def institution_name
institution_metadata_value("name")
end
def institution_url
institution_metadata_value("url")
end
def institution_color
institution_metadata_value("color")
end
def self.connection_config_for(kraken_item)
path_params = ->(extra = {}) do
kraken_item.present? ? extra.merge(kraken_item_id: kraken_item.id) : extra
end
{
key: kraken_item.present? ? "kraken_#{kraken_item.id}" : "kraken",
name: kraken_item.present? ? I18n.t("kraken_items.provider_connection.name", name: kraken_item.name) : I18n.t("kraken_items.provider_connection.default_name"),
description: kraken_item.present? ? I18n.t("kraken_items.provider_connection.description", name: kraken_item.name) : I18n.t("kraken_items.provider_connection.default_description"),
can_connect: true,
new_account_path: ->(accountable_type, return_to) {
Rails.application.routes.url_helpers.select_accounts_kraken_items_path(
path_params.call(accountable_type: accountable_type, return_to: return_to)
)
},
existing_account_path: ->(account_id) {
Rails.application.routes.url_helpers.select_existing_account_kraken_items_path(
path_params.call(account_id: account_id)
)
}
}
end
private_class_method :connection_config_for
def self.resolve_kraken_item(family, kraken_item_id)
if kraken_item_id.present?
item = family.kraken_items.active.credentials_configured.find_by(id: kraken_item_id)
return item if item&.credentials_configured?
return nil
end
credentialed_items = family.kraken_items.active.credentials_configured.ordered.select(&:credentials_configured?)
return credentialed_items.first if credentialed_items.one?
nil
end
private_class_method :resolve_kraken_item
private
def institution_metadata_value(key)
metadata = provider_account.institution_metadata || {}
metadata[key] || item&.public_send("institution_#{key}")
end
end

View File

@@ -6,9 +6,12 @@ class Provider
enable_banking: { region: "EU", kind: "Bank", maturity: :beta, logo_text: "EB", logo_bg: "bg-purple-600" },
coinstats: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "CS", logo_bg: "bg-pink-600" },
mercury: { region: "US", kind: "Bank", maturity: :beta, logo_text: "ME", logo_bg: "bg-cyan-600" },
brex: { region: "US", kind: "Bank", maturity: :beta, logo_text: "BX", logo_bg: "bg-emerald-600" },
coinbase: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "CB", logo_bg: "bg-blue-500" },
binance: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "BI", logo_bg: "bg-yellow-600" },
kraken: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "KR", logo_bg: "bg-violet-600" },
snaptrade: { region: "US / CA", kind: "Investment", maturity: :beta, logo_text: "ST", logo_bg: "bg-green-600" },
ibkr: { region: "Global", kind: "Investment", maturity: :beta, logo_text: "IB", logo_bg: "bg-red-600" },
indexa_capital: { region: "ES", kind: "Investment", maturity: :alpha, logo_text: "IC", logo_bg: "bg-red-600" },
sophtron: { region: "US", kind: "Bank", maturity: :alpha, logo_text: "SO", logo_bg: "bg-teal-600" },
plaid: { region: "US", kind: "Bank", tier: "Paid", maturity: :stable, logo_text: "PL", logo_bg: "bg-indigo-600" },

View File

@@ -8,9 +8,12 @@ class ProviderConnectionStatus
{ key: "enable_banking", type: "EnableBankingItem", association: :enable_banking_items, accounts: :enable_banking_accounts },
{ key: "coinbase", type: "CoinbaseItem", association: :coinbase_items, accounts: :coinbase_accounts },
{ key: "binance", type: "BinanceItem", association: :binance_items, accounts: :binance_accounts },
{ key: "kraken", type: "KrakenItem", association: :kraken_items, accounts: :kraken_accounts },
{ key: "coinstats", type: "CoinstatsItem", association: :coinstats_items, accounts: :coinstats_accounts },
{ key: "snaptrade", type: "SnaptradeItem", association: :snaptrade_items, accounts: :snaptrade_accounts, linked_accounts: :linked_accounts },
{ key: "ibkr", type: "IbkrItem", association: :ibkr_items, accounts: :ibkr_accounts },
{ key: "mercury", type: "MercuryItem", association: :mercury_items, accounts: :mercury_accounts },
{ key: "brex", type: "BrexItem", association: :brex_items, accounts: :brex_accounts },
{ key: "sophtron", type: "SophtronItem", association: :sophtron_items, accounts: :sophtron_accounts },
{ key: "indexa_capital", type: "IndexaCapitalItem", association: :indexa_capital_items, accounts: :indexa_capital_accounts }
].freeze

View File

@@ -1,5 +1,5 @@
class ProviderMerchant < Merchant
enum :source, { plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats", mercury: "mercury", indexa_capital: "indexa_capital", sophtron: "sophtron" }
enum :source, { plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats", mercury: "mercury", brex: "brex", indexa_capital: "indexa_capital", sophtron: "sophtron" }
validates :name, uniqueness: { scope: [ :source ] }
validates :source, presence: true

View File

@@ -3,6 +3,7 @@ class RecurringTransaction < ApplicationRecord
belongs_to :family
belongs_to :account, optional: true
belongs_to :destination_account, optional: true, class_name: "Account"
belongs_to :merchant, optional: true
monetize :amount
@@ -19,6 +20,7 @@ class RecurringTransaction < ApplicationRecord
validates :occurrence_count, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validate :merchant_or_name_present
validate :amount_variance_consistency
validate :transfer_endpoints_consistent
def merchant_or_name_present
if merchant_id.blank? && name.blank?
@@ -36,10 +38,50 @@ class RecurringTransaction < ApplicationRecord
end
end
# When this row represents a recurring transfer, both endpoints must be
# present, belong to the same family, and not be the same account.
def transfer_endpoints_consistent
return if destination_account_id.blank?
if account_id.blank?
errors.add(:account, "must be present on a recurring transfer")
elsif account.blank?
# account_id references a row that was destroyed. Mirror the
# destination_account.blank? branch so the source side surfaces a
# normal validation error too.
errors.add(:account, "must exist")
elsif destination_account.blank?
# destination_account_id references a row that was destroyed (or never
# existed). Surface as a normal validation error instead of letting
# the FK fire on save.
errors.add(:destination_account, "must exist")
elsif account_id == destination_account_id
errors.add(:destination_account, "cannot be the same as the source account")
elsif account.family_id != destination_account.family_id
errors.add(:destination_account, "must belong to the same family as the source account")
end
end
def transfer?
destination_account_id.present?
end
scope :for_family, ->(family) { where(family: family) }
scope :expected_soon, -> { active.where("next_expected_date <= ?", 1.month.from_now) }
scope :accessible_by, ->(user) {
where(account_id: Account.accessible_by(user).select(:id)).or(where(account_id: nil))
accessible_account_ids = Account.accessible_by(user).select(:id)
# A recurring row is accessible when:
# * its account_id is in the user's accessible set or null (legacy rows
# with no account scoping survive), AND
# * its destination_account_id is also accessible OR null (so a recurring
# transfer never leaks into the list of a user without access to BOTH
# endpoints).
where(account_id: accessible_account_ids)
.or(where(account_id: nil))
.merge(
where(destination_account_id: accessible_account_ids)
.or(where(destination_account_id: nil))
)
}
# Class methods for identification and cleanup
@@ -58,6 +100,44 @@ class RecurringTransaction < ApplicationRecord
Cleaner.new(family).cleanup_stale_transactions
end
# Create a manual recurring transfer from an existing Transfer pair.
# Mirrors `create_from_transaction` but populates source + destination
# accounts and skips merchant / variance lookup -- transfers are
# account-pair-shaped, not merchant-shaped.
def self.create_from_transfer(transfer)
outflow_entry = transfer.outflow_transaction&.entry
inflow_entry = transfer.inflow_transaction&.entry
raise ArgumentError, "transfer is missing one of its entries" unless outflow_entry && inflow_entry
source_account = outflow_entry.account
destination_account = inflow_entry.account
family = source_account.family
expected_day = outflow_entry.date.day
next_expected = calculate_next_expected_date_from_today(expected_day)
create!(
family: family,
account: source_account,
destination_account: destination_account,
merchant_id: nil,
# Transfer#name yields "Payment to ..." for liability destinations
# and "Transfer to ..." otherwise, matching Transfer::Creator's
# name_prefix logic so the recurring row reads consistently with
# the originating Transfer.
name: transfer.name,
amount: outflow_entry.amount, # positive (outflow), per Sure sign convention
currency: outflow_entry.currency,
expected_day_of_month: expected_day,
last_occurrence_date: outflow_entry.date,
next_expected_date: next_expected,
status: "active",
occurrence_count: 1,
manual: true
)
end
# Create a manual recurring transaction from an existing transaction
# Automatically calculates amount variance from past 6 months of matching transactions
def self.create_from_transaction(transaction, date_variance: 2)
@@ -313,7 +393,10 @@ class RecurringTransaction < ApplicationRecord
amount_min: expected_amount_min,
amount_max: expected_amount_max,
amount_avg: expected_amount_avg,
has_variance: has_amount_variance?
has_variance: has_amount_variance?,
transfer: transfer?,
source_account: account,
destination_account: destination_account
)
end

View File

@@ -7,11 +7,21 @@ class RecurringTransaction
end
# Mark recurring transactions as inactive if they haven't occurred recently
# Uses 2 months for automatic recurring, 6 months for manual recurring
# Uses 2 months for automatic recurring, 6 months for manual recurring.
#
# Transfer rows (destination_account_id present) are skipped: their
# `matching_transactions` helper looks at single-account name/amount
# which never matches a Transfer pair, so the Cleaner would
# incorrectly mark a still-recurring transfer inactive at the
# 6-month threshold. Issue #1590 tracks pair-detection-aware
# matching for recurring transfers.
def cleanup_stale_transactions
stale_count = 0
family.recurring_transactions.active.find_each do |recurring_transaction|
family.recurring_transactions
.active
.where(destination_account_id: nil)
.find_each do |recurring_transaction|
next unless recurring_transaction.should_be_inactive?
# Determine threshold based on manual flag

View File

@@ -10,14 +10,20 @@ class RecurringTransaction
def identify_recurring_patterns
three_months_ago = 3.months.ago.to_date
# Get all transactions from the last 3 months
# Skip transfer-kind transactions: they're one half of a Transfer pair, so grouping them
# under their single account would produce incoherent recurring "patterns" that don't
# represent the underlying account-pair flow. Recurring transfers are tracked on a
# different shape (RecurringTransaction with destination_account_id). Filtering at the
# SQL level avoids loading and discarding transfer entries for a busy family.
entries_with_transactions = family.entries
.joins("INNER JOIN transactions ON transactions.id = entries.entryable_id")
.where(entryable_type: "Transaction")
.where("entries.date >= ?", three_months_ago)
.where.not("transactions.kind": Transaction::TRANSFER_KINDS)
.includes(:entryable)
.to_a
# Group by merchant (if present) or name, along with amount (preserve sign) and currency
# Group by merchant (if present) or name, along with amount (preserve sign) and currency.
grouped_transactions = entries_with_transactions
.select { |entry| entry.entryable.is_a?(Transaction) }
.group_by do |entry|
@@ -140,9 +146,17 @@ class RecurringTransaction
recurring_patterns.size
end
# Update variance for existing manual recurring transactions
# Update variance for existing manual recurring transactions.
#
# Transfer rows (destination_account_id present) are skipped: their
# variance / occurrence tracking would need pair-detection across
# both endpoints rather than the single-account name/merchant match
# the helper performs. Issue #1590 tracks the proper Cleaner-aware
# matching for recurring transfers.
def update_manual_recurring_transactions(since_date)
family.recurring_transactions.where(manual: true, status: "active").find_each do |recurring|
family.recurring_transactions
.where(manual: true, status: "active", destination_account_id: nil)
.find_each do |recurring|
# Find matching transactions in the recent period
matching_entries = RecurringTransaction.find_matching_transaction_entries(
family: family,

View File

@@ -48,7 +48,7 @@ class SimplefinAccount::Investments::HoldingsProcessor
qty = parse_decimal(any_of(simplefin_holding, %w[shares quantity qty units]))
market_value = parse_decimal(any_of(simplefin_holding, %w[market_value current_value]))
raw_cost_basis, cost_basis_source_key = cost_basis_from(simplefin_holding)
cost_basis = normalize_cost_basis(raw_cost_basis, qty, cost_basis_source_key)
cost_basis = normalize_cost_basis(raw_cost_basis, qty, cost_basis_source_key, institution_reports_total_basis?)
# Derive price from market_value when possible; otherwise fall back to any price field
fallback_price = parse_decimal(any_of(simplefin_holding, %w[purchase_price price unit_price average_cost avg_cost]))
@@ -124,19 +124,50 @@ class SimplefinAccount::Investments::HoldingsProcessor
[ nil, nil ]
end
# Sure stores holding cost_basis as per-share average cost. Some SimpleFIN
# providers expose total position basis via total_cost/value, so normalize only
# when the selected provider field is known to represent total position basis.
def normalize_cost_basis(raw_cost_basis, qty, source_key)
# Sure stores holding cost_basis as per-share average cost. SimpleFIN
# brokerages are inconsistent about which field carries which shape:
#
# - total_cost / value: always a total position cost per the SimpleFIN
# spec and observed payloads; divide by qty unconditionally.
# - cost_basis / basis: the spec calls this per-share, and most
# brokerages comply. Keep these values unchanged by default.
#
# Exception: a small allowlist of brokerages (Vanguard, Fidelity) is
# known to populate cost_basis with the total position cost in violation
# of the spec (#1718, #1182). For those connections only, divide by qty.
#
# An earlier revision of this fix used a magnitude heuristic
# (share_price × √qty midpoint). It was withdrawn because a legitimate
# per-share basis on a holding with a large unrealized loss
# (e.g. 100 shares with basis $100 now worth $5) trips the midpoint and
# gets mis-divided to $1/share — corrupting compliant providers. The
# allowlist trades some manual maintenance for that safety.
def normalize_cost_basis(raw_cost_basis, qty, source_key, total_basis_institution = false)
return nil if raw_cost_basis.nil?
if %w[total_cost value].include?(source_key)
if %w[total_cost value].include?(source_key) ||
(total_basis_institution && %w[cost_basis basis].include?(source_key))
return nil unless qty.to_d.positive?
raw_cost_basis / qty
else
raw_cost_basis
return raw_cost_basis / qty
end
raw_cost_basis
end
# Institutions known to populate the SimpleFIN `cost_basis` / `basis`
# field with the total position cost rather than the per-share value the
# spec requires. Matched as case-insensitive substrings against the
# account's stored org name and domain.
TOTAL_BASIS_INSTITUTIONS = %w[vanguard fidelity].freeze
def institution_reports_total_basis?
org = simplefin_account.respond_to?(:org_data) ? simplefin_account.org_data : nil
return false if org.blank?
candidates = [ org["name"], org[:name], org["domain"], org[:domain] ].compact.map(&:to_s).map(&:downcase)
return false if candidates.empty?
TOTAL_BASIS_INSTITUTIONS.any? { |needle| candidates.any? { |c| c.include?(needle) } }
end
def resolve_security(symbol, description)

View File

@@ -247,9 +247,9 @@ class SnaptradeAccount::ActivitiesProcessor
def normalize_cash_amount(amount, activity_type)
case activity_type
when "WITHDRAWAL", "TRANSFER_OUT", "FEE", "TAX"
-amount.abs # These should be negative (money out)
amount.abs # Money out should be positive in Sure
when "CONTRIBUTION", "TRANSFER_IN", "DIVIDEND", "DIV", "INTEREST", "CASH"
amount.abs # These should be positive (money in)
-amount.abs # Money in should be negative in Sure
else
amount
end

View File

@@ -71,6 +71,11 @@ class SnaptradeAccount::Processor
end
def calculate_total_balance
if use_api_total_balance?
Rails.logger.debug "SnaptradeAccount::Processor - Using API total for multi-currency holdings for snaptrade_account=#{snaptrade_account.id}"
return snaptrade_account.current_balance || 0
end
# Calculate total from holdings + cash for accuracy
# SnapTrade's current_balance can sometimes be stale or just the cash value
holdings_value = calculate_holdings_value
@@ -109,4 +114,24 @@ class SnaptradeAccount::Processor
units * price
end
end
def use_api_total_balance?
return false unless snaptrade_account.current_balance.present?
holdings_currencies.any? { |currency| currency.present? && currency != snaptrade_account.currency }
end
def holdings_currencies
Array(snaptrade_account.raw_holdings_payload).filter_map do |holding|
data = holding.respond_to?(:with_indifferent_access) ? holding.with_indifferent_access : {}
extract_currency(data, extract_symbol_data(data), snaptrade_account.currency)
end.uniq
end
def extract_symbol_data(data)
symbol_wrapper = data[:symbol].is_a?(Hash) ? data[:symbol].with_indifferent_access : {}
raw_symbol_data = symbol_wrapper[:symbol]
raw_symbol_data.is_a?(Hash) ? raw_symbol_data.with_indifferent_access : {}
end
end

View File

@@ -160,13 +160,14 @@ module SnaptradeItem::Provided
return [] unless credentials_configured? && user_registered?
all_users = list_all_users
all_users.reject { |uid| uid == snaptrade_user_id }
all_users.select { |uid| uid != snaptrade_user_id && uid.start_with?("family_#{family_id}_") }
end
# Delete an orphaned SnapTrade user and all their connections
def delete_orphaned_user(user_id)
return false unless credentials_configured?
return false if user_id == snaptrade_user_id # Don't delete current user
return false unless user_id.start_with?("family_#{family_id}_")
snaptrade_provider.delete_user(user_id: user_id)
true

View File

@@ -1,5 +1,17 @@
class SureImport < Import
MAX_NDJSON_SIZE = 10.megabytes
IMPORTABLE_NDJSON_TYPES = {
"Account" => :accounts,
"Category" => :categories,
"Tag" => :tags,
"Merchant" => :merchants,
"Transaction" => :transactions,
"Trade" => :trades,
"Valuation" => :valuations,
"Budget" => :budgets,
"BudgetCategory" => :budget_categories,
"Rule" => :rules
}.freeze
ALLOWED_NDJSON_CONTENT_TYPES = %w[
application/x-ndjson
application/ndjson
@@ -11,6 +23,14 @@ class SureImport < Import
has_one_attached :ndjson_file, dependent: :purge_later
class << self
def max_row_count
100_000
end
def max_ndjson_size
MAX_NDJSON_SIZE
end
# Counts JSON lines by top-level "type" (used for dry-run summaries and row limits).
def ndjson_line_type_counts(content)
return {} unless content.present?
@@ -21,7 +41,7 @@ class SureImport < Import
begin
record = JSON.parse(line)
counts[record["type"]] += 1 if record["type"]
counts[record["type"]] += 1 if record.is_a?(Hash) && record["type"] && record.key?("data")
rescue JSON::ParserError
# Skip invalid lines
end
@@ -30,19 +50,17 @@ class SureImport < Import
end
def dry_run_totals_from_ndjson(content)
counts = ndjson_line_type_counts(content)
{
accounts: counts["Account"] || 0,
categories: counts["Category"] || 0,
tags: counts["Tag"] || 0,
merchants: counts["Merchant"] || 0,
transactions: counts["Transaction"] || 0,
trades: counts["Trade"] || 0,
valuations: counts["Valuation"] || 0,
budgets: counts["Budget"] || 0,
budget_categories: counts["BudgetCategory"] || 0,
rules: counts["Rule"] || 0
}
dry_run_totals_from_line_type_counts(ndjson_line_type_counts(content))
end
def dry_run_totals_from_line_type_counts(counts)
IMPORTABLE_NDJSON_TYPES.to_h do |record_type, entity_key|
[ entity_key, counts[record_type] || 0 ]
end
end
def importable_ndjson_types
IMPORTABLE_NDJSON_TYPES.keys
end
def valid_ndjson_first_line?(str)
@@ -53,7 +71,7 @@ class SureImport < Import
begin
record = JSON.parse(first_line)
record.key?("type") && record.key?("data")
record.is_a?(Hash) && record.key?("type") && record.key?("data")
rescue JSON::ParserError
false
end
@@ -121,7 +139,7 @@ class SureImport < Import
end
def max_row_count
100_000
self.class.max_row_count
end
# Row total for max-row enforcement (counts every parsed line with a "type", including unsupported types).

View File

@@ -14,6 +14,27 @@ class Trade < ApplicationRecord
validates :price, :currency, presence: true
validates :investment_activity_label, inclusion: { in: ACTIVITY_LABELS }, allow_nil: true
def exchange_rate
extra&.dig("exchange_rate")
end
def exchange_rate=(value)
if value.blank?
self.extra = (extra || {}).merge("exchange_rate" => nil, "exchange_rate_invalid" => false)
else
begin
normalized_value = Float(value)
raise ArgumentError unless normalized_value.finite?
self.extra = (extra || {}).merge("exchange_rate" => normalized_value, "exchange_rate_invalid" => false)
rescue ArgumentError, TypeError
self.extra = (extra || {}).merge("exchange_rate" => value, "exchange_rate_invalid" => true)
end
end
end
validate :exchange_rate_must_be_valid
# Trade types for categorization
def buy?
qty.positive?
@@ -57,6 +78,17 @@ class Trade < ApplicationRecord
private
def exchange_rate_must_be_valid
if extra&.dig("exchange_rate_invalid")
errors.add(:exchange_rate, "must be a number")
elsif exchange_rate.present?
numeric_rate = Float(exchange_rate) rescue nil
if numeric_rate.nil? || !numeric_rate.finite? || numeric_rate <= 0
errors.add(:exchange_rate, "must be greater than 0")
end
end
end
def calculate_realized_gain_loss
return nil unless sell?

View File

@@ -35,11 +35,13 @@ class Transaction < ApplicationRecord
def exchange_rate=(value)
if value.blank?
self.extra = (extra || {}).merge("exchange_rate" => nil)
self.extra = (extra || {}).merge("exchange_rate" => nil, "exchange_rate_invalid" => false)
else
begin
normalized_value = Float(value)
self.extra = (extra || {}).merge("exchange_rate" => normalized_value)
raise ArgumentError unless normalized_value.finite?
self.extra = (extra || {}).merge("exchange_rate" => normalized_value, "exchange_rate_invalid" => false)
rescue ArgumentError, TypeError
# Store the raw value for validation error reporting
self.extra = (extra || {}).merge("exchange_rate" => value, "exchange_rate_invalid" => true)
@@ -55,9 +57,8 @@ class Transaction < ApplicationRecord
if extra&.dig("exchange_rate_invalid")
errors.add(:exchange_rate, "must be a number")
elsif exchange_rate.present?
# Convert to float for comparison
numeric_rate = exchange_rate.to_d rescue nil
if numeric_rate.nil? || numeric_rate <= 0
numeric_rate = Float(exchange_rate) rescue nil
if numeric_rate.nil? || !numeric_rate.finite? || numeric_rate <= 0
errors.add(:exchange_rate, "must be greater than 0")
end
end
@@ -151,6 +152,24 @@ class Transaction < ApplicationRecord
false
end
def activity_security_id
extra&.dig("security_id").presence || extra&.dig("security", "id").presence
end
def activity_security
security_id = activity_security_id.to_s
return @activity_security = nil if security_id.blank?
return @activity_security if defined?(@activity_security_id) && @activity_security_id == security_id
@activity_security_id = security_id
@activity_security = Security.find_by(id: security_id)
end
def set_preloaded_activity_security(security)
@activity_security_id = security&.id&.to_s
@activity_security = security
end
# Potential duplicate matching methods
# These help users review and resolve fuzzy-matched pending/posted pairs

View File

@@ -0,0 +1,36 @@
class Transaction::ActivitySecurityPreloader
def initialize(records)
@records = Array(records)
end
def preload
transactions.each do |transaction|
transaction.set_preloaded_activity_security(securities_by_id[transaction.activity_security_id.to_s])
end
records
end
private
attr_reader :records
def transactions
@transactions ||= records.filter_map do |record|
case record
when Transaction
record
when Entry
record.transaction? ? record.entryable : nil
end
end
end
def securities_by_id
@securities_by_id ||= begin
security_ids = transactions.filter_map(&:activity_security_id).uniq
return {} if security_ids.empty?
Security.where(id: security_ids).index_by { |security| security.id.to_s }
end
end
end