diff --git a/app/models/family/data_exporter.rb b/app/models/family/data_exporter.rb index 7ae340858..8c1992ba1 100644 --- a/app/models/family/data_exporter.rb +++ b/app/models/family/data_exporter.rb @@ -220,6 +220,8 @@ class Family::DataExporter account_id: trade.entry.account_id, security_id: trade.security_id, ticker: trade.security.ticker, + security_name: trade.security.name, + exchange_operating_mic: trade.security.exchange_operating_mic, date: trade.entry.date, qty: trade.qty, price: trade.price, @@ -231,6 +233,35 @@ class Family::DataExporter }.to_json end + # Export holding snapshots for backup and portfolio verification. + @family.holdings.includes(:account, :security).find_each do |holding| + lines << { + type: "Holding", + data: { + id: holding.id, + account_id: holding.account_id, + security_id: holding.security_id, + ticker: holding.security.ticker, + security_name: holding.security.name, + exchange_operating_mic: holding.security.exchange_operating_mic, + exchange_mic: holding.security.exchange_mic, + exchange_acronym: holding.security.exchange_acronym, + country_code: holding.security.country_code, + kind: holding.security.kind, + website_url: holding.security.website_url, + date: holding.date, + qty: holding.qty, + price: holding.price, + amount: holding.amount, + currency: holding.currency, + cost_basis: holding.cost_basis, + cost_basis_source: holding.cost_basis_source, + cost_basis_locked: holding.cost_basis_locked, + security_locked: holding.security_locked + } + }.to_json + end + # Export valuations @family.entries.valuations.includes(:account, :entryable).find_each do |entry| lines << { diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb index 88efbcd37..f1b119576 100644 --- a/app/models/family/data_importer.rb +++ b/app/models/family/data_importer.rb @@ -1,7 +1,7 @@ require "set" class Family::DataImporter - SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Trade Valuation Budget BudgetCategory Rule].freeze + SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Trade Holding Valuation Budget BudgetCategory Rule].freeze ACCOUNTABLE_TYPES = Accountable::TYPES.freeze def initialize(family, ndjson_content) @@ -16,6 +16,7 @@ class Family::DataImporter budgets: {}, securities: {} } + @security_cache = {} @created_accounts = [] @created_entries = [] end @@ -34,6 +35,7 @@ class Family::DataImporter import_recurring_transactions(records["RecurringTransaction"] || []) import_transactions(records["Transaction"] || []) import_trades(records["Trade"] || []) + import_holdings(records["Holding"] || []) import_valuations(records["Valuation"] || []) import_budgets(records["Budget"] || []) import_budget_categories(records["BudgetCategory"] || []) @@ -321,7 +323,13 @@ class Family::DataImporter ticker = data["ticker"] next unless ticker.present? - security = find_or_create_security(ticker, data["currency"]) + security = find_or_create_security( + ticker, + data["currency"], + old_security_id: data["security_id"], + name: data["security_name"], + exchange_operating_mic: data["exchange_operating_mic"] + ) trade = Trade.new( security: security, @@ -344,6 +352,51 @@ class Family::DataImporter end end + def import_holdings(records) + accounts_by_id = @family.accounts.where(id: records.filter_map { |record| @id_mappings[:accounts][record.dig("data", "account_id")] }).index_by(&:id) + + records.each do |record| + data = record["data"] + + new_account_id = @id_mappings[:accounts][data["account_id"]] + next unless new_account_id + + account = accounts_by_id[new_account_id] + next unless account + + ticker = data["ticker"] + next unless ticker.present? + + security = find_or_create_security( + ticker, + data["currency"], + old_security_id: data["security_id"], + name: data["security_name"], + exchange_operating_mic: data["exchange_operating_mic"], + exchange_mic: data["exchange_mic"], + exchange_acronym: data["exchange_acronym"], + country_code: data["country_code"], + kind: data["kind"], + website_url: data["website_url"] + ) + + holding_date = Date.parse(data["date"].to_s) + holding_currency = data["currency"] || account.currency + holding_attributes = { + qty: data["qty"].to_d, + price: data["price"].to_d, + amount: data["amount"].to_d, + currency: holding_currency, + cost_basis: data["cost_basis"]&.to_d, + cost_basis_source: importable_cost_basis_source(data["cost_basis_source"]), + cost_basis_locked: truthy?(data["cost_basis_locked"]) || false, + security_locked: truthy?(data["security_locked"]) || false + } + + upsert_imported_holding!(account, security, holding_date, holding_currency, holding_attributes) + end + end + def import_valuations(records) records.each do |record| data = record["data"] @@ -375,7 +428,7 @@ class Family::DataImporter # Account-level opening balances must precede every imported account # activity, including standalone valuation snapshots. - %w[Transaction Trade Valuation].each do |type| + %w[Transaction Trade Holding Valuation].each do |type| records[type].to_a.each do |record| data = record["data"] || {} account_id = data["account_id"] @@ -591,18 +644,95 @@ class Family::DataImporter value end - def find_or_create_security(ticker, currency) + def importable_cost_basis_source(value) + source = value.to_s + Holding::COST_BASIS_SOURCES.include?(source) ? source : nil + end + + def truthy?(value) + ActiveModel::Type::Boolean.new.cast(value) + end + + def find_or_create_security(ticker, currency, old_security_id: nil, **attributes) # Check cache first - cache_key = "#{ticker}:#{currency}" - return @id_mappings[:securities][cache_key] if @id_mappings[:securities][cache_key] + normalized_ticker = ticker.to_s.upcase + exchange_operating_mic = attributes[:exchange_operating_mic].presence&.upcase + cache_key = "#{normalized_ticker}:#{exchange_operating_mic}:#{currency}" - security = Security.find_by(ticker: ticker.upcase) - security ||= Security.create!( - ticker: ticker.upcase, - name: ticker.upcase - ) + if @security_cache[cache_key] + security = @security_cache[cache_key] + apply_security_metadata(security, normalized_ticker, attributes) + return security + end - @id_mappings[:securities][cache_key] = security + if old_security_id.present? && @id_mappings[:securities][old_security_id] + security = Security.find(@id_mappings[:securities][old_security_id]) + apply_security_metadata(security, normalized_ticker, attributes) + @security_cache[cache_key] = security + return security + end + + security = find_security_by_identity(normalized_ticker, exchange_operating_mic) + apply_security_metadata(security, normalized_ticker, attributes) + + @security_cache[cache_key] = security + @id_mappings[:securities][old_security_id] = security.id if old_security_id.present? security end + + def find_security_by_identity(ticker, exchange_operating_mic) + if exchange_operating_mic.present? + return Security.find_or_initialize_by(ticker: ticker, exchange_operating_mic: exchange_operating_mic) + end + + # Without an exchange MIC, matching by ticker is a best-effort restore path and can merge same-ticker securities from different venues. + Security.find_by(ticker: ticker, exchange_operating_mic: nil) || + Security.where(ticker: ticker).order(:created_at).first || + Security.new(ticker: ticker) + end + + def apply_security_metadata(security, ticker, attributes) + assign_if_blank_or_placeholder(security, :name, attributes[:name].presence, placeholder: ticker) + assign_if_blank(security, :exchange_operating_mic, attributes[:exchange_operating_mic].presence&.upcase) + assign_if_blank(security, :exchange_mic, attributes[:exchange_mic].presence) + assign_if_blank(security, :exchange_acronym, attributes[:exchange_acronym].presence) + assign_if_blank(security, :country_code, attributes[:country_code].presence) + assign_if_blank(security, :website_url, attributes[:website_url].presence) + security.kind = security_kind_for(attributes[:kind]) if security.new_record? || security.kind.blank? + + security.save! if security.new_record? || security.changed? + end + + def assign_if_blank(record, attribute, value) + return if value.blank? + return if record.public_send(attribute).present? + + record.public_send("#{attribute}=", value) + end + + def assign_if_blank_or_placeholder(record, attribute, value, placeholder:) + return if value.blank? + + current_value = record.public_send(attribute) + return if current_value.present? && current_value != placeholder + + record.public_send("#{attribute}=", value) + end + + def upsert_imported_holding!(account, security, date, currency, attributes) + holding = account.holdings.find_or_initialize_by(security: security, date: date, currency: currency) + holding.assign_attributes(attributes) + + begin + Holding.transaction(requires_new: true) { holding.save! } + rescue ActiveRecord::RecordNotUnique + existing = account.holdings.find_by!(security: security, date: date, currency: currency) + existing.update!(attributes) + end + end + + def security_kind_for(value) + kind = value.to_s + Security::KINDS.include?(kind) ? kind : Security::KINDS.first + end end diff --git a/test/models/family/data_exporter_test.rb b/test/models/family/data_exporter_test.rb index bd5d36d21..d25575ae4 100644 --- a/test/models/family/data_exporter_test.rb +++ b/test/models/family/data_exporter_test.rb @@ -376,6 +376,55 @@ class Family::DataExporterTest < ActiveSupport::TestCase end end + test "exports holding snapshots in NDJSON" do + investment_account = @family.accounts.create!( + name: "Investment Account", + accountable: Investment.new, + balance: 25_000, + currency: "USD" + ) + security = Security.create!( + ticker: "VTI#{SecureRandom.hex(4).upcase}", + name: "Vanguard Total Stock Market ETF", + country_code: "US", + exchange_operating_mic: "ARCX" + ) + holding = investment_account.holdings.create!( + security: security, + date: Date.parse("2024-01-15"), + qty: 100, + price: 250.25, + amount: 25_025, + currency: "USD", + cost_basis: 200, + cost_basis_source: "manual", + cost_basis_locked: true, + security_locked: true + ) + + zip_data = @exporter.generate_export + + Zip::File.open_buffer(zip_data) do |zip| + ndjson_records = zip.read("all.ndjson").split("\n").map { |line| JSON.parse(line) } + holding_data = ndjson_records.find { |record| record["type"] == "Holding" && record.dig("data", "id") == holding.id } + + assert holding_data + assert_equal investment_account.id, holding_data["data"]["account_id"] + assert_equal security.id, holding_data["data"]["security_id"] + assert_equal security.ticker, holding_data["data"]["ticker"] + assert_equal "ARCX", holding_data["data"]["exchange_operating_mic"] + assert_equal "2024-01-15", holding_data["data"]["date"] + assert_equal "100.0", BigDecimal(holding_data["data"]["qty"].to_s).to_s("F") + assert_equal "250.25", BigDecimal(holding_data["data"]["price"].to_s).to_s("F") + assert_equal "25025.0", BigDecimal(holding_data["data"]["amount"].to_s).to_s("F") + assert_equal "200.0", BigDecimal(holding_data["data"]["cost_basis"].to_s).to_s("F") + assert_equal "manual", holding_data["data"]["cost_basis_source"] + assert_equal true, holding_data["data"]["cost_basis_locked"] + assert_not holding_data["data"].key?("created_at") + assert_not holding_data["data"].key?("updated_at") + end + end + test "only exports rules from the specified family" do # Create a rule for another family that should NOT be exported other_rule = @other_family.rules.build( diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb index fab5ed242..1284a6630 100644 --- a/test/models/family/data_importer_test.rb +++ b/test/models/family/data_importer_test.rb @@ -523,6 +523,302 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal 150.0, trade.price.to_f end + test "imports holding snapshots with security identity" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "inv-acct-1", + name: "Investment Account", + balance: "10000", + currency: "USD", + accountable_type: "Investment" + } + }, + { + type: "Holding", + data: { + id: "holding-1", + account_id: "inv-acct-1", + security_id: "security-1", + ticker: "VTI", + security_name: "Vanguard Total Stock Market ETF", + exchange_operating_mic: "ARCX", + country_code: "US", + date: "2024-01-15", + qty: "100", + price: "250.25", + amount: "25025.00", + currency: "USD", + cost_basis: "200.00", + cost_basis_source: "manual", + cost_basis_locked: true, + security_locked: true + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Investment Account") + holding = account.holdings.first + + assert_not_nil holding + assert_equal Date.parse("2024-01-15"), holding.date + assert_equal "VTI", holding.security.ticker + assert_equal "Vanguard Total Stock Market ETF", holding.security.name + assert_equal "ARCX", holding.security.exchange_operating_mic + assert_equal 100.0, holding.qty.to_f + assert_equal 250.25, holding.price.to_f + assert_equal 25_025.0, holding.amount.to_f + assert_equal 200.0, holding.cost_basis.to_f + assert_equal "manual", holding.cost_basis_source + assert holding.cost_basis_locked + assert holding.security_locked + + opening_anchor = account.valuations.opening_anchor.first + assert_equal Date.parse("2024-01-14"), opening_anchor.entry.date + end + + test "imports duplicate holding snapshots idempotently by account security date and currency" do + holding_record = { + type: "Holding", + data: { + id: "holding-1", + account_id: "inv-acct-1", + security_id: "security-1", + ticker: "VTI", + security_name: "Vanguard Total Stock Market ETF", + exchange_operating_mic: "ARCX", + kind: "unsupported", + date: "2024-01-15", + qty: "100", + price: "250.25", + amount: "25025.00", + currency: "USD" + } + } + + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "inv-acct-1", + name: "Investment Account", + balance: "10000", + currency: "USD", + accountable_type: "Investment" + } + }, + holding_record, + holding_record.deep_merge(data: { id: "holding-1-duplicate", qty: "101", amount: "25275.25" }) + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Investment Account") + assert_equal 1, account.holdings.count + + holding = account.holdings.first + assert_equal 101.0, holding.qty.to_f + assert_equal 25_275.25, holding.amount.to_f + assert_equal "standard", holding.security.kind + end + + test "imports same holding date in different currencies separately" do + holding_record = { + type: "Holding", + data: { + id: "holding-1", + account_id: "inv-acct-1", + security_id: "security-1", + ticker: "VTI", + security_name: "Vanguard Total Stock Market ETF", + exchange_operating_mic: "ARCX", + date: "2024-01-15", + qty: "100", + price: "250.25", + amount: "25025.00", + currency: "USD" + } + } + + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "inv-acct-1", + name: "Investment Account", + balance: "10000", + currency: "USD", + accountable_type: "Investment" + } + }, + holding_record, + holding_record.deep_merge(data: { id: "holding-2", currency: "CAD", amount: "34034.00" }) + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Investment Account") + assert_equal 2, account.holdings.count + assert_equal %w[CAD USD], account.holdings.order(:currency).pluck(:currency) + end + + test "round trips holding snapshots through full export" do + source_family = Family.create!( + name: "Source Family", + currency: "USD", + locale: "en", + date_format: "%Y-%m-%d" + ) + source_account = source_family.accounts.create!( + name: "Round Trip Investment", + accountable: Investment.new, + balance: 25_000, + currency: "USD" + ) + source_security = Security.create!( + ticker: "VTI#{SecureRandom.hex(4).upcase}", + name: "Vanguard Total Stock Market ETF", + country_code: "US", + exchange_operating_mic: "ARCX" + ) + source_account.holdings.create!( + security: source_security, + date: Date.parse("2024-01-15"), + qty: 100, + price: 250.25, + amount: 25_025, + currency: "USD", + cost_basis: 200, + cost_basis_source: "manual", + cost_basis_locked: true, + security_locked: true + ) + + zip_data = Family::DataExporter.new(source_family).generate_export + ndjson = nil + Zip::File.open_buffer(zip_data) do |zip| + ndjson = zip.read("all.ndjson") + end + + Family::DataImporter.new(@family, ndjson).import! + + imported_account = @family.accounts.find_by!(name: "Round Trip Investment") + imported_holding = imported_account.holdings.find_by!(date: Date.parse("2024-01-15")) + + assert_equal source_security.ticker, imported_holding.security.ticker + assert_equal "ARCX", imported_holding.security.exchange_operating_mic + assert_equal 100.0, imported_holding.qty.to_f + assert_equal 250.25, imported_holding.price.to_f + assert_equal 25_025.0, imported_holding.amount.to_f + assert_equal 200.0, imported_holding.cost_basis.to_f + assert_equal "manual", imported_holding.cost_basis_source + assert imported_holding.cost_basis_locked + assert imported_holding.security_locked + end + + test "imports holding snapshots with ticker fallback when exchange mic is missing" do + existing_security = Security.create!( + ticker: "VTI", + name: "Existing VTI", + exchange_operating_mic: "ARCX" + ) + + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "inv-acct-1", + name: "Investment Account", + balance: "10000", + currency: "USD", + accountable_type: "Investment" + } + }, + { + type: "Holding", + data: { + id: "holding-1", + account_id: "inv-acct-1", + ticker: "VTI", + security_name: "Imported VTI", + date: "2024-01-15", + qty: "100", + price: "250.25", + amount: "25025.00", + currency: "USD", + cost_basis_locked: false, + security_locked: false + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + holding = @family.accounts.find_by!(name: "Investment Account").holdings.first + assert_equal existing_security, holding.security + assert_equal 1, Security.where(ticker: "VTI").count + end + + test "updates cached security with safe holding metadata" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "inv-acct-1", + name: "Investment Account", + balance: "10000", + currency: "USD", + accountable_type: "Investment" + } + }, + { + type: "Trade", + data: { + id: "trade-1", + account_id: "inv-acct-1", + security_id: "security-1", + ticker: "VTI", + date: "2024-01-10", + qty: "10", + price: "250.00", + amount: "-2500.00", + currency: "USD" + } + }, + { + type: "Holding", + data: { + id: "holding-1", + account_id: "inv-acct-1", + security_id: "security-1", + ticker: "VTI", + security_name: "Vanguard Total Stock Market ETF", + exchange_operating_mic: "ARCX", + country_code: "US", + website_url: "https://investor.vanguard.com", + date: "2024-01-15", + qty: "100", + price: "250.25", + amount: "25025.00", + currency: "USD", + cost_basis_locked: false, + security_locked: false + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + security = @family.holdings.first.security + assert_equal "Vanguard Total Stock Market ETF", security.name + assert_equal "ARCX", security.exchange_operating_mic + assert_equal "US", security.country_code + assert_equal "https://investor.vanguard.com", security.website_url + end + test "imports valuations" do ndjson = build_ndjson([ {