QT/sure
1
0
Fork 0
mirror of https://github.com/we-promise/sure.git synced 2026-05-07 12:54:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(exports): include holding snapshots (#1643)

Browse Source 
* feat(exports): include holding snapshots

* fix(exports): resolve holding securities without mic

* fix(exports): harden holding snapshot imports

* fix(exports): harden holding snapshot upserts

* fix(exports): keep holding upserts database-driven
This commit is contained in:
ghost
2026-05-04 16:44:29 -06:00
committed by GitHub
parent b30f3f5bce
commit a108e6501e
4 changed files with 518 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
app/models/family/data_exporter.rb
View File

@@ -220,6 +220,8 @@ class Family::DataExporter
account_id: trade.entry.account_id,
security_id: trade.security_id,
ticker: trade.security.ticker,
security_name: trade.security.name,
exchange_operating_mic: trade.security.exchange_operating_mic,
date: trade.entry.date,
qty: trade.qty,
price: trade.price,
@@ -231,6 +233,35 @@ class Family::DataExporter
}.to_json
end
# Export holding snapshots for backup and portfolio verification.
@family.holdings.includes(:account, :security).find_each do |holding|
lines << {
type: "Holding",
data: {
id: holding.id,
account_id: holding.account_id,
security_id: holding.security_id,
ticker: holding.security.ticker,
security_name: holding.security.name,
exchange_operating_mic: holding.security.exchange_operating_mic,
exchange_mic: holding.security.exchange_mic,
exchange_acronym: holding.security.exchange_acronym,
country_code: holding.security.country_code,
kind: holding.security.kind,
website_url: holding.security.website_url,
date: holding.date,
qty: holding.qty,
price: holding.price,
amount: holding.amount,
currency: holding.currency,
cost_basis: holding.cost_basis,
cost_basis_source: holding.cost_basis_source,
cost_basis_locked: holding.cost_basis_locked,
security_locked: holding.security_locked
}
}.to_json
end
# Export valuations
@family.entries.valuations.includes(:account, :entryable).find_each do |entry|
lines << {

154
app/models/family/data_importer.rb
View File

@@ -1,7 +1,7 @@
require "set"
class Family::DataImporter
SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Trade Valuation Budget BudgetCategory Rule].freeze
SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Trade Holding Valuation Budget BudgetCategory Rule].freeze
ACCOUNTABLE_TYPES = Accountable::TYPES.freeze
def initialize(family, ndjson_content)
@@ -16,6 +16,7 @@ class Family::DataImporter
budgets: {},
securities: {}
}
@security_cache = {}
@created_accounts = []
@created_entries = []
end
@@ -34,6 +35,7 @@ class Family::DataImporter
import_recurring_transactions(records["RecurringTransaction"] || [])
import_transactions(records["Transaction"] || [])
import_trades(records["Trade"] || [])
import_holdings(records["Holding"] || [])
import_valuations(records["Valuation"] || [])
import_budgets(records["Budget"] || [])
import_budget_categories(records["BudgetCategory"] || [])
@@ -321,7 +323,13 @@ class Family::DataImporter
ticker = data["ticker"]
next unless ticker.present?
security = find_or_create_security(ticker, data["currency"])
security = find_or_create_security(
ticker,
data["currency"],
old_security_id: data["security_id"],
name: data["security_name"],
exchange_operating_mic: data["exchange_operating_mic"]
)
trade = Trade.new(
security: security,
@@ -344,6 +352,51 @@ class Family::DataImporter
end
end
def import_holdings(records)
accounts_by_id = @family.accounts.where(id: records.filter_map { |record| @id_mappings[:accounts][record.dig("data", "account_id")] }).index_by(&:id)
records.each do |record|
data = record["data"]
new_account_id = @id_mappings[:accounts][data["account_id"]]
next unless new_account_id
account = accounts_by_id[new_account_id]
next unless account
ticker = data["ticker"]
next unless ticker.present?
security = find_or_create_security(
ticker,
data["currency"],
old_security_id: data["security_id"],
name: data["security_name"],
exchange_operating_mic: data["exchange_operating_mic"],
exchange_mic: data["exchange_mic"],
exchange_acronym: data["exchange_acronym"],
country_code: data["country_code"],
kind: data["kind"],
website_url: data["website_url"]
)
holding_date = Date.parse(data["date"].to_s)
holding_currency = data["currency"] || account.currency
holding_attributes = {
qty: data["qty"].to_d,
price: data["price"].to_d,
amount: data["amount"].to_d,
currency: holding_currency,
cost_basis: data["cost_basis"]&.to_d,
cost_basis_source: importable_cost_basis_source(data["cost_basis_source"]),
cost_basis_locked: truthy?(data["cost_basis_locked"]) || false,
security_locked: truthy?(data["security_locked"]) || false
}
upsert_imported_holding!(account, security, holding_date, holding_currency, holding_attributes)
end
end
def import_valuations(records)
records.each do |record|
data = record["data"]
@@ -375,7 +428,7 @@ class Family::DataImporter
# Account-level opening balances must precede every imported account
# activity, including standalone valuation snapshots.
%w[Transaction Trade Valuation].each do |type|
%w[Transaction Trade Holding Valuation].each do |type|
records[type].to_a.each do |record|
data = record["data"] || {}
account_id = data["account_id"]
@@ -591,18 +644,95 @@ class Family::DataImporter
value
end
def find_or_create_security(ticker, currency)
def importable_cost_basis_source(value)
source = value.to_s
Holding::COST_BASIS_SOURCES.include?(source) ? source : nil
end
def truthy?(value)
ActiveModel::Type::Boolean.new.cast(value)
end
def find_or_create_security(ticker, currency, old_security_id: nil, **attributes)
# Check cache first
cache_key = "#{ticker}:#{currency}"
return @id_mappings[:securities][cache_key] if @id_mappings[:securities][cache_key]
normalized_ticker = ticker.to_s.upcase
exchange_operating_mic = attributes[:exchange_operating_mic].presence&.upcase
cache_key = "#{normalized_ticker}:#{exchange_operating_mic}:#{currency}"
security = Security.find_by(ticker: ticker.upcase)
security ||= Security.create!(
ticker: ticker.upcase,
name: ticker.upcase
)
if @security_cache[cache_key]
security = @security_cache[cache_key]
apply_security_metadata(security, normalized_ticker, attributes)
return security
end
@id_mappings[:securities][cache_key] = security
if old_security_id.present? && @id_mappings[:securities][old_security_id]
security = Security.find(@id_mappings[:securities][old_security_id])
apply_security_metadata(security, normalized_ticker, attributes)
@security_cache[cache_key] = security
return security
end
security = find_security_by_identity(normalized_ticker, exchange_operating_mic)
apply_security_metadata(security, normalized_ticker, attributes)
@security_cache[cache_key] = security
@id_mappings[:securities][old_security_id] = security.id if old_security_id.present?
security
end
def find_security_by_identity(ticker, exchange_operating_mic)
if exchange_operating_mic.present?
return Security.find_or_initialize_by(ticker: ticker, exchange_operating_mic: exchange_operating_mic)
end
# Without an exchange MIC, matching by ticker is a best-effort restore path and can merge same-ticker securities from different venues.
Security.find_by(ticker: ticker, exchange_operating_mic: nil) ||
Security.where(ticker: ticker).order(:created_at).first ||
Security.new(ticker: ticker)
end
def apply_security_metadata(security, ticker, attributes)
assign_if_blank_or_placeholder(security, :name, attributes[:name].presence, placeholder: ticker)
assign_if_blank(security, :exchange_operating_mic, attributes[:exchange_operating_mic].presence&.upcase)
assign_if_blank(security, :exchange_mic, attributes[:exchange_mic].presence)
assign_if_blank(security, :exchange_acronym, attributes[:exchange_acronym].presence)
assign_if_blank(security, :country_code, attributes[:country_code].presence)
assign_if_blank(security, :website_url, attributes[:website_url].presence)
security.kind = security_kind_for(attributes[:kind]) if security.new_record? || security.kind.blank?
security.save! if security.new_record? || security.changed?
end
def assign_if_blank(record, attribute, value)
return if value.blank?
return if record.public_send(attribute).present?
record.public_send("#{attribute}=", value)
end
def assign_if_blank_or_placeholder(record, attribute, value, placeholder:)
return if value.blank?
current_value = record.public_send(attribute)
return if current_value.present? && current_value != placeholder
record.public_send("#{attribute}=", value)
end
def upsert_imported_holding!(account, security, date, currency, attributes)
holding = account.holdings.find_or_initialize_by(security: security, date: date, currency: currency)
holding.assign_attributes(attributes)
begin
Holding.transaction(requires_new: true) { holding.save! }
rescue ActiveRecord::RecordNotUnique
existing = account.holdings.find_by!(security: security, date: date, currency: currency)
existing.update!(attributes)
end
end
def security_kind_for(value)
kind = value.to_s
Security::KINDS.include?(kind) ? kind : Security::KINDS.first
end
end

49
test/models/family/data_exporter_test.rb
View File

@@ -376,6 +376,55 @@ class Family::DataExporterTest < ActiveSupport::TestCase
end
end
test "exports holding snapshots in NDJSON" do
investment_account = @family.accounts.create!(
name: "Investment Account",
accountable: Investment.new,
balance: 25_000,
currency: "USD"
)
security = Security.create!(
ticker: "VTI#{SecureRandom.hex(4).upcase}",
name: "Vanguard Total Stock Market ETF",
country_code: "US",
exchange_operating_mic: "ARCX"
)
holding = investment_account.holdings.create!(
security: security,
date: Date.parse("2024-01-15"),
qty: 100,
price: 250.25,
amount: 25_025,
currency: "USD",
cost_basis: 200,
cost_basis_source: "manual",
cost_basis_locked: true,
security_locked: true
)
zip_data = @exporter.generate_export
Zip::File.open_buffer(zip_data) do |zip|
ndjson_records = zip.read("all.ndjson").split("\n").map { |line| JSON.parse(line) }
holding_data = ndjson_records.find { |record| record["type"] == "Holding" && record.dig("data", "id") == holding.id }
assert holding_data
assert_equal investment_account.id, holding_data["data"]["account_id"]
assert_equal security.id, holding_data["data"]["security_id"]
assert_equal security.ticker, holding_data["data"]["ticker"]
assert_equal "ARCX", holding_data["data"]["exchange_operating_mic"]
assert_equal "2024-01-15", holding_data["data"]["date"]
assert_equal "100.0", BigDecimal(holding_data["data"]["qty"].to_s).to_s("F")
assert_equal "250.25", BigDecimal(holding_data["data"]["price"].to_s).to_s("F")
assert_equal "25025.0", BigDecimal(holding_data["data"]["amount"].to_s).to_s("F")
assert_equal "200.0", BigDecimal(holding_data["data"]["cost_basis"].to_s).to_s("F")
assert_equal "manual", holding_data["data"]["cost_basis_source"]
assert_equal true, holding_data["data"]["cost_basis_locked"]
assert_not holding_data["data"].key?("created_at")
assert_not holding_data["data"].key?("updated_at")
end
end
test "only exports rules from the specified family" do
# Create a rule for another family that should NOT be exported
other_rule = @other_family.rules.build(

296
test/models/family/data_importer_test.rb
View File

@@ -523,6 +523,302 @@ class Family::DataImporterTest < ActiveSupport::TestCase
assert_equal 150.0, trade.price.to_f
end
test "imports holding snapshots with security identity" do
ndjson = build_ndjson([
{
type: "Account",
data: {
id: "inv-acct-1",
name: "Investment Account",
balance: "10000",
currency: "USD",
accountable_type: "Investment"
}
},
{
type: "Holding",
data: {
id: "holding-1",
account_id: "inv-acct-1",
security_id: "security-1",
ticker: "VTI",
security_name: "Vanguard Total Stock Market ETF",
exchange_operating_mic: "ARCX",
country_code: "US",
date: "2024-01-15",
qty: "100",
price: "250.25",
amount: "25025.00",
currency: "USD",
cost_basis: "200.00",
cost_basis_source: "manual",
cost_basis_locked: true,
security_locked: true
}
}
])
Family::DataImporter.new(@family, ndjson).import!
account = @family.accounts.find_by!(name: "Investment Account")
holding = account.holdings.first
assert_not_nil holding
assert_equal Date.parse("2024-01-15"), holding.date
assert_equal "VTI", holding.security.ticker
assert_equal "Vanguard Total Stock Market ETF", holding.security.name
assert_equal "ARCX", holding.security.exchange_operating_mic
assert_equal 100.0, holding.qty.to_f
assert_equal 250.25, holding.price.to_f
assert_equal 25_025.0, holding.amount.to_f
assert_equal 200.0, holding.cost_basis.to_f
assert_equal "manual", holding.cost_basis_source
assert holding.cost_basis_locked
assert holding.security_locked
opening_anchor = account.valuations.opening_anchor.first
assert_equal Date.parse("2024-01-14"), opening_anchor.entry.date
end
test "imports duplicate holding snapshots idempotently by account security date and currency" do
holding_record = {
type: "Holding",
data: {
id: "holding-1",
account_id: "inv-acct-1",
security_id: "security-1",
ticker: "VTI",
security_name: "Vanguard Total Stock Market ETF",
exchange_operating_mic: "ARCX",
kind: "unsupported",
date: "2024-01-15",
qty: "100",
price: "250.25",
amount: "25025.00",
currency: "USD"
}
}
ndjson = build_ndjson([
{
type: "Account",
data: {
id: "inv-acct-1",
name: "Investment Account",
balance: "10000",
currency: "USD",
accountable_type: "Investment"
}
},
holding_record,
holding_record.deep_merge(data: { id: "holding-1-duplicate", qty: "101", amount: "25275.25" })
])
Family::DataImporter.new(@family, ndjson).import!
account = @family.accounts.find_by!(name: "Investment Account")
assert_equal 1, account.holdings.count
holding = account.holdings.first
assert_equal 101.0, holding.qty.to_f
assert_equal 25_275.25, holding.amount.to_f
assert_equal "standard", holding.security.kind
end
test "imports same holding date in different currencies separately" do
holding_record = {
type: "Holding",
data: {
id: "holding-1",
account_id: "inv-acct-1",
security_id: "security-1",
ticker: "VTI",
security_name: "Vanguard Total Stock Market ETF",
exchange_operating_mic: "ARCX",
date: "2024-01-15",
qty: "100",
price: "250.25",
amount: "25025.00",
currency: "USD"
}
}
ndjson = build_ndjson([
{
type: "Account",
data: {
id: "inv-acct-1",
name: "Investment Account",
balance: "10000",
currency: "USD",
accountable_type: "Investment"
}
},
holding_record,
holding_record.deep_merge(data: { id: "holding-2", currency: "CAD", amount: "34034.00" })
])
Family::DataImporter.new(@family, ndjson).import!
account = @family.accounts.find_by!(name: "Investment Account")
assert_equal 2, account.holdings.count
assert_equal %w[CAD USD], account.holdings.order(:currency).pluck(:currency)
end
test "round trips holding snapshots through full export" do
source_family = Family.create!(
name: "Source Family",
currency: "USD",
locale: "en",
date_format: "%Y-%m-%d"
)
source_account = source_family.accounts.create!(
name: "Round Trip Investment",
accountable: Investment.new,
balance: 25_000,
currency: "USD"
)
source_security = Security.create!(
ticker: "VTI#{SecureRandom.hex(4).upcase}",
name: "Vanguard Total Stock Market ETF",
country_code: "US",
exchange_operating_mic: "ARCX"
)
source_account.holdings.create!(
security: source_security,
date: Date.parse("2024-01-15"),
qty: 100,
price: 250.25,
amount: 25_025,
currency: "USD",
cost_basis: 200,
cost_basis_source: "manual",
cost_basis_locked: true,
security_locked: true
)
zip_data = Family::DataExporter.new(source_family).generate_export
ndjson = nil
Zip::File.open_buffer(zip_data) do |zip|
ndjson = zip.read("all.ndjson")
end
Family::DataImporter.new(@family, ndjson).import!
imported_account = @family.accounts.find_by!(name: "Round Trip Investment")
imported_holding = imported_account.holdings.find_by!(date: Date.parse("2024-01-15"))
assert_equal source_security.ticker, imported_holding.security.ticker
assert_equal "ARCX", imported_holding.security.exchange_operating_mic
assert_equal 100.0, imported_holding.qty.to_f
assert_equal 250.25, imported_holding.price.to_f
assert_equal 25_025.0, imported_holding.amount.to_f
assert_equal 200.0, imported_holding.cost_basis.to_f
assert_equal "manual", imported_holding.cost_basis_source
assert imported_holding.cost_basis_locked
assert imported_holding.security_locked
end
test "imports holding snapshots with ticker fallback when exchange mic is missing" do
existing_security = Security.create!(
ticker: "VTI",
name: "Existing VTI",
exchange_operating_mic: "ARCX"
)
ndjson = build_ndjson([
{
type: "Account",
data: {
id: "inv-acct-1",
name: "Investment Account",
balance: "10000",
currency: "USD",
accountable_type: "Investment"
}
},
{
type: "Holding",
data: {
id: "holding-1",
account_id: "inv-acct-1",
ticker: "VTI",
security_name: "Imported VTI",
date: "2024-01-15",
qty: "100",
price: "250.25",
amount: "25025.00",
currency: "USD",
cost_basis_locked: false,
security_locked: false
}
}
])
Family::DataImporter.new(@family, ndjson).import!
holding = @family.accounts.find_by!(name: "Investment Account").holdings.first
assert_equal existing_security, holding.security
assert_equal 1, Security.where(ticker: "VTI").count
end
test "updates cached security with safe holding metadata" do
ndjson = build_ndjson([
{
type: "Account",
data: {
id: "inv-acct-1",
name: "Investment Account",
balance: "10000",
currency: "USD",
accountable_type: "Investment"
}
},
{
type: "Trade",
data: {
id: "trade-1",
account_id: "inv-acct-1",
security_id: "security-1",
ticker: "VTI",
date: "2024-01-10",
qty: "10",
price: "250.00",
amount: "-2500.00",
currency: "USD"
}
},
{
type: "Holding",
data: {
id: "holding-1",
account_id: "inv-acct-1",
security_id: "security-1",
ticker: "VTI",
security_name: "Vanguard Total Stock Market ETF",
exchange_operating_mic: "ARCX",
country_code: "US",
website_url: "https://investor.vanguard.com",
date: "2024-01-15",
qty: "100",
price: "250.25",
amount: "25025.00",
currency: "USD",
cost_basis_locked: false,
security_locked: false
}
}
])
Family::DataImporter.new(@family, ndjson).import!
security = @family.holdings.first.security
assert_equal "Vanguard Total Stock Market ETF", security.name
assert_equal "ARCX", security.exchange_operating_mic
assert_equal "US", security.country_code
assert_equal "https://investor.vanguard.com", security.website_url
end
test "imports valuations" do
ndjson = build_ndjson([
{