From 98df770547fe620c793a484e223884dd4d2dc3ea Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Sun, 3 May 2026 17:04:06 -0600 Subject: [PATCH] feat(exports): preserve recurring transactions (#1638) * feat(exports): preserve recurring transactions * fix(exports): harden recurring import records --- app/models/family/data_exporter.rb | 30 ++++ app/models/family/data_importer.rb | 66 +++++++- test/models/family/data_exporter_test.rb | 38 +++++ test/models/family/data_importer_test.rb | 192 +++++++++++++++++++++++ 4 files changed, 325 insertions(+), 1 deletion(-) diff --git a/app/models/family/data_exporter.rb b/app/models/family/data_exporter.rb index 98075bf0c..7ae340858 100644 --- a/app/models/family/data_exporter.rb +++ b/app/models/family/data_exporter.rb @@ -178,6 +178,14 @@ class Family::DataExporter }.to_json end + # Export recurring transactions after accounts and merchants so import can remap dependencies. + @family.recurring_transactions.includes(:account, :merchant).find_each do |recurring_transaction| + lines << { + type: "RecurringTransaction", + data: serialize_recurring_transaction_for_export(recurring_transaction) + }.to_json + end + # Export transactions with full data (exclude split parents, export children instead) @family.transactions.includes(:category, :merchant, :tags, entry: :account).merge(Entry.excluding_split_parents).find_each do |transaction| lines << { @@ -270,6 +278,28 @@ class Family::DataExporter lines.join("\n") end + def serialize_recurring_transaction_for_export(recurring_transaction) + { + id: recurring_transaction.id, + account_id: recurring_transaction.account_id, + merchant_id: recurring_transaction.merchant_id, + amount: recurring_transaction.amount, + currency: recurring_transaction.currency, + expected_day_of_month: recurring_transaction.expected_day_of_month, + last_occurrence_date: recurring_transaction.last_occurrence_date, + next_expected_date: recurring_transaction.next_expected_date, + status: recurring_transaction.status, + occurrence_count: recurring_transaction.occurrence_count, + name: recurring_transaction.name, + manual: recurring_transaction.manual, + expected_amount_min: recurring_transaction.expected_amount_min, + expected_amount_max: recurring_transaction.expected_amount_max, + expected_amount_avg: recurring_transaction.expected_amount_avg, + created_at: recurring_transaction.created_at, + updated_at: recurring_transaction.updated_at + } + end + def serialize_rule_for_export(rule) { name: rule.name, diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb index b20f08836..88efbcd37 100644 --- a/app/models/family/data_importer.rb +++ b/app/models/family/data_importer.rb @@ -1,7 +1,7 @@ require "set" class Family::DataImporter - SUPPORTED_TYPES = %w[Account Category Tag Merchant Transaction Trade Valuation Budget BudgetCategory Rule].freeze + SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Trade Valuation Budget BudgetCategory Rule].freeze ACCOUNTABLE_TYPES = Accountable::TYPES.freeze def initialize(family, ndjson_content) @@ -12,6 +12,7 @@ class Family::DataImporter categories: {}, tags: {}, merchants: {}, + recurring_transactions: {}, budgets: {}, securities: {} } @@ -30,6 +31,7 @@ class Family::DataImporter import_categories(records["Category"] || []) import_tags(records["Tag"] || []) import_merchants(records["Merchant"] || []) + import_recurring_transactions(records["RecurringTransaction"] || []) import_transactions(records["Transaction"] || []) import_trades(records["Trade"] || []) import_valuations(records["Valuation"] || []) @@ -187,6 +189,68 @@ class Family::DataImporter end end + def import_recurring_transactions(records) + records.each do |record| + data = record["data"] + old_id = data["id"] + + new_account_id = remap_optional_id(:accounts, data["account_id"]) + next if data["account_id"].present? && new_account_id.blank? + + new_merchant_id = remap_optional_id(:merchants, data["merchant_id"]) + next if data["merchant_id"].present? && new_merchant_id.blank? + + expected_day_of_month = recurring_expected_day_for(data["expected_day_of_month"]) + next unless expected_day_of_month + last_occurrence_date = parse_import_date(data["last_occurrence_date"]) + next_expected_date = parse_import_date(data["next_expected_date"]) + next unless last_occurrence_date && next_expected_date + + recurring_transaction = @family.recurring_transactions.build( + account_id: new_account_id, + merchant_id: new_merchant_id, + amount: data["amount"].to_d, + currency: data["currency"] || @family.currency, + expected_day_of_month: expected_day_of_month, + last_occurrence_date: last_occurrence_date, + next_expected_date: next_expected_date, + status: recurring_transaction_status_for(data["status"]), + occurrence_count: data["occurrence_count"].to_i, + name: data["name"], + manual: boolean_import_value(data, "manual", default: false), + expected_amount_min: data["expected_amount_min"]&.to_d, + expected_amount_max: data["expected_amount_max"]&.to_d, + expected_amount_avg: data["expected_amount_avg"]&.to_d + ) + + recurring_transaction.save! + @id_mappings[:recurring_transactions][old_id] = recurring_transaction.id + end + end + + def remap_optional_id(mapping_key, old_id) + return if old_id.blank? + + @id_mappings[mapping_key][old_id] + end + + def recurring_transaction_status_for(status) + status.to_s.in?(RecurringTransaction.statuses.keys) ? status.to_s : "active" + end + + def recurring_expected_day_for(value) + return if value.blank? + + expected_day = value.to_i + expected_day if expected_day.between?(1, 31) + end + + def boolean_import_value(data, key, default:) + return default unless data.key?(key) + + ActiveModel::Type::Boolean.new.cast(data[key]) + end + def import_transactions(records) records.each do |record| data = record["data"] diff --git a/test/models/family/data_exporter_test.rb b/test/models/family/data_exporter_test.rb index d4d13fdb2..bd5d36d21 100644 --- a/test/models/family/data_exporter_test.rb +++ b/test/models/family/data_exporter_test.rb @@ -338,6 +338,44 @@ class Family::DataExporterTest < ActiveSupport::TestCase end end + test "exports recurring transactions in NDJSON" do + merchant = @family.merchants.create!(name: "Internet Provider") + recurring_transaction = @family.recurring_transactions.create!( + account: @account, + merchant: merchant, + amount: -89.99, + currency: "USD", + expected_day_of_month: 14, + last_occurrence_date: Date.parse("2024-01-14"), + next_expected_date: Date.parse("2024-02-14"), + status: "active", + occurrence_count: 6, + manual: true, + expected_amount_min: -95, + expected_amount_max: -85, + expected_amount_avg: -89.99 + ) + + zip_data = @exporter.generate_export + + Zip::File.open_buffer(zip_data) do |zip| + ndjson_content = zip.read("all.ndjson") + recurring_data = ndjson_content + .split("\n") + .map { |line| JSON.parse(line) } + .find { |line| line["type"] == "RecurringTransaction" && line.dig("data", "id") == recurring_transaction.id } + + assert recurring_data + assert_equal recurring_transaction.id, recurring_data["data"]["id"] + assert_equal @account.id, recurring_data["data"]["account_id"] + assert_equal merchant.id, recurring_data["data"]["merchant_id"] + assert_equal "-89.99", BigDecimal(recurring_data["data"]["amount"].to_s).to_s("F") + assert_equal "active", recurring_data["data"]["status"] + assert_equal true, recurring_data["data"]["manual"] + assert_not recurring_data["data"].key?("family_id") + end + end + test "only exports rules from the specified family" do # Create a rule for another family that should NOT be exported other_rule = @other_family.rules.build( diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb index 1989fad32..fab5ed242 100644 --- a/test/models/family/data_importer_test.rb +++ b/test/models/family/data_importer_test.rb @@ -254,6 +254,176 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_not_nil merchant end + test "imports recurring transactions with remapped account and merchant references" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Main Checking", + balance: "5000", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Merchant", + data: { + id: "merchant-1", + name: "Internet Provider" + } + }, + { + type: "RecurringTransaction", + data: { + id: "recurring-1", + account_id: "acct-1", + merchant_id: "merchant-1", + amount: "-89.99", + currency: "USD", + expected_day_of_month: 14, + last_occurrence_date: "2024-01-14", + next_expected_date: "2024-02-14", + status: "active", + occurrence_count: 6, + manual: true, + expected_amount_min: "-95.00", + expected_amount_max: "-85.00", + expected_amount_avg: "-89.99" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + recurring_transaction = @family.recurring_transactions.first + assert_not_nil recurring_transaction + assert_equal "Main Checking", recurring_transaction.account.name + assert_equal "Internet Provider", recurring_transaction.merchant.name + assert_equal(-89.99, recurring_transaction.amount.to_f) + assert_equal "USD", recurring_transaction.currency + assert_equal 14, recurring_transaction.expected_day_of_month + assert_equal Date.parse("2024-01-14"), recurring_transaction.last_occurrence_date + assert_equal Date.parse("2024-02-14"), recurring_transaction.next_expected_date + assert_equal "active", recurring_transaction.status + assert_equal 6, recurring_transaction.occurrence_count + assert_equal true, recurring_transaction.manual + assert_equal(-95.0, recurring_transaction.expected_amount_min.to_f) + assert_equal(-85.0, recurring_transaction.expected_amount_max.to_f) + assert_equal(-89.99, recurring_transaction.expected_amount_avg.to_f) + end + + test "imports recurring transactions with unknown status fallback" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Main Checking", + balance: "5000", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Merchant", + data: { + id: "merchant-1", + name: "Streaming Service" + } + }, + { + type: "RecurringTransaction", + data: { + id: "recurring-1", + account_id: "acct-1", + merchant_id: "merchant-1", + amount: "-15.99", + currency: "USD", + expected_day_of_month: "8", + last_occurrence_date: "2024-01-08", + next_expected_date: "2024-02-08", + status: "paused", + occurrence_count: 2 + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + recurring_transaction = @family.recurring_transactions.first + assert_not_nil recurring_transaction + assert_equal 8, recurring_transaction.expected_day_of_month + assert_equal Date.parse("2024-01-08"), recurring_transaction.last_occurrence_date + assert_equal Date.parse("2024-02-08"), recurring_transaction.next_expected_date + assert_equal "active", recurring_transaction.status + end + + test "skips recurring transactions with missing recurrence dates" do + ndjson = build_ndjson([ + { + type: "RecurringTransaction", + data: { + id: "recurring-1", + amount: "-15.99", + currency: "USD", + expected_day_of_month: "8", + last_occurrence_date: nil, + status: "active", + occurrence_count: 2, + name: "Streaming Service" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + assert_equal 0, @family.recurring_transactions.count + end + + test "skips recurring transactions when referenced account is missing" do + ndjson = build_ndjson([ + { + type: "RecurringTransaction", + data: { + id: "recurring-1", + account_id: "missing-account", + amount: "-89.99", + currency: "USD", + expected_day_of_month: 14, + last_occurrence_date: "2024-01-14", + next_expected_date: "2024-02-14", + status: "active", + name: "Internet Provider" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + assert_equal 0, @family.recurring_transactions.count + end + + test "skips recurring transactions with blank expected day" do + ndjson = build_ndjson([ + { + type: "RecurringTransaction", + data: { + id: "recurring-1", + amount: "-89.99", + currency: "USD", + expected_day_of_month: "", + status: "active", + name: "Internet Provider" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + assert_equal 0, @family.recurring_transactions.count + end + test "imports transactions with references" do ndjson = build_ndjson([ { @@ -675,6 +845,23 @@ class Family::DataImporterTest < ActiveSupport::TestCase } }, # Transaction + { + type: "RecurringTransaction", + data: { + id: "recurring-grocery", + account_id: "acct-main", + merchant_id: "merchant-1", + amount: "-75.50", + currency: "USD", + expected_day_of_month: 15, + last_occurrence_date: "2024-01-15", + next_expected_date: "2024-02-15", + status: "active", + occurrence_count: 3, + manual: false + } + }, + # Transaction { type: "Transaction", data: { @@ -738,6 +925,7 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal 1, @family.categories.count assert_equal 1, @family.tags.count assert_equal 1, @family.merchants.count + assert_equal 1, @family.recurring_transactions.count assert_equal 1, @family.transactions.count assert_equal 1, @family.budgets.count assert_equal 1, @family.budget_categories.count @@ -748,6 +936,10 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal "Food", transaction.category.name assert_equal "Local Grocery", transaction.merchant.name assert_equal "Weekly", transaction.tags.first.name + + recurring_transaction = @family.recurring_transactions.first + assert_equal "Main Checking", recurring_transaction.account.name + assert_equal "Local Grocery", recurring_transaction.merchant.name end private