fix(imports): preserve Sure opening balance history (#1595)

app/models/family/data_exporter.rb

@@ -235,6 +235,7 @@ class Family::DataExporter
             amount: entry.amount,
             currency: entry.currency,
             name: entry.name,
+            kind: entry.entryable.kind,
             created_at: entry.created_at,
             updated_at: entry.updated_at
           }

app/models/family/data_importer.rb

@@ -1,3 +1,5 @@
+require "set"
+
 class Family::DataImporter
   SUPPORTED_TYPES = %w[Account Category Tag Merchant Transaction Trade Valuation Budget BudgetCategory Rule].freeze
   ACCOUNTABLE_TYPES = Accountable::TYPES.freeze
@@ -19,6 +21,8 @@ class Family::DataImporter
 
   def import!
     records = parse_ndjson
+    @oldest_import_entry_dates_by_account = oldest_import_entry_dates_by_account(records)
+    @imported_opening_anchor_account_ids = imported_opening_anchor_account_ids(records["Valuation"] || [])
 
     Import.transaction do
       # Import in dependency order
@@ -97,10 +101,15 @@ class Family::DataImporter
 
         account.save!
 
-        # Set opening balance if we have a historical balance
+        # Set opening balance if we have a historical balance and the import
-        if data["balance"].present?
+        # does not provide an explicit opening-anchor valuation for this account.
+        if data["balance"].present? && !@imported_opening_anchor_account_ids.include?(old_id)
           manager = Account::OpeningBalanceManager.new(account)
-          manager.set_opening_balance(balance: data["balance"].to_d)
+          result = manager.set_opening_balance(
+            balance: data["balance"].to_d,
+            date: opening_balance_date_for(old_id, data)
+          )
+          log_failed_opening_balance_import(account, old_id, result) unless result.success?
         end
 
         @id_mappings[:accounts][old_id] = account.id
@@ -281,7 +290,7 @@ class Family::DataImporter
 
         account = @family.accounts.find(new_account_id)
 
-        valuation = Valuation.new
+        valuation = Valuation.new(kind: valuation_kind_for(data["kind"]))
 
         entry = Entry.new(
           account: account,
@@ -297,6 +306,63 @@ class Family::DataImporter
       end
     end
 
+    def oldest_import_entry_dates_by_account(records)
+      dates_by_account = {}
+
+      # Account-level opening balances must precede every imported account
+      # activity, including standalone valuation snapshots.
+      %w[Transaction Trade Valuation].each do |type|
+        records[type].to_a.each do |record|
+          data = record["data"] || {}
+          account_id = data["account_id"]
+          date = parse_import_date(data["date"])
+          next if account_id.blank? || date.blank?
+
+          dates_by_account[account_id] = [ dates_by_account[account_id], date ].compact.min
+        end
+      end
+
+      dates_by_account
+    end
+
+    def imported_opening_anchor_account_ids(records)
+      records.each_with_object(Set.new) do |record, account_ids|
+        data = record["data"] || {}
+        next unless data["kind"].to_s == "opening_anchor"
+        next if data["account_id"].blank?
+
+        account_ids.add(data["account_id"])
+      end
+    end
+
+    def opening_balance_date_for(old_id, data)
+      explicit_date = parse_import_date(
+        data["opening_balance_date"] || data["opening_balance_on"]
+      )
+
+      max_allowed_date = @oldest_import_entry_dates_by_account[old_id]&.prev_day
+      [ explicit_date, max_allowed_date ].compact.min
+    end
+
+    def log_failed_opening_balance_import(account, old_id, result)
+      Rails.logger.warn(
+        "Failed to import opening balance for account #{account.id} from source account #{old_id}: #{result.error}"
+      )
+    end
+
+    def valuation_kind_for(value)
+      kind = value.to_s
+      Valuation.kinds.key?(kind) ? kind : "reconciliation"
+    end
+
+    def parse_import_date(value)
+      return if value.blank?
+
+      Date.parse(value.to_s)
+    rescue Date::Error
+      nil
+    end
+
     def import_budgets(records)
       records.each do |record|
         data = record["data"]

test/models/family/data_exporter_test.rb

@@ -310,6 +310,34 @@ class Family::DataExporterTest < ActiveSupport::TestCase
     end
   end
 
+  test "exports valuation kind in NDJSON" do
+    valuation_entry = @account.entries.create!(
+      date: Date.parse("2020-04-01"),
+      amount: 1000,
+      name: "Opening balance",
+      currency: "USD",
+      entryable: Valuation.new(kind: "opening_anchor")
+    )
+
+    zip_data = @exporter.generate_export
+
+    Zip::File.open_buffer(zip_data) do |zip|
+      ndjson_content = zip.read("all.ndjson")
+      valuation_lines = ndjson_content.split("\n").select do |line|
+        JSON.parse(line)["type"] == "Valuation"
+      end
+
+      assert valuation_lines.any?
+
+      valuation_data = valuation_lines
+        .map { |line| JSON.parse(line) }
+        .find { |line| line.dig("data", "entry_id") == valuation_entry.id }
+
+      assert valuation_data
+      assert_equal "opening_anchor", valuation_data["data"]["kind"]
+    end
+  end
+
   test "only exports rules from the specified family" do
     # Create a rule for another family that should NOT be exported
     other_rule = @other_family.rules.build(

test/models/family/data_importer_test.rb

@@ -77,6 +77,111 @@ class Family::DataImporterTest < ActiveSupport::TestCase
     assert_equal "active", account.status
   end
 
+  test "dates synthesized account opening balance before oldest imported activity" do
+    ndjson = build_ndjson([
+      {
+        type: "Account",
+        data: {
+          id: "acct-1",
+          name: "Main Account",
+          balance: "5000",
+          currency: "USD",
+          accountable_type: "Depository"
+        }
+      },
+      {
+        type: "Transaction",
+        data: {
+          id: "txn-1",
+          account_id: "acct-1",
+          date: "2020-04-02",
+          amount: "-50.00",
+          name: "Grocery Store",
+          currency: "USD"
+        }
+      }
+    ])
+
+    Family::DataImporter.new(@family, ndjson).import!
+
+    account = @family.accounts.find_by!(name: "Main Account")
+    opening_anchor = account.valuations.opening_anchor.first
+
+    assert_not_nil opening_anchor
+    assert_equal Date.parse("2020-04-01"), opening_anchor.entry.date
+  end
+
+  test "clamps explicit account opening balance dates before imported activity" do
+    ndjson = build_ndjson([
+      {
+        type: "Account",
+        data: {
+          id: "acct-1",
+          name: "Main Account",
+          balance: "5000",
+          currency: "USD",
+          accountable_type: "Depository",
+          opening_balance_date: "2020-04-02"
+        }
+      },
+      {
+        type: "Transaction",
+        data: {
+          id: "txn-1",
+          account_id: "acct-1",
+          date: "2020-04-02",
+          amount: "-50.00",
+          name: "Grocery Store",
+          currency: "USD"
+        }
+      }
+    ])
+
+    Family::DataImporter.new(@family, ndjson).import!
+
+    account = @family.accounts.find_by!(name: "Main Account")
+    opening_anchor = account.valuations.opening_anchor.first
+
+    assert_not_nil opening_anchor
+    assert_equal Date.parse("2020-04-01"), opening_anchor.entry.date
+  end
+
+  test "imports explicit opening anchor valuations without synthesizing duplicates" do
+    ndjson = build_ndjson([
+      {
+        type: "Account",
+        data: {
+          id: "acct-1",
+          name: "Main Account",
+          balance: "5000",
+          currency: "USD",
+          accountable_type: "Depository"
+        }
+      },
+      {
+        type: "Valuation",
+        data: {
+          id: "val-opening",
+          account_id: "acct-1",
+          date: "2020-04-01",
+          amount: "5000",
+          name: "Opening balance",
+          currency: "USD",
+          kind: "opening_anchor"
+        }
+      }
+    ])
+
+    Family::DataImporter.new(@family, ndjson).import!
+
+    account = @family.accounts.find_by!(name: "Main Account")
+    opening_anchors = account.valuations.opening_anchor.to_a
+
+    assert_equal 1, opening_anchors.count
+    assert_equal Date.parse("2020-04-01"), opening_anchors.first.entry.date
+    assert_equal 5000.0, opening_anchors.first.entry.amount.to_f
+  end
+
   test "imports categories with parent relationships" do
     ndjson = build_ndjson([
       {
@@ -284,6 +389,39 @@ class Family::DataImporterTest < ActiveSupport::TestCase
     assert_equal 520000.0, valuation.entry.amount.to_f
   end
 
+  test "imports unknown valuation kinds as reconciliations" do
+    ndjson = build_ndjson([
+      {
+        type: "Account",
+        data: {
+          id: "prop-acct-1",
+          name: "Property",
+          balance: "500000",
+          currency: "USD",
+          accountable_type: "Property"
+        }
+      },
+      {
+        type: "Valuation",
+        data: {
+          id: "val-1",
+          account_id: "prop-acct-1",
+          date: "2024-06-15",
+          amount: "520000",
+          name: "Updated valuation",
+          currency: "USD",
+          kind: "legacy_unknown_kind"
+        }
+      }
+    ])
+
+    Family::DataImporter.new(@family, ndjson).import!
+
+    account = @family.accounts.find_by!(name: "Property")
+    valuation = account.valuations.joins(:entry).find_by!(entries: { name: "Updated valuation" })
+    assert_equal "reconciliation", valuation.kind
+  end
+
   test "imports budgets" do
     ndjson = build_ndjson([
       {