Files
sure/app/models/qif_import.rb
Serge L 78b334277c QIF imports: Add date format auto-detection and manual override (#1368)
* feat: improve QIF import date format selection

- Added a reusable date format auto-detection method.

- Show a live preview of the first parsed date that updates client-side
  as the user changes the dropdown selection, via a new
  qif-date-format Stimulus controller.

- Show an error alert and disable the submit button when no supported
  date format can parse the file's dates.

* A few polishing fixes:
- Missing return on redirects
Stale REASONABLE_DATE_RANGE constant.
- Replaced the frozen constant with a class method
Bare inline rescue — Replaced Date.strptime(s, fmt) rescue nil with an explicit begin/rescue catching.
- save!(validate: false) in controller — Changed to update_column(:column_mappings, ...) in qif_category_selections_controller.rb:22, matching the pattern used in detect_and_set_qif_date_format!.
- Unescaped JSON in HTML attribute — Replaced the raw <div> with tag.div ... do block in show.html.erb:16, letting Rails properly escape the data attribute value.

* fix: address review feedback for QIF date format feature

- Add missing `return` after redirect for non-QIF imports
- Pass date_format to parse_opening_balance in will_adjust_opening_anchor?
- Return empty array when no usable date sample exists for format preview
- Add sr-only label to date format select for accessibility
- Consolidate duplicate try_parse_date/parse_qif_date into single method
- Remove misleading ambiguity scoring comment from detect_date_format
- Skip redundant sync_mappings when date format already triggered a sync

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Use %{product_name} interpolation in locale strings

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-05 09:27:24 +02:00

418 lines
14 KiB
Ruby
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
class QifImport < Import
after_create :set_default_config
# The date format used to parse the raw QIF file's D-fields (e.g. "%m/%d/%Y").
# Stored in column_mappings so it doesn't conflict with date_format, which is
# always "%Y-%m-%d" because QIF rows store dates in ISO 8601 after parsing.
def qif_date_format
column_mappings&.dig("qif_date_format") || "%m/%d/%Y"
end
def qif_date_format=(fmt)
self.column_mappings = (column_mappings || {}).merge("qif_date_format" => fmt)
end
# Parses the stored QIF content and creates Import::Row records.
# Overrides the base CSV-based method with QIF-specific parsing.
#
# On first run (qif_date_format not yet set), auto-detects the date format
# from the QIF file's D-field samples.
def generate_rows_from_csv
detect_and_set_qif_date_format! unless column_mappings&.key?("qif_date_format")
rows.destroy_all
if investment_account?
generate_investment_rows
else
generate_transaction_rows
end
update_column(:rows_count, rows.count)
end
def import!
transaction do
mappings.each(&:create_mappable!)
if investment_account?
import_investment_rows!
else
import_transaction_rows!
if (ob = QifParser.parse_opening_balance(raw_file_str, date_format: qif_date_format))
Account::OpeningBalanceManager.new(account).set_opening_balance(
balance: ob[:amount],
date: ob[:date]
)
else
adjust_opening_anchor_if_needed!
end
end
end
end
# QIF has a fixed format no CSV column mapping step needed.
def requires_csv_workflow?
false
end
def rows_ordered
rows.order(date: :desc, id: :desc)
end
def column_keys
if qif_account_type == "Invst"
%i[date ticker qty price amount currency name]
else
%i[date amount name currency category tags notes]
end
end
def publishable?
account.present? && super
end
# Returns true if import! will move the opening anchor back to cover transactions
# that predate the current anchor date. Used to show a notice in the confirm step.
def will_adjust_opening_anchor?
return false if investment_account?
return false if QifParser.parse_opening_balance(raw_file_str, date_format: qif_date_format).present?
return false unless account.present?
manager = Account::OpeningBalanceManager.new(account)
return false unless manager.has_opening_anchor?
earliest = earliest_row_date
earliest.present? && earliest < manager.opening_date
end
# The date the opening anchor will be moved to when will_adjust_opening_anchor? is true.
def adjusted_opening_anchor_date
earliest = earliest_row_date
(earliest - 1.day) if earliest.present?
end
# The account type declared in the QIF file (e.g. "CCard", "Bank", "Invst").
def qif_account_type
return @qif_account_type if instance_variable_defined?(:@qif_account_type)
@qif_account_type = raw_file_str.present? ? QifParser.account_type(raw_file_str) : nil
end
# Unique categories used across all rows (blank entries excluded).
def row_categories
rows.distinct.pluck(:category).reject(&:blank?).sort
end
# Returns true if the QIF file contains any split transactions.
def has_split_transactions?
return @has_split_transactions if defined?(@has_split_transactions)
@has_split_transactions = parsed_transactions_with_splits.any?(&:split)
end
# Categories that appear on split transactions in the QIF file.
# Split transactions use S/$ fields to break a total into sub-amounts;
# the app does not yet support splits, so these categories are flagged.
def split_categories
return @split_categories if defined?(@split_categories)
split_cats = parsed_transactions_with_splits.select(&:split).map(&:category).reject(&:blank?).uniq.sort
@split_categories = split_cats & row_categories
end
# Unique tags used across all rows (blank entries excluded).
def row_tags
rows.flat_map(&:tags_list).uniq.reject(&:blank?).sort
end
# True once the category/tag selection step has been completed
# (sync_mappings has been called, which always produces at least one mapping).
def categories_selected?
mappings.any?
end
def mapping_steps
[ Import::CategoryMapping, Import::TagMapping ]
end
# QIF dates need normalization (apostrophe → separator, 2-digit year expansion)
# before strptime can parse them, so we delegate to QifParser.
def raw_date_samples
QifParser.extract_raw_dates(raw_file_str)
end
def try_parse_date_sample(sample, format:)
QifParser.try_parse_date(sample, date_format: format)
end
private
def parsed_transactions_with_splits
@parsed_transactions_with_splits ||= QifParser.parse(raw_file_str)
end
def investment_account?
qif_account_type == "Invst"
end
# ------------------------------------------------------------------
# Row generation
# ------------------------------------------------------------------
def generate_transaction_rows
transactions = QifParser.parse(raw_file_str, date_format: qif_date_format)
mapped_rows = transactions.map do |trn|
{
date: trn.date.to_s,
amount: trn.amount.to_s,
currency: default_currency.to_s,
name: (trn.payee.presence || default_row_name).to_s,
notes: trn.memo.to_s,
category: trn.category.to_s,
tags: trn.tags.join("|"),
account: "",
qty: "",
ticker: "",
price: "",
exchange_operating_mic: "",
entity_type: ""
}
end
if mapped_rows.any?
rows.insert_all!(mapped_rows)
rows.reset
end
end
def generate_investment_rows
inv_transactions = QifParser.parse_investment_transactions(raw_file_str, date_format: qif_date_format)
mapped_rows = inv_transactions.map do |trn|
if QifParser::TRADE_ACTIONS.include?(trn.action)
qty = trade_qty_for(trn.action, trn.qty)
{
date: trn.date.to_s,
ticker: trn.security_ticker.to_s,
qty: qty.to_s,
price: trn.price.to_s,
amount: trn.amount.to_s,
currency: default_currency.to_s,
name: trade_row_name(trn),
notes: trn.memo.to_s,
category: "",
tags: "",
account: "",
exchange_operating_mic: "",
entity_type: trn.action
}
else
{
date: trn.date.to_s,
amount: trn.amount.to_s,
currency: default_currency.to_s,
name: transaction_row_name(trn),
notes: trn.memo.to_s,
category: trn.category.to_s,
tags: trn.tags.join("|"),
account: "",
qty: "",
ticker: "",
price: "",
exchange_operating_mic: "",
entity_type: trn.action
}
end
end
if mapped_rows.any?
rows.insert_all!(mapped_rows)
rows.reset
end
end
# ------------------------------------------------------------------
# Import execution
# ------------------------------------------------------------------
def import_transaction_rows!
transactions = rows.map do |row|
category = mappings.categories.mappable_for(row.category)
tags = row.tags_list.map { |tag| mappings.tags.mappable_for(tag) }.compact
Transaction.new(
category: category,
tags: tags,
entry: Entry.new(
account: account,
date: row.date_iso,
amount: row.signed_amount,
name: row.name,
currency: row.currency,
notes: row.notes,
import: self,
import_locked: true
)
)
end
Transaction.import!(transactions, recursive: true)
end
def import_investment_rows!
trade_rows = rows.select { |r| QifParser::TRADE_ACTIONS.include?(r.entity_type) }
transaction_rows = rows.reject { |r| QifParser::TRADE_ACTIONS.include?(r.entity_type) }
if trade_rows.any?
trades = trade_rows.map do |row|
security = find_or_create_security(ticker: row.ticker)
# Use the stored T-field amount for accuracy (includes any fees/commissions).
# Buy-like actions are cash outflows (positive); sell-like are inflows (negative).
entry_amount = QifParser::BUY_LIKE_ACTIONS.include?(row.entity_type) ? row.amount.to_d : -row.amount.to_d
Trade.new(
security: security,
qty: row.qty.to_d,
price: row.price.to_d,
currency: row.currency,
investment_activity_label: investment_activity_label_for(row.entity_type),
entry: Entry.new(
account: account,
date: row.date_iso,
amount: entry_amount,
name: row.name,
currency: row.currency,
import: self,
import_locked: true
)
)
end
Trade.import!(trades, recursive: true)
end
if transaction_rows.any?
transactions = transaction_rows.map do |row|
# Inflow actions: money entering account → negative Entry.amount
# Outflow actions: money leaving account → positive Entry.amount
entry_amount = QifParser::INFLOW_TRANSACTION_ACTIONS.include?(row.entity_type) ? -row.amount.to_d : row.amount.to_d
category = mappings.categories.mappable_for(row.category)
tags = row.tags_list.map { |tag| mappings.tags.mappable_for(tag) }.compact
Transaction.new(
category: category,
tags: tags,
entry: Entry.new(
account: account,
date: row.date_iso,
amount: entry_amount,
name: row.name,
currency: row.currency,
notes: row.notes,
import: self,
import_locked: true
)
)
end
Transaction.import!(transactions, recursive: true)
end
end
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def adjust_opening_anchor_if_needed!
manager = Account::OpeningBalanceManager.new(account)
return unless manager.has_opening_anchor?
earliest = earliest_row_date
return unless earliest.present? && earliest < manager.opening_date
Account::OpeningBalanceManager.new(account).set_opening_balance(
balance: manager.opening_balance,
date: earliest - 1.day
)
end
def earliest_row_date
str = rows.minimum(:date)
Date.parse(str) if str.present?
end
def set_default_config
update!(
signage_convention: "inflows_positive",
date_format: "%Y-%m-%d",
number_format: "1,234.56"
)
end
# Auto-detects the QIF file's date format from D-field samples and persists it.
# Falls back to "%m/%d/%Y" (US convention) if detection is inconclusive.
def detect_and_set_qif_date_format!
samples = QifParser.extract_raw_dates(raw_file_str)
detected = Import.detect_date_format(samples, fallback: "%m/%d/%Y")
self.qif_date_format = detected
update_column(:column_mappings, column_mappings)
end
# Returns the signed qty for a trade row:
# buy-like actions keep qty positive; sell-like negate it.
def trade_qty_for(action, raw_qty)
qty = raw_qty.to_d
QifParser::SELL_LIKE_ACTIONS.include?(action) ? -qty : qty
end
def investment_activity_label_for(action)
return nil if action.blank?
QifParser::BUY_LIKE_ACTIONS.include?(action) ? "Buy" : "Sell"
end
def trade_row_name(trn)
type = QifParser::BUY_LIKE_ACTIONS.include?(trn.action) ? "buy" : "sell"
ticker = trn.security_ticker.presence || trn.security_name || "Unknown"
Trade.build_name(type, trn.qty.to_d.abs, ticker)
end
def transaction_row_name(trn)
security = trn.security_name.presence
payee = trn.payee.presence
case trn.action
when "Div" then payee || (security ? "Dividend: #{security}" : "Dividend")
when "IntInc" then payee || (security ? "Interest: #{security}" : "Interest")
when "XIn" then payee || "Cash Transfer In"
when "XOut" then payee || "Cash Transfer Out"
when "CGLong" then payee || (security ? "Capital Gain (Long): #{security}" : "Capital Gain (Long)")
when "CGShort" then payee || (security ? "Capital Gain (Short): #{security}" : "Capital Gain (Short)")
when "MiscInc" then payee || trn.memo.presence || "Miscellaneous Income"
when "MiscExp" then payee || trn.memo.presence || "Miscellaneous Expense"
else payee || trn.action
end
end
def find_or_create_security(ticker: nil, exchange_operating_mic: nil)
return nil unless ticker.present?
@security_cache ||= {}
cache_key = [ ticker, exchange_operating_mic ].compact.join(":")
security = @security_cache[cache_key]
return security if security.present?
security = Security::Resolver.new(
ticker,
exchange_operating_mic: exchange_operating_mic.presence
).resolve
@security_cache[cache_key] = security
security
end
end