From d637cd2f75ebc77ba717d0aa7b8b734e8ca1c239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orange=F0=9F=8D=8A?= Date: Sun, 28 Jun 2026 23:02:07 +0800 Subject: [PATCH] fix(imports): support QIF dd mmm yyyy date format (#2500) * Fix QIF import for dd mmm yyyy dates (#2498) American Express QIF exports use dd mmm yyyy D-fields (e.g. D26 Jan 2026). Import previously failed with "Unable to detect date format" for two reasons: 1. QifParser.normalize_qif_date stripped all internal whitespace, turning 26 Jan 2026 into the unparseable 26Jan2026. For month-name dates the space IS the separator, so collapse multiples to a single space instead of removing them. Numeric dates keep the existing strip-all behavior. 2. Family::DATE_FORMATS had no candidate mapping to dd mmm yyyy, so detect_date_format could not match it. Add "%d %b %Y" (DD MMM YYYY). Extend the 2-digit-year expansion separator class to include space so month-name dates with 2-digit years also normalize (26 Jan 26 -> 26 Jan 2026). Covered by normalize/parse/detect and Amex-style row-generation regression tests in qif_import_test.rb. * test(qif): cover month-name 2-digit year normalization & parse Addresses CodeRabbit nitpick on #2500: the production change extends the 2-digit-year expansion regex separator class to include a space so month-name dates like "26 Jan 26" normalize to "26 Jan 2026". Add normalize_qif_date and parse_qif_date regressions for that branch. --- app/models/concerns/qif_parser.rb | 21 ++++++--- app/models/family.rb | 4 +- test/models/qif_import_test.rb | 71 +++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/app/models/concerns/qif_parser.rb b/app/models/concerns/qif_parser.rb index 65e8a9ba8..654cae5d4 100644 --- a/app/models/concerns/qif_parser.rb +++ b/app/models/concerns/qif_parser.rb @@ -354,12 +354,15 @@ module QifParser # - Optional spaces around components: 6/ 4'20 → 6/4/20 # - Dot separators: 04.06.2020 # - Dash separators: 04-06-2020 + # - Month-name separators: 26 Jan 2026 (Amex-style "dd mmm yyyy") # # This method: # 1. Strips whitespace # 2. Replaces the Quicken apostrophe with the file's date separator - # 3. Expands 2-digit years to 4-digit (00-99 → 2000-2099, capped at current year) - # 4. Returns a cleaned date string suitable for Date.strptime + # 3. Collapses whitespace (removes it for numeric dates; keeps single + # spaces for month-name dates since the space IS the separator) + # 4. Expands 2-digit years to 4-digit (00-99 → 2000-2099, capped at current year) + # 5. Returns a cleaned date string suitable for Date.strptime def self.normalize_qif_date(date_str) return nil if date_str.blank? @@ -371,12 +374,20 @@ module QifParser s = s.gsub("'", sep) end - # Remove internal spaces (e.g. "6/ 4/20" → "6/4/20") - s = s.gsub(/\s+/, "") + # Whitespace handling depends on the date style: + # - Month-name dates (e.g. "26 Jan 2026") use spaces as the field + # separator, so collapse multiples to a single space. + # - Numeric dates (e.g. Quicken's padded "6/ 4/20") use / . or - as + # separators and may carry stray padding spaces, so strip them. + if s.match?(/[A-Za-z]/) + s = s.gsub(/\s+/, " ").strip + else + s = s.gsub(/\s+/, "") + end # Expand 2-digit year at end to 4-digit, but only when the string doesn't # already contain a 4-digit number (which would be a full year). - if !s.match?(/\d{4}/) && (m = s.match(%r{\A(.+[/.\-])(\d{2})\z})) + if !s.match?(/\d{4}/) && (m = s.match(%r{\A(.+[/.\- ])(\d{2})\z})) short_year = m[2].to_i full_year = 2000 + short_year full_year -= 100 if full_year > Date.today.year diff --git a/app/models/family.rb b/app/models/family.rb index 082316c1e..908fa5702 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -15,7 +15,9 @@ class Family < ApplicationRecord [ "MM/DD/YYYY", "%m/%d/%Y" ], [ "D/MM/YYYY", "%e/%m/%Y" ], [ "YYYY.MM.DD", "%Y.%m.%d" ], - [ "YYYYMMDD", "%Y%m%d" ] + [ "YYYYMMDD", "%Y%m%d" ], + # QIF month-name imports rely on QifParser preserving normalized spaces. + [ "DD MMM YYYY", "%d %b %Y" ] ].freeze diff --git a/test/models/qif_import_test.rb b/test/models/qif_import_test.rb index e370b42be..d48007a7e 100644 --- a/test/models/qif_import_test.rb +++ b/test/models/qif_import_test.rb @@ -854,6 +854,22 @@ class QifImportTest < ActiveSupport::TestCase # ── QifParser: normalize_qif_date ────────────────────────────────────────── + test "normalize_qif_date preserves single spaces for month-name dates" do + assert_equal "26 Jan 2026", QifParser.send(:normalize_qif_date, "26 Jan 2026") + end + + test "normalize_qif_date collapses multiple spaces in month-name dates" do + assert_equal "26 Jan 2026", QifParser.send(:normalize_qif_date, "26 Jan 2026") + end + + test "normalize_qif_date still strips spaces from numeric dates" do + assert_equal "6/4/2020", QifParser.send(:normalize_qif_date, "6/ 4/2020") + end + + test "normalize_qif_date expands 2-digit year for month-name dates" do + assert_equal "26 Jan 2026", QifParser.send(:normalize_qif_date, "26 Jan 26") + end + test "normalize_qif_date converts apostrophe 2-digit year" do assert_equal "6/4/2020", QifParser.send(:normalize_qif_date, "6/ 4'20") end @@ -898,6 +914,14 @@ class QifImportTest < ActiveSupport::TestCase assert_equal "2020-06-04", QifParser.send(:parse_qif_date, "2020-06-04", date_format: "%Y-%m-%d") end + test "parse_qif_date parses month-name format (DD MMM YYYY)" do + assert_equal "2026-01-26", QifParser.send(:parse_qif_date, "26 Jan 2026", date_format: "%d %b %Y") + end + + test "parse_qif_date parses month-name format with 2-digit year" do + assert_equal "2026-01-26", QifParser.send(:parse_qif_date, "26 Jan 26", date_format: "%d %b %Y") + end + test "parse_qif_date returns nil for invalid date" do assert_nil QifParser.send(:parse_qif_date, "13/32/2020", date_format: "%m/%d/%Y") end @@ -962,6 +986,11 @@ class QifImportTest < ActiveSupport::TestCase assert_equal "%Y-%m-%d", Import.detect_date_format(samples) end + test "detect_date_format identifies month-name DD MMM YYYY format" do + samples = [ "26 Jan 2026", "23 Jan 2026", "02 Feb 2026" ] + assert_equal "%d %b %Y", Import.detect_date_format(samples) + end + test "detect_date_format returns fallback for blank samples" do assert_equal "%Y-%m-%d", Import.detect_date_format([]) assert_equal "%Y-%m-%d", Import.detect_date_format(nil) @@ -1016,6 +1045,48 @@ class QifImportTest < ActiveSupport::TestCase assert_equal "%d/%m/%Y", @import.reload.qif_date_format end + # Reproduces #2498: Amex-style QIF with dd mmm yyyy dates previously failed + # with "Unable to detect date format" because normalize_qif_date stripped all + # spaces (turning "26 Jan 2026" into the unparseable "26Jan2026"). + AMEX_DDD_MMM_YYYY_QIF = <<~QIF + !Type:CCard + D26 Jan 2026 + N20260126 + T-25.24 + PAMAZON.COM.CA WWW.AMAZON.CO + M + ^ + D23 Jan 2026 + N20260123 + T-10.49 + PSKIPTHEDISHES WINNIPEG (BROAD + M + ^ + QIF + + test "generate_rows_from_csv auto-detects DD MMM YYYY format" do + @import.update!(raw_file_str: AMEX_DDD_MMM_YYYY_QIF) + @import.generate_rows_from_csv + + assert_equal "%d %b %Y", @import.reload.qif_date_format + row = @import.rows.find_by(name: "AMAZON.COM.CA WWW.AMAZON.CO") + assert_not_nil row + assert_equal "2026-01-26", row.date + assert_equal "-25.24", row.amount + end + + test "DD MMM YYYY format appears in valid_date_formats_with_preview for Amex QIF" do + @import.update!(raw_file_str: AMEX_DDD_MMM_YYYY_QIF) + @import.generate_rows_from_csv + + formats = @import.valid_date_formats_with_preview + format_strs = formats.map { |f| f[:format] } + + assert_includes format_strs, "%d %b %Y" + dd_mmm = formats.find { |f| f[:format] == "%d %b %Y" } + assert_equal "2026-01-26", dd_mmm[:preview] + end + # ── QifParser: try_parse_date ─────────────────────────────────────────────── test "try_parse_date returns ISO date for valid format" do