diff --git a/app/controllers/import/configurations_controller.rb b/app/controllers/import/configurations_controller.rb index 6602e3fbe..65c47fc68 100644 --- a/app/controllers/import/configurations_controller.rb +++ b/app/controllers/import/configurations_controller.rb @@ -5,7 +5,8 @@ class Import::ConfigurationsController < ApplicationController def show # PDF imports are auto-configured from AI extraction, skip to clean step - redirect_to import_clean_path(@import) if @import.is_a?(PdfImport) + redirect_to import_clean_path(@import) and return if @import.is_a?(PdfImport) + redirect_to import_qif_category_selection_path(@import) and return if @import.is_a?(QifImport) end def update diff --git a/app/controllers/import/qif_category_selections_controller.rb b/app/controllers/import/qif_category_selections_controller.rb index 28669c950..2ed7b195a 100644 --- a/app/controllers/import/qif_category_selections_controller.rb +++ b/app/controllers/import/qif_category_selections_controller.rb @@ -4,15 +4,28 @@ class Import::QifCategorySelectionsController < ApplicationController before_action :set_import def show - @categories = @import.row_categories - @tags = @import.row_tags - @category_counts = @import.rows.group(:category).count.reject { |k, _| k.blank? } - @tag_counts = compute_tag_counts + valid_formats = @import.valid_date_formats_with_preview + @date_formats = valid_formats.map { |f| [ f[:label], f[:format] ] } + @date_previews = valid_formats.each_with_object({}) { |f, h| h[f[:format]] = f[:preview] } + @categories = @import.row_categories + @tags = @import.row_tags + @category_counts = @import.rows.group(:category).count.reject { |k, _| k.blank? } + @tag_counts = compute_tag_counts @split_categories = @import.split_categories @has_split_transactions = @import.has_split_transactions? end def update + # If the user changed the date format, re-generate rows with the new format. + format_changed = false + if selection_params[:date_format].present? && selection_params[:date_format] != @import.qif_date_format + format_changed = true + @import.qif_date_format = selection_params[:date_format] + @import.update_column(:column_mappings, @import.column_mappings) + @import.generate_rows_from_csv + @import.sync_mappings + end + all_categories = @import.row_categories all_tags = @import.row_tags @@ -38,7 +51,7 @@ class Import::QifCategorySelectionsController < ApplicationController end end - @import.sync_mappings + @import.sync_mappings unless format_changed end redirect_to import_clean_path(@import), notice: "Categories and tags saved." @@ -50,7 +63,7 @@ class Import::QifCategorySelectionsController < ApplicationController @import = Current.family.imports.find(params[:import_id]) unless @import.is_a?(QifImport) - redirect_to imports_path + redirect_to imports_path and return end end @@ -63,6 +76,6 @@ class Import::QifCategorySelectionsController < ApplicationController end def selection_params - params.permit(categories: [], tags: []) + params.permit(:date_format, categories: [], tags: []) end end diff --git a/app/javascript/controllers/qif_date_format_controller.js b/app/javascript/controllers/qif_date_format_controller.js new file mode 100644 index 000000000..6c1044f58 --- /dev/null +++ b/app/javascript/controllers/qif_date_format_controller.js @@ -0,0 +1,15 @@ +import { Controller } from "@hotwired/stimulus"; + +// Updates the date preview text when the QIF date format dropdown changes. +// Previews are precomputed server-side and passed as a JSON value. +export default class extends Controller { + static targets = ["preview"]; + static values = { previews: Object }; + + change(event) { + const format = event.target.value; + const date = this.previewsValue[format]; + + this.previewTarget.textContent = date || ""; + } +} diff --git a/app/models/concerns/qif_parser.rb b/app/models/concerns/qif_parser.rb index 38f1c1abf..65e8a9ba8 100644 --- a/app/models/concerns/qif_parser.rb +++ b/app/models/concerns/qif_parser.rb @@ -113,7 +113,7 @@ module QifParser # Parses all transactions from the file, excluding the Opening Balance entry. # Returns an array of ParsedTransaction structs. - def self.parse(content) + def self.parse(content, date_format: "%m/%d/%Y") return [] unless valid?(content) content = normalize_encoding(content) @@ -125,7 +125,7 @@ module QifParser section = extract_section(content, type) return [] unless section - parse_records(section).filter_map { |record| build_transaction(record) } + parse_records(section).filter_map { |record| build_transaction(record, date_format: date_format) } end # Returns the opening balance entry from the QIF file, if present. @@ -134,7 +134,7 @@ module QifParser # real transaction – it is the account's starting balance. # # Returns a hash { date: Date, amount: BigDecimal } or nil. - def self.parse_opening_balance(content) + def self.parse_opening_balance(content, date_format: "%m/%d/%Y") return nil unless valid?(content) content = normalize_encoding(content) @@ -149,7 +149,7 @@ module QifParser record = parse_records(section).find { |r| r["P"]&.strip == "Opening Balance" } return nil unless record - date = parse_qif_date(record["D"]) + date = parse_qif_date(record["D"], date_format: date_format) amount = parse_qif_amount(record["T"] || record["U"]) return nil unless date && amount @@ -228,7 +228,7 @@ module QifParser # Parses investment transactions from the !Type:Invst section. # Uses the !Type:Security sections to resolve security names to tickers. # Returns an array of ParsedInvestmentTransaction structs. - def self.parse_investment_transactions(content) + def self.parse_investment_transactions(content, date_format: "%m/%d/%Y") return [] unless valid?(content) content = normalize_encoding(content) @@ -239,7 +239,7 @@ module QifParser section = extract_section(content, "Invst") return [] unless section - parse_records(section).filter_map { |record| build_investment_transaction(record, ticker_by_name) } + parse_records(section).filter_map { |record| build_investment_transaction(record, ticker_by_name, date_format: date_format) } end # ------------------------------------------------------------------ @@ -292,7 +292,7 @@ module QifParser end private_class_method :parse_records - def self.build_transaction(record) + def self.build_transaction(record, date_format: "%m/%d/%Y") # "Opening Balance" is a Quicken convention for the account's starting balance – # it is not a real transaction and must not be imported as one. return nil if record["P"]&.strip == "Opening Balance" @@ -302,7 +302,7 @@ module QifParser return nil unless raw_date.present? && raw_amount.present? - date = parse_qif_date(raw_date) + date = parse_qif_date(raw_date, date_format: date_format) amount = parse_qif_amount(raw_amount) return nil unless date && amount @@ -347,38 +347,82 @@ module QifParser end private_class_method :parse_category_and_tags - # Parses a QIF date string into an ISO 8601 date string. + # Normalizes a QIF date string into a standard format that Date.strptime can + # handle. QIF files use Quicken-specific conventions: # - # Quicken uses several variants: - # M/D'YY → 6/ 4'20 → 2020-06-04 - # M/ D'YY → 6/ 4'20 → 2020-06-04 - # MM/DD/YYYY → 06/04/2020 (less common) - def self.parse_qif_date(date_str) + # - Apostrophe as year separator: 6/ 4'20 or 6/ 4'2020 + # - Optional spaces around components: 6/ 4'20 → 6/4/20 + # - Dot separators: 04.06.2020 + # - Dash separators: 04-06-2020 + # + # This method: + # 1. Strips whitespace + # 2. Replaces the Quicken apostrophe with the file's date separator + # 3. Expands 2-digit years to 4-digit (00-99 → 2000-2099, capped at current year) + # 4. Returns a cleaned date string suitable for Date.strptime + def self.normalize_qif_date(date_str) return nil if date_str.blank? - # Primary format: M/D'YY or M/ D'YY (spaces around day are optional) - if (m = date_str.match(%r{\A(\d{1,2})/\s*(\d{1,2})'(\d{2,4})\z})) - month = m[1].to_i - day = m[2].to_i - if m[3].length == 2 - year = 2000 + m[3].to_i - year -= 100 if year > Date.today.year - else - year = m[3].to_i - end - return Date.new(year, month, day).iso8601 + s = date_str.strip + + # Replace Quicken apostrophe year separator with the preceding separator + if s.include?("'") + sep = s.match(%r{[/.\-]})&.to_s || "/" + s = s.gsub("'", sep) end - # Fallback: MM/DD/YYYY - if (m = date_str.match(%r{\A(\d{1,2})/(\d{1,2})/(\d{4})\z})) - return Date.new(m[3].to_i, m[1].to_i, m[2].to_i).iso8601 + # Remove internal spaces (e.g. "6/ 4/20" → "6/4/20") + s = s.gsub(/\s+/, "") + + # Expand 2-digit year at end to 4-digit, but only when the string doesn't + # already contain a 4-digit number (which would be a full year). + if !s.match?(/\d{4}/) && (m = s.match(%r{\A(.+[/.\-])(\d{2})\z})) + short_year = m[2].to_i + full_year = 2000 + short_year + full_year -= 100 if full_year > Date.today.year + s = "#{m[1]}#{full_year}" end - nil + s + end + private_class_method :normalize_qif_date + + # Parses a QIF date string into an ISO 8601 date string using the given + # strptime format. The date is first normalized (apostrophe → separator, + # 2-digit year expansion, whitespace removal) before parsing. + # + # +date_format+ should be a strptime format string such as "%m/%d/%Y" or + # "%d.%m.%Y". Defaults to "%m/%d/%Y" (US convention) for backwards + # compatibility. + # Attempts to parse a raw QIF date string with the given format. + # Returns the parsed ISO 8601 date string, or nil if parsing fails. + def self.try_parse_date(date_str, date_format: "%m/%d/%Y") + normalized = normalize_qif_date(date_str) + return nil unless normalized + + Date.strptime(normalized, date_format).iso8601 rescue Date::Error, ArgumentError nil end - private_class_method :parse_qif_date + + private_class_method def self.parse_qif_date(date_str, date_format: "%m/%d/%Y") + try_parse_date(date_str, date_format: date_format) + end + + # Extracts all raw date strings from D-fields in transaction sections only. + # Skips metadata sections (Cat, Tag, Security) where D means "description". + # Used by Import.detect_date_format to sample dates before parsing. + def self.extract_raw_dates(content) + return [] if content.blank? + + content = normalize_encoding(content) + content = normalize_line_endings(content) + + transaction_sections = TRANSACTION_TYPES.filter_map { |type| extract_section(content, type) } + transaction_sections.flat_map { |section| section.scan(/^D(.+)$/i).flatten } + .map { |d| normalize_qif_date(d) } + .compact + end # Strips thousands-separator commas and returns a clean decimal string. def self.parse_qif_amount(amount_str) @@ -391,14 +435,14 @@ module QifParser # Builds a ParsedInvestmentTransaction from a raw record hash. # ticker_by_name maps security names (N field in !Type:Security) to tickers (S field). - def self.build_investment_transaction(record, ticker_by_name) + def self.build_investment_transaction(record, ticker_by_name, date_format: "%m/%d/%Y") action = record["N"]&.strip return nil unless action.present? raw_date = record["D"] return nil unless raw_date.present? - date = parse_qif_date(raw_date) + date = parse_qif_date(raw_date, date_format: date_format) return nil unless date security_name = record["Y"]&.strip diff --git a/app/models/import.rb b/app/models/import.rb index bc279b009..941a6ead6 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -20,6 +20,10 @@ class Import < ApplicationRecord "1,234" => { separator: "", delimiter: "," } # Zero-decimal currencies like JPY }.freeze + def self.reasonable_date_range + Date.new(1970, 1, 1)..Date.today.next_year(5) + end + AMOUNT_TYPE_STRATEGIES = %w[signed_amount custom_column].freeze belongs_to :family @@ -64,6 +68,51 @@ class Import < ApplicationRecord liberal_parsing: true ) end + + # Attempts to identify the best-matching date format from a list of candidates + # by trying to parse sample date strings with each format. + # + # Returns the strptime format string (e.g. "%m-%d-%Y") that best matches the + # samples, or the +fallback+ when no candidate can parse any sample. + # + # Scoring: + # 1. Formats that parse ALL samples beat those that only parse some. + # 2. Among equal parse counts, formats whose parsed dates fall within a + # reasonable range (1970..today+5y) score higher. + def detect_date_format(samples, candidates: Family::DATE_FORMATS.map(&:last), fallback: "%Y-%m-%d") + return fallback if samples.blank? + + cleaned = samples.map(&:to_s).reject(&:blank?).uniq.first(50) + return fallback if cleaned.empty? + + reasonable_range = reasonable_date_range + + scored = candidates.map do |fmt| + parsed_count = 0 + reasonable_count = 0 + + cleaned.each do |s| + begin + date = Date.strptime(s, fmt) + rescue Date::Error, ArgumentError + next + end + next unless date + + parsed_count += 1 + reasonable_count += 1 if reasonable_range.cover?(date) + end + + { format: fmt, parsed: parsed_count, reasonable: reasonable_count } + end + + # Filter to candidates that parsed at least one sample + viable = scored.select { |s| s[:parsed] > 0 } + return fallback if viable.empty? + + best = viable.max_by { |s| [ s[:parsed], s[:reasonable] ] } + best[:format] + end end def publish_later @@ -252,6 +301,38 @@ class Import < ApplicationRecord ) end + # Returns date formats that can successfully parse the file's date samples, + # filtered to dates within reasonable_date_range. + # Result: array of { label:, format:, preview: } hashes. + # Subclasses should override #raw_date_samples to provide date strings. + def valid_date_formats_with_preview + first_sample = raw_date_samples.find(&:present?) + return [] if first_sample.blank? + + Family::DATE_FORMATS.filter_map do |label, fmt| + parsed = try_parse_date_sample(first_sample, format: fmt) + next unless parsed + next unless self.class.reasonable_date_range.cover?(Date.parse(parsed)) + + { label: label, format: fmt, preview: parsed } + end + end + + # Returns raw date strings from the import file for format detection/preview. + # Subclasses should override to extract dates from their specific format. + def raw_date_samples + [] + end + + # Attempts to parse a raw date sample with the given strptime format. + # Returns ISO 8601 date string or nil. Subclasses can override for + # format-specific normalization (e.g. QIF apostrophe dates). + def try_parse_date_sample(sample, format:) + Date.strptime(sample, format).iso8601 + rescue Date::Error, ArgumentError + nil + end + def max_row_count 10000 end diff --git a/app/models/qif_import.rb b/app/models/qif_import.rb index 7fcb70c96..90a867d28 100644 --- a/app/models/qif_import.rb +++ b/app/models/qif_import.rb @@ -1,9 +1,25 @@ class QifImport < Import after_create :set_default_config + # The date format used to parse the raw QIF file's D-fields (e.g. "%m/%d/%Y"). + # Stored in column_mappings so it doesn't conflict with date_format, which is + # always "%Y-%m-%d" because QIF rows store dates in ISO 8601 after parsing. + def qif_date_format + column_mappings&.dig("qif_date_format") || "%m/%d/%Y" + end + + def qif_date_format=(fmt) + self.column_mappings = (column_mappings || {}).merge("qif_date_format" => fmt) + end + # Parses the stored QIF content and creates Import::Row records. # Overrides the base CSV-based method with QIF-specific parsing. + # + # On first run (qif_date_format not yet set), auto-detects the date format + # from the QIF file's D-field samples. def generate_rows_from_csv + detect_and_set_qif_date_format! unless column_mappings&.key?("qif_date_format") + rows.destroy_all if investment_account? @@ -24,7 +40,7 @@ class QifImport < Import else import_transaction_rows! - if (ob = QifParser.parse_opening_balance(raw_file_str)) + if (ob = QifParser.parse_opening_balance(raw_file_str, date_format: qif_date_format)) Account::OpeningBalanceManager.new(account).set_opening_balance( balance: ob[:amount], date: ob[:date] @@ -61,7 +77,7 @@ class QifImport < Import # that predate the current anchor date. Used to show a notice in the confirm step. def will_adjust_opening_anchor? return false if investment_account? - return false if QifParser.parse_opening_balance(raw_file_str).present? + return false if QifParser.parse_opening_balance(raw_file_str, date_format: qif_date_format).present? return false unless account.present? manager = Account::OpeningBalanceManager.new(account) @@ -119,6 +135,16 @@ class QifImport < Import [ Import::CategoryMapping, Import::TagMapping ] end + # QIF dates need normalization (apostrophe → separator, 2-digit year expansion) + # before strptime can parse them, so we delegate to QifParser. + def raw_date_samples + QifParser.extract_raw_dates(raw_file_str) + end + + def try_parse_date_sample(sample, format:) + QifParser.try_parse_date(sample, date_format: format) + end + private def parsed_transactions_with_splits @@ -134,7 +160,7 @@ class QifImport < Import # ------------------------------------------------------------------ def generate_transaction_rows - transactions = QifParser.parse(raw_file_str) + transactions = QifParser.parse(raw_file_str, date_format: qif_date_format) mapped_rows = transactions.map do |trn| { @@ -161,7 +187,7 @@ class QifImport < Import end def generate_investment_rows - inv_transactions = QifParser.parse_investment_transactions(raw_file_str) + inv_transactions = QifParser.parse_investment_transactions(raw_file_str, date_format: qif_date_format) mapped_rows = inv_transactions.map do |trn| if QifParser::TRADE_ACTIONS.include?(trn.action) @@ -327,6 +353,15 @@ class QifImport < Import ) end + # Auto-detects the QIF file's date format from D-field samples and persists it. + # Falls back to "%m/%d/%Y" (US convention) if detection is inconclusive. + def detect_and_set_qif_date_format! + samples = QifParser.extract_raw_dates(raw_file_str) + detected = Import.detect_date_format(samples, fallback: "%m/%d/%Y") + self.qif_date_format = detected + update_column(:column_mappings, column_mappings) + end + # Returns the signed qty for a trade row: # buy-like actions keep qty positive; sell-like negate it. def trade_qty_for(action, raw_qty) diff --git a/app/views/import/qif_category_selections/show.html.erb b/app/views/import/qif_category_selections/show.html.erb index fdf8cbbba..4976d9961 100644 --- a/app/views/import/qif_category_selections/show.html.erb +++ b/app/views/import/qif_category_selections/show.html.erb @@ -7,11 +7,39 @@

<%= t(".title") %>

-

<%= t(".description") %>

+

<%= t(".description", product_name: product_name) %>

<%= form_with url: import_qif_category_selection_path(@import), method: :put, class: "space-y-8" do |form| %> + <%# ── Date format ─────────────────────────────────────────────── %> + <%= tag.div class: "space-y-3", data: { controller: "qif-date-format", qif_date_format_previews_value: @date_previews.to_json } do %> +

<%= t("imports.date_format.heading") %>

+

<%= t("imports.date_format.description") %>

+ + <% if @date_formats.any? %> + <%= form.label :date_format, t("imports.date_format.heading"), class: "sr-only" %> + <%= form.select :date_format, + options_for_select(@date_formats, @import.qif_date_format), + {}, + { class: "w-full rounded-lg border border-secondary bg-container px-3 py-2 text-sm text-primary", + data: { action: "change->qif-date-format#change" } } %> + +
+ <%= t("imports.date_format.preview") %>: + <%= @date_previews[@import.qif_date_format] %> +
+ <% else %> +
+ <%= icon("circle-alert", size: "md", class: "text-destructive shrink-0 mt-0.5") %> +
+

<%= t("imports.date_format.error_title") %>

+

<%= t("imports.date_format.error_description") %>

+
+
+ <% end %> + <% end %> + <%# ── Split transaction warning ────────────────────────────── %> <% if @has_split_transactions %>
@@ -121,7 +149,8 @@ <%# ── Submit ──────────────────────────────────────────────────── %>
<%= form.submit t(".submit"), - class: "btn btn-primary w-full md:w-auto" %> + class: "btn btn-primary w-full md:w-auto", + disabled: @date_formats.empty? %>
<% end %> diff --git a/app/views/imports/_nav.html.erb b/app/views/imports/_nav.html.erb index 8bb457cd6..0c9ee5aaa 100644 --- a/app/views/imports/_nav.html.erb +++ b/app/views/imports/_nav.html.erb @@ -15,7 +15,7 @@ elsif import.is_a?(PdfImport) { name: t("imports.steps.confirm", default: "Confirm"), path: import_path(import), is_complete: import.complete?, step_number: 4 } ] elsif import.is_a?(QifImport) - # QIF imports skip Configure (fixed format) and add a category/tag selection step. + # QIF imports combine date-format configuration with category/tag selection. [ { name: t("imports.steps.upload", default: "Upload"), path: import_upload_path(import), is_complete: import.uploaded?, step_number: 1 }, { name: t("imports.steps.select", default: "Select"), path: import.uploaded? ? import_qif_category_selection_path(import) : nil, is_complete: import.categories_selected?, step_number: 2 }, diff --git a/config/locales/views/imports/de.yml b/config/locales/views/imports/de.yml index 263a2bdda..faab67cc0 100644 --- a/config/locales/views/imports/de.yml +++ b/config/locales/views/imports/de.yml @@ -3,8 +3,8 @@ de: import: qif_category_selections: show: - title: "Kategorien und Tags auswählen" - description: "Wähle aus, welche Kategorien und Tags aus deiner QIF-Datei in Sure übernommen werden sollen. Abgewählte Einträge werden aus den betreffenden Transaktionen entfernt." + title: "Konfigurieren und auswählen" + description: "Überprüfe das erkannte Datumsformat und wähle dann die Kategorien und Tags aus deiner QIF-Datei aus, die in %{product_name} importiert werden sollen." categories_heading: Kategorien categories_found: one: "1 Kategorie gefunden" @@ -88,6 +88,12 @@ de: instructions_5: Keine Kommas, Währungssymbole oder Klammern in Zahlen verwenden. title: Daten importieren imports: + date_format: + heading: Datumsformat + description: "Das Datumsformat wurde automatisch aus deiner Datei erkannt. Ändere es, wenn die Datumsangaben falsch aussehen." + preview: "Erstes erkanntes Datum" + error_title: "Datumsformat konnte nicht erkannt werden" + error_description: "Keines der unterstützten Datumsformate konnte die Datumsangaben in dieser Datei lesen. Bitte überprüfe, ob die Datei gültige Datumseinträge enthält." steps: upload: Hochladen configure: Konfigurieren diff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.yml index db1d792d2..9f41ab712 100644 --- a/config/locales/views/imports/en.yml +++ b/config/locales/views/imports/en.yml @@ -3,8 +3,8 @@ en: import: qif_category_selections: show: - title: "Select categories & tags" - description: "Choose which categories and tags from your QIF file to bring into Sure. Deselected items will be removed from those transactions." + title: "Configure & select" + description: "Review the detected date format, then choose which categories and tags from your QIF file to bring into %{product_name}." categories_heading: Categories categories_found: one: "1 category found" @@ -119,6 +119,12 @@ en: hint_html: Upload the all.ndjson file from your data export ZIP ndjson_invalid: Must be valid NDJSON with at least one record imports: + date_format: + heading: Date format + description: "The date format was auto-detected from your file. Change it if dates look incorrect." + preview: "First parsed date" + error_title: "Unable to detect date format" + error_description: "None of the supported date formats could parse the dates in this file. Please check that the file contains valid date entries." type_labels: transaction_import: "Transaction import" trade_import: "Trade import" diff --git a/config/locales/views/imports/es.yml b/config/locales/views/imports/es.yml index 51fe0142c..b8f0c9ca5 100644 --- a/config/locales/views/imports/es.yml +++ b/config/locales/views/imports/es.yml @@ -3,8 +3,8 @@ es: import: qif_category_selections: show: - title: "Seleccionar categorías y etiquetas" - description: "Elige qué categorías y etiquetas de tu archivo QIF importar en Sure. Los elementos deseleccionados se eliminarán de esas transacciones." + title: "Configurar y seleccionar" + description: "Revisa el formato de fecha detectado y luego elige qué categorías y etiquetas de tu archivo QIF importar en %{product_name}." categories_heading: Categorías categories_found: one: "1 categoría encontrada" @@ -88,6 +88,12 @@ es: instructions_5: Sin comas, sin símbolos de moneda y sin paréntesis en los números. title: Importa tus datos imports: + date_format: + heading: Formato de fecha + description: "El formato de fecha se detectó automáticamente desde tu archivo. Cámbialo si las fechas parecen incorrectas." + preview: "Primera fecha analizada" + error_title: "No se puede detectar el formato de fecha" + error_description: "Ninguno de los formatos de fecha compatibles pudo analizar las fechas de este archivo. Por favor, comprueba que el archivo contiene entradas de fecha válidas." steps: upload: Subir configure: Configurar diff --git a/config/locales/views/imports/fr.yml b/config/locales/views/imports/fr.yml index 3d9a2c578..4a34d031f 100644 --- a/config/locales/views/imports/fr.yml +++ b/config/locales/views/imports/fr.yml @@ -3,8 +3,8 @@ fr: import: qif_category_selections: show: - title: "Sélectionner les catégories et étiquettes" - description: "Choisissez les catégories et étiquettes de votre fichier QIF à importer dans Sure. Les éléments désélectionnés seront retirés des transactions correspondantes." + title: "Configurer et sélectionner" + description: "Vérifiez le format de date détecté, puis choisissez les catégories et étiquettes de votre fichier QIF à importer dans %{product_name}." categories_heading: Catégories categories_found: one: "1 catégorie trouvée" @@ -105,6 +105,12 @@ fr: hint_html: Téléversez le fichier all.ndjson de l'archive ZIP d'export de vos données ndjson_invalid: Le fichier doit être un NDJSON valide avec au moins un enregistrement imports: + date_format: + heading: Format de date + description: "Le format de date a été détecté automatiquement depuis votre fichier. Modifiez-le si les dates semblent incorrectes." + preview: "Première date analysée" + error_title: "Impossible de détecter le format de date" + error_description: "Aucun des formats de date pris en charge n'a pu analyser les dates dans ce fichier. Veuillez vérifier que le fichier contient des entrées de date valides." steps: upload: Téléverser configure: Configurer diff --git a/test/models/qif_import_test.rb b/test/models/qif_import_test.rb index c3e55dab8..e370b42be 100644 --- a/test/models/qif_import_test.rb +++ b/test/models/qif_import_test.rb @@ -851,4 +851,205 @@ class QifImportTest < ActiveSupport::TestCase refute import.will_adjust_opening_anchor? end + + # ── QifParser: normalize_qif_date ────────────────────────────────────────── + + test "normalize_qif_date converts apostrophe 2-digit year" do + assert_equal "6/4/2020", QifParser.send(:normalize_qif_date, "6/ 4'20") + end + + test "normalize_qif_date converts apostrophe 4-digit year" do + assert_equal "6/4/2020", QifParser.send(:normalize_qif_date, "6/ 4'2020") + end + + test "normalize_qif_date handles dot-separated dates" do + assert_equal "04.06.2020", QifParser.send(:normalize_qif_date, "04.06.2020") + end + + test "normalize_qif_date handles dot with apostrophe year" do + assert_equal "04.06.2020", QifParser.send(:normalize_qif_date, "04.06'20") + end + + test "normalize_qif_date handles dash-separated dates" do + assert_equal "2020-06-04", QifParser.send(:normalize_qif_date, "2020-06-04") + end + + test "normalize_qif_date returns nil for blank input" do + assert_nil QifParser.send(:normalize_qif_date, nil) + assert_nil QifParser.send(:normalize_qif_date, "") + end + + # ── QifParser: parse_qif_date with different formats ─────────────────────── + + test "parse_qif_date parses US format (MM/DD/YYYY)" do + assert_equal "2020-06-04", QifParser.send(:parse_qif_date, "6/ 4'20", date_format: "%m/%d/%Y") + end + + test "parse_qif_date parses European slash format (DD/MM/YYYY)" do + # 4/ 6'20 → day=4, month=6 → June 4th + assert_equal "2020-06-04", QifParser.send(:parse_qif_date, "4/ 6'20", date_format: "%d/%m/%Y") + end + + test "parse_qif_date parses European dot format (DD.MM.YYYY)" do + assert_equal "2020-06-04", QifParser.send(:parse_qif_date, "04.06.2020", date_format: "%d.%m.%Y") + end + + test "parse_qif_date parses ISO format (YYYY-MM-DD)" do + assert_equal "2020-06-04", QifParser.send(:parse_qif_date, "2020-06-04", date_format: "%Y-%m-%d") + end + + test "parse_qif_date returns nil for invalid date" do + assert_nil QifParser.send(:parse_qif_date, "13/32/2020", date_format: "%m/%d/%Y") + end + + # ── QifParser: extract_raw_dates ─────────────────────────────────────────── + + test "extract_raw_dates returns normalized date strings from D-fields" do + dates = QifParser.extract_raw_dates(SAMPLE_QIF) + assert_includes dates, "6/4/2020" + assert_includes dates, "3/29/2021" + assert_includes dates, "10/1/2020" + end + + test "extract_raw_dates returns empty for blank content" do + assert_empty QifParser.extract_raw_dates(nil) + assert_empty QifParser.extract_raw_dates("") + end + + # ── QifParser: parse with European date format ───────────────────────────── + + EUROPEAN_QIF = <<~QIF + !Type:Bank + D04/06/2020 + U-99.00 + T-99.00 + PMerchant A + ^ + D29/03/2021 + U-50.00 + T-50.00 + PMerchant B + ^ + QIF + + test "parse with DD/MM/YYYY format parses dates correctly" do + transactions = QifParser.parse(EUROPEAN_QIF, date_format: "%d/%m/%Y") + assert_equal "2020-06-04", transactions[0].date + assert_equal "2021-03-29", transactions[1].date + end + + # ── Import.detect_date_format ────────────────────────────────────────────── + + test "detect_date_format identifies US slash format" do + samples = %w[6/4/2020 3/29/2021 10/1/2020] + # 3/29 cannot be DD/MM (month 29 invalid), so must be MM/DD + assert_equal "%m/%d/%Y", Import.detect_date_format(samples) + end + + test "detect_date_format identifies European slash format" do + samples = %w[04/06/2020 29/03/2021 01/10/2020] + # 29/03 cannot be MM/DD (month 29 invalid), so must be DD/MM + assert_equal "%d/%m/%Y", Import.detect_date_format(samples) + end + + test "detect_date_format identifies European dot format" do + samples = %w[04.06.2020 29.03.2021 01.10.2020] + assert_equal "%d.%m.%Y", Import.detect_date_format(samples) + end + + test "detect_date_format identifies ISO format" do + samples = %w[2020-06-04 2021-03-29 2020-10-01] + assert_equal "%Y-%m-%d", Import.detect_date_format(samples) + end + + test "detect_date_format returns fallback for blank samples" do + assert_equal "%Y-%m-%d", Import.detect_date_format([]) + assert_equal "%Y-%m-%d", Import.detect_date_format(nil) + end + + test "detect_date_format returns fallback when no format matches" do + samples = %w[not-a-date garbage] + assert_equal "%Y-%m-%d", Import.detect_date_format(samples) + end + + # ── QifImport: auto-detection integration ────────────────────────────────── + + test "generate_rows_from_csv auto-detects US date format" do + @import.update!(raw_file_str: SAMPLE_QIF) + @import.generate_rows_from_csv + + assert_equal "%m/%d/%Y", @import.reload.qif_date_format + row = @import.rows.find_by(name: "Merchant A") + assert_equal "2020-06-04", row.date + end + + EUROPEAN_BANK_QIF = <<~QIF + !Type:Bank + D13/01/2024 + U-100.00 + T-100.00 + PEuropean Store + ^ + D25/12/2023 + U-50.00 + T-50.00 + PChristmas Shop + ^ + QIF + + test "generate_rows_from_csv auto-detects European DD/MM/YYYY format" do + @import.update!(raw_file_str: EUROPEAN_BANK_QIF) + @import.generate_rows_from_csv + + assert_equal "%d/%m/%Y", @import.reload.qif_date_format + row = @import.rows.find_by(name: "European Store") + assert_equal "2024-01-13", row.date + end + + test "generate_rows_from_csv respects manually set qif_date_format" do + @import.update!(raw_file_str: EUROPEAN_BANK_QIF) + @import.qif_date_format = "%d/%m/%Y" + @import.save!(validate: false) + @import.generate_rows_from_csv + + # Should not re-detect since qif_date_format is already set + assert_equal "%d/%m/%Y", @import.reload.qif_date_format + end + + # ── QifParser: try_parse_date ─────────────────────────────────────────────── + + test "try_parse_date returns ISO date for valid format" do + assert_equal "2020-06-04", QifParser.try_parse_date("6/ 4'20", date_format: "%m/%d/%Y") + end + + test "try_parse_date returns nil for incompatible format" do + assert_nil QifParser.try_parse_date("2020-06-04", date_format: "%d.%m.%Y") + end + + # ── QifImport: valid_date_formats_with_preview ────────────────────────────── + + test "valid_date_formats_with_preview excludes formats that cannot parse the file dates" do + @import.update!(raw_file_str: EUROPEAN_BANK_QIF) + formats = @import.valid_date_formats_with_preview + + format_strs = formats.map { |f| f[:format] } + + # DD/MM/YYYY should be valid (13/01/2024) + assert_includes format_strs, "%d/%m/%Y" + + # MM/DD/YYYY should be excluded (month 13 is invalid) + assert_not_includes format_strs, "%m/%d/%Y" + + # Each valid format should have a preview date + formats.each do |f| + assert_not_nil f[:preview], "Expected preview for #{f[:label]}" + end + end + + test "valid_date_formats_with_preview returns empty array when no raw dates" do + @import.update!(raw_file_str: "") + formats = @import.valid_date_formats_with_preview + + assert_empty formats + end end