From 2595885eb7f8a159fd62fc47e118e7159ce00595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Mon, 23 Mar 2026 14:27:41 +0100 Subject: [PATCH] Full `.ndjson` import / reorganize UI with Financial Tools / Raw Data tabs (#1208) * Reorganize import UI with Financial Tools / Raw Data tabs Split the flat list of import sources into two tabbed sections using DS::Tabs: "Financial Tools" (Mint, Quicken/QIF, YNAB coming soon) and "Raw Data" (transactions, investments, accounts, categories, rules, documents). This prepares for adding more tool-specific importers without cluttering the list. https://claude.ai/code/session_01BM4SBWNhATqoKTEvy3qTS3 * Fix import controller test to account for YNAB coming soon entry The new YNAB "coming soon" disabled entry adds a 5th aria-disabled element to the import dialog. https://claude.ai/code/session_01BM4SBWNhATqoKTEvy3qTS3 * Fix system tests to click Raw Data tab before selecting import type Transaction, trade, and account imports are now under the Raw Data tab and need an explicit tab click before the buttons are visible. https://claude.ai/code/session_01BM4SBWNhATqoKTEvy3qTS3 * feat: Add bulk import for NDJSON export files Implements an import flow that accepts the full all.ndjson file from data exports, allowing users to restore their complete data including: - Accounts with accountable types - Categories with parent relationships - Tags and merchants - Transactions with category, merchant, and tag references - Trades with securities - Valuations - Budgets and budget categories - Rules with conditions and actions (including compound conditions) Key changes: - Add BulkImport model extending Import base class - Add Family::DataImporter to handle NDJSON parsing and import logic - Update imports controller and views to support NDJSON workflow - Skip configuration/mapping steps for structured NDJSON imports - Add i18n translations for bulk import UI - Add tests for BulkImport and DataImporter * fix: Fix category import and test query issues - Add default lucide_icon ("shapes") for categories when not provided - Fix valuation test to use proper ActiveRecord joins syntax * Linter errors * fix: Add default color for tags when not provided in import * fix: Add default kind for transactions when not provided in import * Fix test * Fix tests * Fix remaining merge conflicts from PR 766 cherry-pick Resolve conflict markers in test fixtures and clean up BulkImport entry in new.html.erb to use the _import_option partial consistently. https://claude.ai/code/session_01BM4SBWNhATqoKTEvy3qTS3 * Import Sure `.ndjson` * Remove `.ndjson` import from raw data * Fix support for Sure "bulk" import from old branch * Linter * Fix CI test * Fix more CI tests * Fix tests * Fix tests / move PDF import to first tab * Remove redundant title --------- Co-authored-by: Claude --- app/controllers/import/uploads_controller.rb | 66 +- app/controllers/imports_controller.rb | 40 ++ app/helpers/imports_helper.rb | 7 +- app/models/family/data_importer.rb | 474 +++++++++++++++ app/models/import.rb | 2 +- app/models/sure_import.rb | 132 ++++ app/views/import/confirms/show.html.erb | 79 ++- app/views/import/uploads/show.html.erb | 44 +- app/views/imports/_nav.html.erb | 7 +- app/views/imports/_ready.html.erb | 43 +- app/views/imports/new.html.erb | 304 ++++++---- config/locales/views/imports/ca.yml | 5 + config/locales/views/imports/de.yml | 5 + config/locales/views/imports/en.yml | 58 +- config/locales/views/imports/es.yml | 5 + config/locales/views/imports/fr.yml | 22 + config/locales/views/imports/nb.yml | 7 +- config/locales/views/imports/nl.yml | 5 + config/locales/views/imports/pt-BR.yml | 5 + config/locales/views/imports/ro.yml | 5 + config/locales/views/imports/tr.yml | 7 +- config/locales/views/imports/zh-CN.yml | 5 + config/locales/views/imports/zh-TW.yml | 5 + .../api/v1/categories_controller_test.rb | 6 +- test/controllers/imports_controller_test.rb | 8 +- test/fixtures/categories.yml | 2 +- test/fixtures/imports.yml | 5 + test/fixtures/merchants.yml | 2 +- test/fixtures/tags.yml | 2 +- test/models/family/data_importer_test.rb | 574 ++++++++++++++++++ test/models/sure_import_test.rb | 187 ++++++ test/system/imports_test.rb | 5 + 32 files changed, 1960 insertions(+), 163 deletions(-) create mode 100644 app/models/family/data_importer.rb create mode 100644 app/models/sure_import.rb create mode 100644 test/models/family/data_importer_test.rb create mode 100644 test/models/sure_import_test.rb diff --git a/app/controllers/import/uploads_controller.rb b/app/controllers/import/uploads_controller.rb index fec74b5bc..b7c1b1bbb 100644 --- a/app/controllers/import/uploads_controller.rb +++ b/app/controllers/import/uploads_controller.rb @@ -16,6 +16,8 @@ class Import::UploadsController < ApplicationController def update if @import.is_a?(QifImport) handle_qif_upload + elsif @import.is_a?(SureImport) + update_sure_import_upload elsif csv_valid?(csv_str) @import.account = Current.family.accounts.find_by(id: import_account_id) @import.assign_attributes(raw_file_str: csv_str, col_sep: upload_params[:col_sep]) @@ -23,13 +25,54 @@ class Import::UploadsController < ApplicationController redirect_to import_configuration_path(@import, template_hint: true), notice: "CSV uploaded successfully." else - flash.now[:alert] = "Must be valid CSV with headers and at least one row of data" - - render :show, status: :unprocessable_entity + update_csv_import end end private + def update_csv_import + if csv_valid?(csv_str) + @import.account = Current.family.accounts.find_by(id: params.dig(:import, :account_id)) + @import.assign_attributes(raw_file_str: csv_str, col_sep: upload_params[:col_sep]) + @import.save!(validate: false) + + redirect_to import_configuration_path(@import, template_hint: true), notice: t("imports.create.csv_uploaded") + else + flash.now[:alert] = t("import.uploads.show.csv_invalid", default: "Must be valid CSV with headers and at least one row of data") + + render :show, status: :unprocessable_entity + end + end + + def update_sure_import_upload + uploaded = upload_params[:ndjson_file] + unless uploaded.present? + flash.now[:alert] = t("import.uploads.sure_import.ndjson_invalid", default: "Must be valid NDJSON with at least one record") + render :show, status: :unprocessable_entity + return + end + + if uploaded.size > SureImport::MAX_NDJSON_SIZE + flash.now[:alert] = t("imports.create.file_too_large", max_size: SureImport::MAX_NDJSON_SIZE / 1.megabyte) + render :show, status: :unprocessable_entity + return + end + + content = uploaded.read + uploaded.rewind + + if ndjson_valid?(content) + uploaded.rewind + @import.ndjson_file.attach(uploaded) + @import.sync_ndjson_rows_count! + redirect_to import_path(@import), notice: t("imports.create.ndjson_uploaded") + else + flash.now[:alert] = t("import.uploads.sure_import.ndjson_invalid", default: "Must be valid NDJSON with at least one record") + + render :show, status: :unprocessable_entity + end + end + def set_import @import = Current.family.imports.find(params[:import_id]) end @@ -71,8 +114,23 @@ class Import::UploadsController < ApplicationController end end + def ndjson_valid?(str) + return false if str.blank? + + # Check at least first line is valid NDJSON + first_line = str.lines.first&.strip + return false if first_line.blank? + + begin + record = JSON.parse(first_line) + record.key?("type") && record.key?("data") + rescue JSON::ParserError + false + end + end + def upload_params - params.require(:import).permit(:raw_file_str, :import_file, :col_sep) + params.require(:import).permit(:raw_file_str, :import_file, :ndjson_file, :col_sep) end def import_account_id diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 046e43632..d8f581c1b 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -49,6 +49,11 @@ class ImportsController < ApplicationController return end + if file.present? && sure_import_request? + create_sure_import(file) + return + end + # Handle PDF file uploads - process with AI if file.present? && Import::ALLOWED_PDF_MIME_TYPES.include?(file.content_type) unless valid_pdf_file?(file) @@ -85,6 +90,7 @@ class ImportsController < ApplicationController # Stream reading is not fully applicable here as we store the raw string in the DB, # but we have validated size beforehand to prevent memory exhaustion from massive files. import.update!(raw_file_str: file.read) + redirect_to import_configuration_path(import), notice: t("imports.create.csv_uploaded") else redirect_to import_upload_path(import) @@ -200,6 +206,40 @@ class ImportsController < ApplicationController params.dig(:import, :type) == "DocumentImport" end + def sure_import_request? + params.dig(:import, :type) == "SureImport" + end + + def create_sure_import(file) + if file.size > SureImport::MAX_NDJSON_SIZE + redirect_to new_import_path, alert: t("imports.create.file_too_large", max_size: SureImport::MAX_NDJSON_SIZE / 1.megabyte) + return + end + + ext = File.extname(file.original_filename.to_s).downcase + unless ext.in?(%w[.ndjson .json]) + redirect_to new_import_path, alert: t("imports.create.invalid_ndjson_file_type") + return + end + + content = file.read + file.rewind + unless SureImport.valid_ndjson_first_line?(content) + redirect_to new_import_path, alert: t("imports.create.invalid_ndjson_file_type") + return + end + + import = Current.family.imports.create!(type: "SureImport") + import.ndjson_file.attach( + io: StringIO.new(content), + filename: file.original_filename, + content_type: file.content_type + ) + import.sync_ndjson_rows_count! + + redirect_to import_path(import), notice: t("imports.create.ndjson_uploaded") + end + def valid_pdf_file?(file) header = file.read(5) file.rewind diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb index 65ae6e43c..6cf680452 100644 --- a/app/helpers/imports_helper.rb +++ b/app/helpers/imports_helper.rb @@ -35,7 +35,12 @@ module ImportsHelper accounts: DryRunResource.new(label: "Accounts", icon: "layers", text_class: "text-orange-500", bg_class: "bg-orange-500/5"), categories: DryRunResource.new(label: "Categories", icon: "shapes", text_class: "text-blue-500", bg_class: "bg-blue-500/5"), tags: DryRunResource.new(label: "Tags", icon: "tags", text_class: "text-violet-500", bg_class: "bg-violet-500/5"), - rules: DryRunResource.new(label: "Rules", icon: "workflow", text_class: "text-green-500", bg_class: "bg-green-500/5") + rules: DryRunResource.new(label: "Rules", icon: "workflow", text_class: "text-green-500", bg_class: "bg-green-500/5"), + merchants: DryRunResource.new(label: "Merchants", icon: "store", text_class: "text-amber-500", bg_class: "bg-amber-500/5"), + trades: DryRunResource.new(label: "Trades", icon: "arrow-left-right", text_class: "text-emerald-500", bg_class: "bg-emerald-500/5"), + valuations: DryRunResource.new(label: "Valuations", icon: "trending-up", text_class: "text-pink-500", bg_class: "bg-pink-500/5"), + budgets: DryRunResource.new(label: "Budgets", icon: "wallet", text_class: "text-indigo-500", bg_class: "bg-indigo-500/5"), + budget_categories: DryRunResource.new(label: "Budget Categories", icon: "pie-chart", text_class: "text-teal-500", bg_class: "bg-teal-500/5") } map[key] diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb new file mode 100644 index 000000000..4b6f2339b --- /dev/null +++ b/app/models/family/data_importer.rb @@ -0,0 +1,474 @@ +class Family::DataImporter + SUPPORTED_TYPES = %w[Account Category Tag Merchant Transaction Trade Valuation Budget BudgetCategory Rule].freeze + ACCOUNTABLE_TYPES = Accountable::TYPES.freeze + + def initialize(family, ndjson_content) + @family = family + @ndjson_content = ndjson_content + @id_mappings = { + accounts: {}, + categories: {}, + tags: {}, + merchants: {}, + budgets: {}, + securities: {} + } + @created_accounts = [] + @created_entries = [] + end + + def import! + records = parse_ndjson + + Import.transaction do + # Import in dependency order + import_accounts(records["Account"] || []) + import_categories(records["Category"] || []) + import_tags(records["Tag"] || []) + import_merchants(records["Merchant"] || []) + import_transactions(records["Transaction"] || []) + import_trades(records["Trade"] || []) + import_valuations(records["Valuation"] || []) + import_budgets(records["Budget"] || []) + import_budget_categories(records["BudgetCategory"] || []) + import_rules(records["Rule"] || []) + end + + { accounts: @created_accounts, entries: @created_entries } + end + + private + + def parse_ndjson + records = Hash.new { |h, k| h[k] = [] } + + @ndjson_content.each_line do |line| + next if line.strip.empty? + + begin + record = JSON.parse(line) + type = record["type"] + next unless SUPPORTED_TYPES.include?(type) + + records[type] << record + rescue JSON::ParserError + # Skip invalid lines + end + end + + records + end + + def import_accounts(records) + records.each do |record| + data = record["data"] + old_id = data["id"] + accountable_data = data["accountable"] || {} + accountable_type = data["accountable_type"] + + # Skip if accountable type is not valid + next unless ACCOUNTABLE_TYPES.include?(accountable_type) + + # Build accountable + accountable_class = accountable_type.constantize + accountable = accountable_class.new + accountable.subtype = accountable_data["subtype"] if accountable.respond_to?(:subtype=) && accountable_data["subtype"] + + # Copy any other accountable attributes + safe_accountable_attrs = %w[subtype locked_attributes] + safe_accountable_attrs.each do |attr| + if accountable.respond_to?("#{attr}=") && accountable_data[attr].present? + accountable.send("#{attr}=", accountable_data[attr]) + end + end + + account = @family.accounts.build( + name: data["name"], + balance: data["balance"].to_d, + cash_balance: data["cash_balance"]&.to_d || data["balance"].to_d, + currency: data["currency"] || @family.currency, + accountable: accountable, + subtype: data["subtype"], + institution_name: data["institution_name"], + institution_domain: data["institution_domain"], + notes: data["notes"], + status: "active" + ) + + account.save! + + # Set opening balance if we have a historical balance + if data["balance"].present? + manager = Account::OpeningBalanceManager.new(account) + manager.set_opening_balance(balance: data["balance"].to_d) + end + + @id_mappings[:accounts][old_id] = account.id + @created_accounts << account + end + end + + def import_categories(records) + # First pass: create all categories without parent relationships + parent_mappings = {} + + records.each do |record| + data = record["data"] + old_id = data["id"] + parent_id = data["parent_id"] + + # Store parent relationship for second pass + parent_mappings[old_id] = parent_id if parent_id.present? + + category = @family.categories.build( + name: data["name"], + color: data["color"] || Category::UNCATEGORIZED_COLOR, + classification_unused: data["classification_unused"] || data["classification"] || "expense", + lucide_icon: data["lucide_icon"] || "shapes" + ) + + category.save! + @id_mappings[:categories][old_id] = category.id + end + + # Second pass: establish parent relationships + parent_mappings.each do |old_id, old_parent_id| + new_id = @id_mappings[:categories][old_id] + new_parent_id = @id_mappings[:categories][old_parent_id] + + next unless new_id && new_parent_id + + category = @family.categories.find(new_id) + category.update!(parent_id: new_parent_id) + end + end + + def import_tags(records) + records.each do |record| + data = record["data"] + old_id = data["id"] + + tag = @family.tags.build( + name: data["name"], + color: data["color"] || Tag::COLORS.sample + ) + + tag.save! + @id_mappings[:tags][old_id] = tag.id + end + end + + def import_merchants(records) + records.each do |record| + data = record["data"] + old_id = data["id"] + + merchant = @family.merchants.build( + name: data["name"], + color: data["color"], + logo_url: data["logo_url"] + ) + + merchant.save! + @id_mappings[:merchants][old_id] = merchant.id + end + end + + def import_transactions(records) + records.each do |record| + data = record["data"] + + # Map account ID + new_account_id = @id_mappings[:accounts][data["account_id"]] + next unless new_account_id + + account = @family.accounts.find(new_account_id) + + # Map category ID (optional) + new_category_id = nil + if data["category_id"].present? + new_category_id = @id_mappings[:categories][data["category_id"]] + end + + # Map merchant ID (optional) + new_merchant_id = nil + if data["merchant_id"].present? + new_merchant_id = @id_mappings[:merchants][data["merchant_id"]] + end + + # Map tag IDs (optional) + new_tag_ids = [] + if data["tag_ids"].present? + new_tag_ids = Array(data["tag_ids"]).map { |old_tag_id| @id_mappings[:tags][old_tag_id] }.compact + end + + transaction = Transaction.new( + category_id: new_category_id, + merchant_id: new_merchant_id, + kind: data["kind"] || "standard" + ) + + entry = Entry.new( + account: account, + date: Date.parse(data["date"].to_s), + amount: data["amount"].to_d, + name: data["name"] || "Imported transaction", + currency: data["currency"] || account.currency, + notes: data["notes"], + excluded: data["excluded"] || false, + entryable: transaction + ) + + entry.save! + + # Add tags through the tagging association + new_tag_ids.each do |tag_id| + transaction.taggings.create!(tag_id: tag_id) + end + + @created_entries << entry + end + end + + def import_trades(records) + records.each do |record| + data = record["data"] + + # Map account ID + new_account_id = @id_mappings[:accounts][data["account_id"]] + next unless new_account_id + + account = @family.accounts.find(new_account_id) + + # Resolve or create security + ticker = data["ticker"] + next unless ticker.present? + + security = find_or_create_security(ticker, data["currency"]) + + trade = Trade.new( + security: security, + qty: data["qty"].to_d, + price: data["price"].to_d, + currency: data["currency"] || account.currency + ) + + entry = Entry.new( + account: account, + date: Date.parse(data["date"].to_s), + amount: data["amount"].to_d, + name: "#{data["qty"].to_d >= 0 ? 'Buy' : 'Sell'} #{ticker}", + currency: data["currency"] || account.currency, + entryable: trade + ) + + entry.save! + @created_entries << entry + end + end + + def import_valuations(records) + records.each do |record| + data = record["data"] + + # Map account ID + new_account_id = @id_mappings[:accounts][data["account_id"]] + next unless new_account_id + + account = @family.accounts.find(new_account_id) + + valuation = Valuation.new + + entry = Entry.new( + account: account, + date: Date.parse(data["date"].to_s), + amount: data["amount"].to_d, + name: data["name"] || "Valuation", + currency: data["currency"] || account.currency, + entryable: valuation + ) + + entry.save! + @created_entries << entry + end + end + + def import_budgets(records) + records.each do |record| + data = record["data"] + old_id = data["id"] + + budget = @family.budgets.build( + start_date: Date.parse(data["start_date"].to_s), + end_date: Date.parse(data["end_date"].to_s), + budgeted_spending: data["budgeted_spending"]&.to_d, + expected_income: data["expected_income"]&.to_d, + currency: data["currency"] || @family.currency + ) + + budget.save! + @id_mappings[:budgets][old_id] = budget.id + end + end + + def import_budget_categories(records) + records.each do |record| + data = record["data"] + + # Map budget ID + new_budget_id = @id_mappings[:budgets][data["budget_id"]] + next unless new_budget_id + + # Map category ID + new_category_id = @id_mappings[:categories][data["category_id"]] + next unless new_category_id + + budget = @family.budgets.find(new_budget_id) + + budget_category = budget.budget_categories.build( + category_id: new_category_id, + budgeted_spending: data["budgeted_spending"].to_d, + currency: data["currency"] || budget.currency + ) + + budget_category.save! + end + end + + def import_rules(records) + records.each do |record| + data = record["data"] + + rule = @family.rules.build( + name: data["name"], + resource_type: data["resource_type"] || "transaction", + active: data["active"] || false, + effective_date: data["effective_date"].present? ? Date.parse(data["effective_date"].to_s) : nil + ) + + # Build conditions + (data["conditions"] || []).each do |condition_data| + build_rule_condition(rule, condition_data) + end + + # Build actions + (data["actions"] || []).each do |action_data| + build_rule_action(rule, action_data) + end + + rule.save! + end + end + + def build_rule_condition(rule, condition_data, parent: nil) + value = resolve_rule_condition_value(condition_data) + + condition = if parent + parent.sub_conditions.build( + condition_type: condition_data["condition_type"], + operator: condition_data["operator"], + value: value + ) + else + rule.conditions.build( + condition_type: condition_data["condition_type"], + operator: condition_data["operator"], + value: value + ) + end + + # Handle nested sub_conditions for compound conditions + (condition_data["sub_conditions"] || []).each do |sub_condition_data| + build_rule_condition(rule, sub_condition_data, parent: condition) + end + + condition + end + + def build_rule_action(rule, action_data) + value = resolve_rule_action_value(action_data) + + rule.actions.build( + action_type: action_data["action_type"], + value: value + ) + end + + def resolve_rule_condition_value(condition_data) + condition_type = condition_data["condition_type"] + value = condition_data["value"] + + return value unless value.present? + + # Map category names to IDs + if condition_type == "transaction_category" + category = @family.categories.find_by(name: value) + category ||= @family.categories.create!( + name: value, + color: Category::UNCATEGORIZED_COLOR, + classification_unused: "expense", + lucide_icon: "shapes" + ) + return category.id + end + + # Map merchant names to IDs + if condition_type == "transaction_merchant" + merchant = @family.merchants.find_by(name: value) + merchant ||= @family.merchants.create!(name: value) + return merchant.id + end + + value + end + + def resolve_rule_action_value(action_data) + action_type = action_data["action_type"] + value = action_data["value"] + + return value unless value.present? + + # Map category names to IDs + if action_type == "set_transaction_category" + category = @family.categories.find_by(name: value) + category ||= @family.categories.create!( + name: value, + color: Category::UNCATEGORIZED_COLOR, + classification_unused: "expense", + lucide_icon: "shapes" + ) + return category.id + end + + # Map merchant names to IDs + if action_type == "set_transaction_merchant" + merchant = @family.merchants.find_by(name: value) + merchant ||= @family.merchants.create!(name: value) + return merchant.id + end + + # Map tag names to IDs + if action_type == "set_transaction_tags" + tag = @family.tags.find_by(name: value) + tag ||= @family.tags.create!(name: value) + return tag.id + end + + value + end + + def find_or_create_security(ticker, currency) + # Check cache first + cache_key = "#{ticker}:#{currency}" + return @id_mappings[:securities][cache_key] if @id_mappings[:securities][cache_key] + + security = Security.find_by(ticker: ticker.upcase) + security ||= Security.create!( + ticker: ticker.upcase, + name: ticker.upcase + ) + + @id_mappings[:securities][cache_key] = security + security + end +end diff --git a/app/models/import.rb b/app/models/import.rb index 2f2a0043e..bc279b009 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -9,7 +9,7 @@ class Import < ApplicationRecord DOCUMENT_TYPES = %w[bank_statement credit_card_statement investment_statement financial_document contract other].freeze - TYPES = %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport PdfImport QifImport].freeze + TYPES = %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport PdfImport QifImport SureImport].freeze SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative] SEPARATORS = [ [ "Comma (,)", "," ], [ "Semicolon (;)", ";" ] ].freeze diff --git a/app/models/sure_import.rb b/app/models/sure_import.rb new file mode 100644 index 000000000..05d927c4d --- /dev/null +++ b/app/models/sure_import.rb @@ -0,0 +1,132 @@ +class SureImport < Import + MAX_NDJSON_SIZE = 10.megabytes + ALLOWED_NDJSON_CONTENT_TYPES = %w[ + application/x-ndjson + application/ndjson + application/json + application/octet-stream + text/plain + ].freeze + + has_one_attached :ndjson_file, dependent: :purge_later + + class << self + # Counts JSON lines by top-level "type" (used for dry-run summaries and row limits). + def ndjson_line_type_counts(content) + return {} unless content.present? + + counts = Hash.new(0) + content.each_line do |line| + next if line.strip.empty? + + begin + record = JSON.parse(line) + counts[record["type"]] += 1 if record["type"] + rescue JSON::ParserError + # Skip invalid lines + end + end + counts + end + + def dry_run_totals_from_ndjson(content) + counts = ndjson_line_type_counts(content) + { + accounts: counts["Account"] || 0, + categories: counts["Category"] || 0, + tags: counts["Tag"] || 0, + merchants: counts["Merchant"] || 0, + transactions: counts["Transaction"] || 0, + trades: counts["Trade"] || 0, + valuations: counts["Valuation"] || 0, + budgets: counts["Budget"] || 0, + budget_categories: counts["BudgetCategory"] || 0, + rules: counts["Rule"] || 0 + } + end + + def valid_ndjson_first_line?(str) + return false if str.blank? + + first_line = str.lines.first&.strip + return false if first_line.blank? + + begin + record = JSON.parse(first_line) + record.key?("type") && record.key?("data") + rescue JSON::ParserError + false + end + end + end + + def requires_csv_workflow? + false + end + + def column_keys + [] + end + + def required_column_keys + [] + end + + def mapping_steps + [] + end + + def csv_template + nil + end + + def dry_run + return {} unless uploaded? + + self.class.dry_run_totals_from_ndjson(ndjson_blob_string) + end + + def import! + importer = Family::DataImporter.new(family, ndjson_blob_string) + result = importer.import! + + result[:accounts].each { |account| accounts << account } + result[:entries].each { |entry| entries << entry } + end + + def uploaded? + return false unless ndjson_file.attached? + + self.class.valid_ndjson_first_line?(ndjson_blob_string) + end + + def configured? + uploaded? + end + + def cleaned? + configured? + end + + def publishable? + cleaned? && dry_run.values.sum.positive? + end + + def max_row_count + 100_000 + end + + # Row total for max-row enforcement (counts every parsed line with a "type", including unsupported types). + def sync_ndjson_rows_count! + return unless ndjson_file.attached? + + total = self.class.ndjson_line_type_counts(ndjson_blob_string).values.sum + update_column(:rows_count, total) + end + + private + + def ndjson_blob_string + ndjson_file.download.force_encoding(Encoding::UTF_8) + end +end diff --git a/app/views/import/confirms/show.html.erb b/app/views/import/confirms/show.html.erb index 8d3122ab2..e8fe1a47c 100644 --- a/app/views/import/confirms/show.html.erb +++ b/app/views/import/confirms/show.html.erb @@ -2,32 +2,65 @@ <%= render "imports/nav", import: @import %> <% end %> -<%= content_for :previous_path, import_clean_path(@import) %> +<%= content_for :previous_path, @import.is_a?(SureImport) ? imports_path : import_clean_path(@import) %> -<% step_idx = (params[:step] || "1").to_i - 1 %> -<% step_mapping_class = @import.mapping_steps[step_idx] %> +<% if @import.is_a?(SureImport) %> +
+
+

<%= t("import.confirms.sure_import.title") %>

+

<%= t("import.confirms.sure_import.description") %>

+
-
-
- <% @import.mapping_steps.each_with_index do |step_mapping_class, idx| %> - <% is_active = step_idx == idx %> - - <%= link_to url_for(step: idx + 1), class: "w-5 h-[3px] #{is_active ? 'bg-gray-900' : 'bg-gray-100'} rounded-xl hover:bg-gray-300 transition-colors duration-200" do %> - Step <%= idx + 1 %> +
+

<%= t("import.confirms.sure_import.summary") %>

+ <% dry_run = @import.dry_run %> + <% sure_summary_empty = dry_run.values.none?(&:positive?) %> + <% if sure_summary_empty %> +

<%= t("import.confirms.sure_import.empty_summary") %>

+ <% else %> +
    + <% dry_run.each do |key, count| %> + <% next if count.zero? %> +
  • + <%= key.to_s.humanize %> + <%= count %> +
    • + <% end %> +
<% end %> - <% end %> +
+ +
+ <%= button_to t("import.confirms.sure_import.publish_button"), publish_import_path(@import), method: :post, class: "btn btn--primary w-full", disabled: !@import.publishable? %> + <%= link_to t("import.confirms.sure_import.cancel"), imports_path, class: "btn btn--ghost w-full text-center" %> +
+
+<% else %> + <% step_idx = (params[:step] || "1").to_i - 1 %> + <% step_mapping_class = @import.mapping_steps[step_idx] %> + +
+
+ <% @import.mapping_steps.each_with_index do |step_mapping_class, idx| %> + <% is_active = step_idx == idx %> + + <%= link_to url_for(step: idx + 1), class: "w-5 h-[3px] #{is_active ? 'bg-gray-900' : 'bg-gray-100'} rounded-xl hover:bg-gray-300 transition-colors duration-200" do %> + Step <%= idx + 1 %> + <% end %> + <% end %> +
+ +
+

+ <%= t(".#{step_mapping_class.name.demodulize.underscore}_title", import_type: @import.type.underscore.humanize) %> +

+

+ <%= t(".#{step_mapping_class.name.demodulize.underscore}_description", import_type: @import.type.underscore.humanize, product_name: product_name) %> +

+
-
-

- <%= t(".#{step_mapping_class.name.demodulize.underscore}_title", import_type: @import.type.underscore.humanize) %> -

-

- <%= t(".#{step_mapping_class.name.demodulize.underscore}_description", import_type: @import.type.underscore.humanize, product_name: product_name) %> -

+
+ <%= render partial: "import/confirms/mappings", locals: { import: @import, mapping_class: step_mapping_class, step_idx: step_idx } %>
-
- -
- <%= render partial: "import/confirms/mappings", locals: { import: @import, mapping_class: step_mapping_class, step_idx: step_idx } %> -
+<% end %> diff --git a/app/views/import/uploads/show.html.erb b/app/views/import/uploads/show.html.erb index 2915703cf..bb2b64961 100644 --- a/app/views/import/uploads/show.html.erb +++ b/app/views/import/uploads/show.html.erb @@ -4,7 +4,49 @@ <%= content_for :previous_path, imports_path %> -<% if @import.is_a?(QifImport) %> +<% if @import.is_a?(SureImport) %> +
+ + <%= render "imports/drag_drop_overlay", title: t("import.uploads.sure_import.drop_title"), subtitle: t("import.uploads.sure_import.drop_subtitle") %> + +
+
+

<%= t("import.uploads.sure_import.title") %>

+

<%= t("import.uploads.sure_import.description") %>

+
+ + <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2", data: { drag_and_drop_import_target: "form" } do |form| %> +
+
+
+ <%= icon("database", size: "lg", class: "mb-4 mx-auto") %> +

+ <%= t("import.uploads.sure_import.browse") %> <%= t("import.uploads.sure_import.browse_hint") %> +

+
+ + + + <%= form.file_field :ndjson_file, class: "hidden", accept: ".ndjson,.json", "data-auto-submit-form-target": "auto", "data-file-upload-target": "input", "data-drag-and-drop-import-target": "input" %> +
+
+ + <%= form.submit t("import.uploads.sure_import.upload_button"), disabled: @import.complete? %> + <% end %> +
+ +
+ + <%= t("import.uploads.sure_import.hint_html") %> + +
+
+<% elsif @import.is_a?(QifImport) %> <%# ── QIF upload – fixed format, account required ── %>
diff --git a/app/views/imports/_nav.html.erb b/app/views/imports/_nav.html.erb index 5d56bce7b..8bb457cd6 100644 --- a/app/views/imports/_nav.html.erb +++ b/app/views/imports/_nav.html.erb @@ -1,6 +1,11 @@ <%# locals: (import:) %> -<% steps = if import.is_a?(PdfImport) +<% steps = if import.is_a?(SureImport) + [ + { name: t("imports.steps.upload", default: "Upload"), path: import_upload_path(import), is_complete: import.uploaded?, step_number: 1 }, + { name: t("imports.steps.confirm", default: "Confirm"), path: import_path(import), is_complete: import.complete?, step_number: 2 } + ] +elsif import.is_a?(PdfImport) # PDF imports have a simplified flow: Upload -> Confirm # Upload/Configure/Clean are always complete for processed PDF imports [ diff --git a/app/views/imports/_ready.html.erb b/app/views/imports/_ready.html.erb index 9ab7c0eb8..9e2e0236d 100644 --- a/app/views/imports/_ready.html.erb +++ b/app/views/imports/_ready.html.erb @@ -1,4 +1,7 @@ <%# locals: (import:) %> +<% dry_run = import.dry_run %> +<% resources_with_counts = dry_run.select { |_, count| count > 0 }.filter_map { |key, count| [dry_run_resource(key), count] if dry_run_resource(key) } %> +<% import_summary_empty = resources_with_counts.empty? %>

<%= t(".title") %>

@@ -8,32 +11,40 @@
-

item

-

count

+

<%= t(".summary_item_label") %>

+

<%= t(".summary_count_label") %>

- <% import.dry_run.each do |key, count| %> - <% resource = dry_run_resource(key) %> + <% if import_summary_empty %> +
+

<%= t(".empty_summary") %>

+
+ <% else %> + <% resources_with_counts.each_with_index do |(resource, count), index| %> +
+
+ <%= tag.div class: class_names(resource.bg_class, resource.text_class, "w-8 h-8 rounded-full flex justify-center items-center") do %> + <%= icon resource.icon, color: "current" %> + <% end %> -
-
- <%= tag.div class: class_names(resource.bg_class, resource.text_class, "w-8 h-8 rounded-full flex justify-center items-center") do %> - <%= icon resource.icon, color: "current" %> - <% end %> +

<%= resource.label %>

+
-

<%= resource.label %>

+

<%= count %>

-

<%= count %>

-
- - <% if key != import.dry_run.keys.last %> - <%= render "shared/ruler" %> + <% unless index == resources_with_counts.length - 1 %> + <%= render "shared/ruler" %> + <% end %> <% end %> <% end %>
- <%= render DS::Button.new(text: "Publish import", href: publish_import_path(import), full_width: true) %> + <% if import_summary_empty %> + <%= render DS::Button.new(text: t(".back_to_imports"), href: imports_path, variant: :secondary, full_width: true) %> + <% else %> + <%= render DS::Button.new(text: t(".publish_import"), href: publish_import_path(import), method: :post, full_width: true) %> + <% end %>
diff --git a/app/views/imports/new.html.erb b/app/views/imports/new.html.erb index 5fd42d76b..f0a41ecc6 100644 --- a/app/views/imports/new.html.erb +++ b/app/views/imports/new.html.erb @@ -5,11 +5,10 @@ <% has_accounts = Current.family.accounts.any? %> <% requires_account_message = t(".requires_account") %> -
-

<%= t(".sources") %>

-
    -
  • - <% if @pending_import.present? && (params[:type].nil? || params[:type] == @pending_import.type) %> + <% if @pending_import.present? && params[:type].nil? %> +
    +
      +
    • <%= link_to import_path(@pending_import), class: "flex items-center justify-between p-4 group cursor-pointer", data: { turbo: false } do %>
      @@ -26,109 +25,208 @@ <%= render "shared/ruler" %>
      • - <% end %> +
    +
    + <% end %> - <% if params[:type].nil? || params[:type] == "TransactionImport" %> - <%= render "imports/import_option", - type: "TransactionImport", - icon_name: "file-spreadsheet", - icon_bg_class: "bg-indigo-500/5", - icon_text_class: "text-indigo-500", - label: t(".import_transactions"), - enabled: has_accounts, - disabled_message: requires_account_message %> - <% end %> + <% + import_type = params[:type].presence || @pending_import&.type + active_tab = import_type.present? && !import_type.in?(%w[MintImport QifImport SureImport DocumentImport PdfImport]) ? "raw_data" : "financial_tools" + %> + <%= render DS::Tabs.new(active_tab: active_tab) do |tabs| %> + <% tabs.with_nav do |nav| %> + <% nav.with_btn(id: "financial_tools", label: t(".tab_financial_tools")) %> + <% nav.with_btn(id: "raw_data", label: t(".tab_raw_data")) %> + <% end %> - <% if params[:type].nil? || params[:type] == "TradeImport" %> - <%= render "imports/import_option", - type: "TradeImport", - icon_name: "square-percent", - icon_bg_class: "bg-yellow-500/5", - icon_text_class: "text-yellow-500", - label: t(".import_portfolio"), - enabled: has_accounts, - disabled_message: requires_account_message %> - <% end %> - - <% if params[:type].nil? || params[:type] == "AccountImport" %> - <%= render "imports/import_option", - type: "AccountImport", - icon_name: "building", - icon_bg_class: "bg-violet-500/5", - icon_text_class: "text-violet-500", - label: t(".import_accounts"), - enabled: true %> - <% end %> - - <% if params[:type].nil? || params[:type] == "CategoryImport" %> - <%= render "imports/import_option", - type: "CategoryImport", - icon_name: "shapes", - icon_bg_class: "bg-blue-500/5", - icon_text_class: "text-blue-500", - label: t(".import_categories"), - enabled: true %> - <% end %> - - <% if params[:type].nil? || params[:type] == "RuleImport" %> - <%= render "imports/import_option", - type: "RuleImport", - icon_name: "workflow", - icon_bg_class: "bg-green-500/5", - icon_text_class: "text-green-500", - label: t(".import_rules"), - enabled: true %> - <% end %> - - <% if params[:type].nil? || params[:type] == "MintImport" || params[:type] == "TransactionImport" %> - <%= render "imports/import_option", - type: "MintImport", - image: "mint-logo.jpeg", - label: t(".import_mint"), - enabled: has_accounts, - disabled_message: requires_account_message %> - <% end %> - - <% if params[:type].nil? || params[:type] == "QifImport" %> - <%= render "imports/import_option", - type: "QifImport", - icon_name: "file-clock", - icon_bg_class: "bg-teal-500/5", - icon_text_class: "text-teal-500", - label: t(".import_qif"), - enabled: has_accounts, - disabled_message: requires_account_message %> - <% end %> - - <% if (params[:type].nil? || params[:type].in?(%w[DocumentImport PdfImport])) && @document_upload_extensions.any? %> -
  • - <%= styled_form_with url: imports_path, scope: :import, multipart: true, class: "w-full" do |form| %> - <%= form.hidden_field :type, value: "DocumentImport" %> -
    • <% end %> - <%= render "shared/ruler" %> - - <% end %> -
-
+ <% if params[:type].nil? || params[:type] == "SureImport" %> +
  • + <%= styled_form_with url: imports_path, scope: :import, multipart: true, class: "w-full", data: { turbo: false } do |form| %> + <%= form.hidden_field :type, value: "SureImport" %> + + <% end %> + + <%= render "shared/ruler" %> +
    • + <% end %> + + <% if params[:type].nil? || params[:type] == "MintImport" || params[:type] == "TransactionImport" %> + <%= render "imports/import_option", + type: "MintImport", + image: "mint-logo.jpeg", + label: t(".import_mint"), + enabled: true %> + <% end %> + + <% if params[:type].nil? || params[:type] == "QifImport" %> + <%= render "imports/import_option", + type: "QifImport", + icon_name: "file-clock", + icon_bg_class: "bg-teal-500/5", + icon_text_class: "text-teal-500", + label: t(".import_qif"), + enabled: true %> + <% end %> + + <%= render "imports/import_option", + type: "TransactionImport", + icon_name: "bar-chart-2", + icon_bg_class: "bg-gray-500/5", + icon_text_class: "text-gray-400", + label: t(".import_ynab"), + enabled: false, + disabled_message: t(".coming_soon") %> + + <% if (params[:type].nil? || params[:type].in?(%w[DocumentImport PdfImport])) && @document_upload_extensions.any? %> +
  • + <%= styled_form_with url: imports_path, scope: :import, multipart: true, class: "w-full", data: { turbo: false } do |form| %> + <%= form.hidden_field :type, value: "DocumentImport" %> + + <% end %> + + <%= render "shared/ruler" %> +
    • + <% end %> + +
    + <% end %> + + <% tabs.with_panel(tab_id: "raw_data") do %> +
    +
      + <% if @pending_import.present? && params[:type].present? && !params[:type].in?(%w[MintImport QifImport SureImport DocumentImport PdfImport]) %> +
    • + <%= link_to import_path(@pending_import), class: "flex items-center justify-between p-4 group cursor-pointer", data: { turbo: false } do %> +
      +
      + + <%= icon("loader", color: "current") %> + +
      + + <%= t(".resume", type: @pending_import.type.titleize) %> + +
      + <%= icon("chevron-right") %> + <% end %> + + <%= render "shared/ruler" %> +
      • + <% end %> + + <% if params[:type].nil? || params[:type] == "TransactionImport" %> + <%= render "imports/import_option", + type: "TransactionImport", + icon_name: "file-spreadsheet", + icon_bg_class: "bg-indigo-500/5", + icon_text_class: "text-indigo-500", + label: t(".import_transactions"), + enabled: has_accounts, + disabled_message: requires_account_message %> + <% end %> + + <% if params[:type].nil? || params[:type] == "TradeImport" %> + <%= render "imports/import_option", + type: "TradeImport", + icon_name: "square-percent", + icon_bg_class: "bg-yellow-500/5", + icon_text_class: "text-yellow-500", + label: t(".import_portfolio"), + enabled: has_accounts, + disabled_message: requires_account_message %> + <% end %> + + <% if params[:type].nil? || params[:type] == "AccountImport" %> + <%= render "imports/import_option", + type: "AccountImport", + icon_name: "building", + icon_bg_class: "bg-violet-500/5", + icon_text_class: "text-violet-500", + label: t(".import_accounts"), + enabled: true %> + <% end %> + + <% if params[:type].nil? || params[:type] == "CategoryImport" %> + <%= render "imports/import_option", + type: "CategoryImport", + icon_name: "shapes", + icon_bg_class: "bg-blue-500/5", + icon_text_class: "text-blue-500", + label: t(".import_categories"), + enabled: true %> + <% end %> + + <% if params[:type].nil? || params[:type] == "RuleImport" %> + <%= render "imports/import_option", + type: "RuleImport", + icon_name: "workflow", + icon_bg_class: "bg-green-500/5", + icon_text_class: "text-green-500", + label: t(".import_rules"), + enabled: true %> + <% end %> +
    +
    + <% end %> + <% end %> <% end %> <% end %> diff --git a/config/locales/views/imports/ca.yml b/config/locales/views/imports/ca.yml index b6f188dd7..8de803977 100644 --- a/config/locales/views/imports/ca.yml +++ b/config/locales/views/imports/ca.yml @@ -110,3 +110,8 @@ ca: description: Aquí tens un resum dels nous elements que s'afegiran al teu compte un cop publiquis aquesta importació. title: Confirma les teves dades d'importació + summary_item_label: Element + summary_count_label: Quantitat + empty_summary: No s'han trobat registres importables en aquest fitxer. Pot estar buit, o les línies no coincideixen amb el format d'exportació esperat (cada línia ha de ser un objecte JSON amb les claus «type» i «data», amb tipus admesos per aquesta importació). + publish_import: Publicar la importació + back_to_imports: Tornar a les importacions diff --git a/config/locales/views/imports/de.yml b/config/locales/views/imports/de.yml index 0af711a79..263a2bdda 100644 --- a/config/locales/views/imports/de.yml +++ b/config/locales/views/imports/de.yml @@ -181,3 +181,8 @@ de: ready: description: Hier ist eine Zusammenfassung der neuen Elemente, die deinem Konto hinzugefügt werden, sobald du diesen Import veröffentlichst. title: Importdaten bestätigen + summary_item_label: Eintrag + summary_count_label: Anzahl + empty_summary: In dieser Datei wurden keine importierbaren Datensätze gefunden. Sie ist möglicherweise leer, oder die Zeilen entsprechen nicht dem erwarteten Exportformat (jede Zeile sollte ein JSON-Objekt mit den Schlüsseln „type“ und „data“ und unterstützten Typen sein). + publish_import: Import veröffentlichen + back_to_imports: Zurück zu Importen diff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.yml index 3a80af608..db1d792d2 100644 --- a/config/locales/views/imports/en.yml +++ b/config/locales/views/imports/en.yml @@ -57,6 +57,13 @@ en: date_format_label: Date format rows_to_skip_label: Skip first n rows confirms: + sure_import: + title: Confirm your import + description: Review the data that will be imported from your export file. + summary: Import summary + empty_summary: We could not find any importable records in this file. It may be empty, or the lines may not match the expected export format (each line should be a JSON object with "type" and "data" keys, using types this import supports). + publish_button: Start import + cancel: Cancel mappings: create_account: Create account csv_mapping_label: "%{mapping} in CSV" @@ -101,7 +108,28 @@ en: instructions_4: Columns marked with an asterisk (*) are required data. instructions_5: No commas, no currency symbols, and no parentheses in numbers. title: Import your data + sure_import: + title: Import from export + description: Upload the all.ndjson file from your data export to restore your accounts, transactions, categories, and more. + drop_title: Drop NDJSON to upload + drop_subtitle: Your file will be uploaded automatically + browse: Browse + browse_hint: to add your all.ndjson file here + upload_button: Upload NDJSON + hint_html: Upload the all.ndjson file from your data export ZIP + ndjson_invalid: Must be valid NDJSON with at least one record imports: + type_labels: + transaction_import: "Transaction import" + trade_import: "Trade import" + account_import: "Account import" + mint_import: "Mint import" + qif_import: "QIF import" + category_import: "Category import" + rule_import: "Rule import" + pdf_import: "PDF import" + document_import: "Document import" + sure_import: "Sure import" steps: upload: Upload configure: Configure @@ -120,6 +148,17 @@ en: status: Status actions: Actions row: + type_labels: + transaction_import: "Transaction" + trade_import: "Trade" + account_import: "Account" + mint_import: "Mint" + qif_import: "QIF" + category_import: "Category" + rule_import: "Rule" + pdf_import: "PDF" + document_import: "Document" + sure_import: "Sure" status: in_progress: In progress uploading: Processing rows @@ -134,8 +173,11 @@ en: view: View empty: No imports yet. new: - description: You can manually import various types of data via CSV or use one - of our import templates like Mint. + description: Import from a financial tool or upload raw data files. + tab_financial_tools: Financial Tools & Files + tab_raw_data: Raw Data + coming_soon: Coming soon + import_ynab: Import from YNAB import_accounts: Import accounts import_categories: Import categories import_mint: Import from Mint @@ -143,8 +185,10 @@ en: import_rules: Import rules import_transactions: Import transactions import_qif: Import from Quicken (QIF) + import_sure: Import from Sure + import_sure_description: Full-export .ndjson file import_file: Import document - import_file_description: AI-powered analysis for PDFs and searchable upload for other supported files + import_file_description: AI-powered analysis for PDFs and searchable file upload requires_account: Import accounts first to unlock this option. resume: Resume %{type} sources: Sources @@ -153,6 +197,7 @@ en: file_too_large: File is too large. Maximum size is %{max_size}MB. invalid_file_type: Invalid file type. Please upload a CSV file. csv_uploaded: CSV uploaded successfully. + ndjson_uploaded: NDJSON file uploaded successfully. pdf_too_large: PDF file is too large. Maximum size is %{max_size}MB. pdf_processing: Your PDF is being processed. You will receive an email when analysis is complete. invalid_pdf: The uploaded file is not a valid PDF. @@ -160,6 +205,8 @@ en: invalid_document_file_type: Invalid document file type for the active vector store. document_uploaded: Document uploaded successfully. document_upload_failed: We couldn't upload the document to the vector store. Please try again. + invalid_ndjson_file_type: Invalid file type or format. Please upload a valid .ndjson or .json export file. + ndjson_uploaded: NDJSON file uploaded successfully. document_provider_not_configured: No vector store is configured for document uploads. show: finalize_upload: Please finalize your file upload. @@ -168,6 +215,11 @@ en: description: Here's a summary of the new items that will be added to your account once you publish this import. title: Confirm your import data + summary_item_label: Item + summary_count_label: Count + empty_summary: We could not find any importable records in this file. It may be empty, or the lines may not match the expected export format (each line should be a JSON object with "type" and "data" keys, using types this import supports). + publish_import: Publish import + back_to_imports: Back to imports errors: custom_column_requires_inflow: "Custom column imports require an inflow column to be selected" document_types: diff --git a/config/locales/views/imports/es.yml b/config/locales/views/imports/es.yml index 7f67f6e0b..51fe0142c 100644 --- a/config/locales/views/imports/es.yml +++ b/config/locales/views/imports/es.yml @@ -152,6 +152,11 @@ es: ready: description: Aquí tienes un resumen de los nuevos elementos que se añadirán a tu cuenta una vez publiques esta importación. title: Confirma tus datos de importación + summary_item_label: Elemento + summary_count_label: Cantidad + empty_summary: No se han encontrado registros importables en este archivo. Puede estar vacío, o las líneas no coinciden con el formato de exportación esperado (cada línea debe ser un objeto JSON con las claves «type» y «data», usando tipos que admite esta importación). + publish_import: Publicar importación + back_to_imports: Volver a importaciones errors: custom_column_requires_inflow: "Las importaciones de columnas personalizadas requieren que se seleccione una columna de entrada de fondos (inflow)" document_types: diff --git a/config/locales/views/imports/fr.yml b/config/locales/views/imports/fr.yml index 39e0567e3..3d9a2c578 100644 --- a/config/locales/views/imports/fr.yml +++ b/config/locales/views/imports/fr.yml @@ -53,6 +53,13 @@ fr: date_format_label: Format de date rows_to_skip_label: Ignorer les n premières lignes confirms: + sure_import: + title: Confirmer votre importation + description: Vérifiez les données qui seront importées depuis votre fichier d'export. + summary: Résumé de l'importation + empty_summary: Aucun enregistrement importable n'a été trouvé dans ce fichier. Il est peut-être vide, ou les lignes ne correspondent pas au format d'export attendu (chaque ligne doit être un objet JSON avec les clés « type » et « data », pour des types pris en charge par cet import). + publish_button: Démarrer l'importation + cancel: Annuler mappings: create_account: Créer un compte csv_mapping_label: "%{mapping} dans le CSV" @@ -87,6 +94,16 @@ fr: instructions_4: Les colonnes marquées avec une étoile (*) sont des données requises. instructions_5: Pas de virgules, pas de symboles monétaires et pas de parenthèses dans les nombres. title: Importez vos données + sure_import: + title: Importer depuis l'export + description: Téléversez le fichier all.ndjson de votre export de données pour restaurer vos comptes, transactions, catégories et plus encore. + drop_title: Déposez le NDJSON pour téléverser + drop_subtitle: Votre fichier sera téléversé automatiquement + browse: Parcourir + browse_hint: pour ajouter votre fichier all.ndjson ici + upload_button: Téléverser le NDJSON + hint_html: Téléversez le fichier all.ndjson de l'archive ZIP d'export de vos données + ndjson_invalid: Le fichier doit être un NDJSON valide avec au moins un enregistrement imports: steps: upload: Téléverser @@ -182,3 +199,8 @@ fr: ready: description: Voici un résumé des nouveaux éléments qui seront ajoutés à votre compte une fois que vous aurez publié cette importation. title: Confirmez vos données d'importation + summary_item_label: Élément + summary_count_label: Nombre + empty_summary: Aucun enregistrement importable n'a été trouvé dans ce fichier. Il est peut-être vide, ou les lignes ne correspondent pas au format d'export attendu (chaque ligne doit être un objet JSON avec les clés « type » et « data », pour des types pris en charge par cet import). + publish_import: Publier l'importation + back_to_imports: Retour aux importations diff --git a/config/locales/views/imports/nb.yml b/config/locales/views/imports/nb.yml index b0544f12e..cebd1ddb0 100644 --- a/config/locales/views/imports/nb.yml +++ b/config/locales/views/imports/nb.yml @@ -94,4 +94,9 @@ nb: ready: description: Her er en oppsummering av de nye elementene som vil bli lagt til kontoen din når du publiserer denne importen. - title: Bekreft importdataene dine \ No newline at end of file + title: Bekreft importdataene dine + summary_item_label: Element + summary_count_label: Antall + empty_summary: Vi fant ingen poster som kan importeres i denne filen. Den kan være tom, eller linjene samsvarer ikke med det forventede eksportformatet (hver linje skal være et JSON-objekt med nøklene «type» og «data», med typer denne importen støtter). + publish_import: Publiser import + back_to_imports: Tilbake til importer \ No newline at end of file diff --git a/config/locales/views/imports/nl.yml b/config/locales/views/imports/nl.yml index 898d82c96..31a2d3678 100644 --- a/config/locales/views/imports/nl.yml +++ b/config/locales/views/imports/nl.yml @@ -93,3 +93,8 @@ nl: ready: description: Hier is een samenvatting van de nieuwe items die aan uw account worden toegevoegd zodra u deze import publiceert. title: Uw importgegevens bevestigen + summary_item_label: Item + summary_count_label: Aantal + empty_summary: Er zijn geen importeerbare records in dit bestand gevonden. Het bestand is mogelijk leeg, of de regels voldoen niet aan het verwachte exportformaat (elke regel moet een JSON-object zijn met de sleutels „type“ en „data“, met typen die deze import ondersteunt). + publish_import: Import publiceren + back_to_imports: Terug naar importen diff --git a/config/locales/views/imports/pt-BR.yml b/config/locales/views/imports/pt-BR.yml index 6183ac6bd..84edb7b83 100644 --- a/config/locales/views/imports/pt-BR.yml +++ b/config/locales/views/imports/pt-BR.yml @@ -101,3 +101,8 @@ pt-BR: description: Aqui está um resumo dos novos itens que serão adicionados à sua conta assim que você publicar esta importação. title: Confirmar seus dados de importação + summary_item_label: Item + summary_count_label: Quantidade + empty_summary: Não foi possível encontrar registros importáveis neste arquivo. Ele pode estar vazio, ou as linhas não correspondem ao formato de exportação esperado (cada linha deve ser um objeto JSON com as chaves «type» e «data», usando tipos suportados por esta importação). + publish_import: Publicar importação + back_to_imports: Voltar às importações diff --git a/config/locales/views/imports/ro.yml b/config/locales/views/imports/ro.yml index 76a74371f..bec1cf695 100644 --- a/config/locales/views/imports/ro.yml +++ b/config/locales/views/imports/ro.yml @@ -80,3 +80,8 @@ ro: ready: description: Iată un rezumat al elementelor noi care vor fi adăugate contului tău odată ce vei publica acest import. title: Confirmă datele importate + summary_item_label: Element + summary_count_label: Număr + empty_summary: Nu s-au găsit înregistrări importabile în acest fișier. Fișierul poate fi gol sau liniile nu respectă formatul de export așteptat (fiecare linie trebuie să fie un obiect JSON cu cheile „type” și „data”, folosind tipuri acceptate de acest import). + publish_import: Publică importul + back_to_imports: Înapoi la importuri diff --git a/config/locales/views/imports/tr.yml b/config/locales/views/imports/tr.yml index 1edec3ae2..8296d655c 100644 --- a/config/locales/views/imports/tr.yml +++ b/config/locales/views/imports/tr.yml @@ -79,4 +79,9 @@ tr: title: Yeni CSV İçe Aktarma ready: description: Bu içe aktarmayı yayınladığınızda hesabınıza eklenecek yeni öğelerin özeti aşağıdadır. - title: İçe aktarma verilerinizi onaylayın \ No newline at end of file + title: İçe aktarma verilerinizi onaylayın + summary_item_label: Öğe + summary_count_label: Adet + empty_summary: Bu dosyada içe aktarılabilir kayıt bulunamadı. Dosya boş olabilir veya satırlar beklenen dışa aktarma biçimiyle eşleşmiyor (her satır, bu içe aktarmanın desteklediği türlerle «type» ve «data» anahtarlarına sahip bir JSON nesnesi olmalıdır). + publish_import: İçe aktarmayı yayınla + back_to_imports: İçe aktarmalara dön \ No newline at end of file diff --git a/config/locales/views/imports/zh-CN.yml b/config/locales/views/imports/zh-CN.yml index 9ce64c768..8ea66765b 100644 --- a/config/locales/views/imports/zh-CN.yml +++ b/config/locales/views/imports/zh-CN.yml @@ -90,3 +90,8 @@ zh-CN: ready: description: 以下是发布导入后将添加到您账户的新项目摘要。 title: 确认导入数据 + summary_item_label: 项目 + summary_count_label: 数量 + empty_summary: 在此文件中未找到可导入的记录。文件可能为空，或各行不符合预期的导出格式（每行应为包含「type」和「data」键的 JSON 对象，且类型须为本导入支持的类型）。 + publish_import: 发布导入 + back_to_imports: 返回导入列表 diff --git a/config/locales/views/imports/zh-TW.yml b/config/locales/views/imports/zh-TW.yml index 25c7bd541..a7a47849b 100644 --- a/config/locales/views/imports/zh-TW.yml +++ b/config/locales/views/imports/zh-TW.yml @@ -90,3 +90,8 @@ zh-TW: ready: description: 以下是發佈此匯入後，將新增至您帳戶的項目摘要。 title: 確認您的匯入資料 + summary_item_label: 項目 + summary_count_label: 數量 + empty_summary: 在此檔案中找不到可匯入的記錄。檔案可能是空的，或各行不符合預期的匯出格式（每行應為包含「type」與「data」鍵的 JSON 物件，且類型須為此匯入支援的類型）。 + publish_import: 發佈匯入 + back_to_imports: 返回匯入列表 diff --git a/test/controllers/api/v1/categories_controller_test.rb b/test/controllers/api/v1/categories_controller_test.rb index d3a26fbbc..c60fcacc1 100644 --- a/test/controllers/api/v1/categories_controller_test.rb +++ b/test/controllers/api/v1/categories_controller_test.rb @@ -176,7 +176,11 @@ class Api::V1::CategoriesControllerTest < ActionDispatch::IntegrationTest end test "should not return category from another family" do - other_family_category = categories(:one) # belongs to :empty family + other_family_category = families(:empty).categories.create!( + name: "Other Family Category", + color: "#FF0000", + classification_unused: "expense" + ) get "/api/v1/categories/#{other_family_category.id}", params: {}, headers: { "Authorization" => "Bearer #{@access_token.token}" diff --git a/test/controllers/imports_controller_test.rb b/test/controllers/imports_controller_test.rb index 280ec2d41..dbea78d2d 100644 --- a/test/controllers/imports_controller_test.rb +++ b/test/controllers/imports_controller_test.rb @@ -32,10 +32,10 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest assert_select "button", text: "Import accounts" assert_select "button", text: "Import transactions", count: 0 assert_select "button", text: "Import investments", count: 0 - assert_select "button", text: "Import from Mint", count: 0 - assert_select "button", text: "Import from Quicken (QIF)", count: 0 - assert_select "span", text: "Import accounts first to unlock this option.", count: 4 - assert_select "div[aria-disabled=true]", count: 4 + assert_select "button", text: "Import from Mint", count: 1 + assert_select "button", text: "Import from Quicken (QIF)", count: 1 + assert_select "span", text: "Import accounts first to unlock this option.", count: 2 + assert_select "div[aria-disabled=true]", count: 3 end test "creates import" do diff --git a/test/fixtures/categories.yml b/test/fixtures/categories.yml index 958e450e3..fb5bd41c3 100644 --- a/test/fixtures/categories.yml +++ b/test/fixtures/categories.yml @@ -1,6 +1,6 @@ one: name: Test - family: empty + family: dylan_family income: name: Income diff --git a/test/fixtures/imports.yml b/test/fixtures/imports.yml index 964585593..a63f9a5de 100644 --- a/test/fixtures/imports.yml +++ b/test/fixtures/imports.yml @@ -45,3 +45,8 @@ pdf_with_rows: category: "Income" notes: "" rows_count: 2 + +sure: + family: dylan_family + type: SureImport + status: pending diff --git a/test/fixtures/merchants.yml b/test/fixtures/merchants.yml index 6ac64ef48..40b20c44d 100644 --- a/test/fixtures/merchants.yml +++ b/test/fixtures/merchants.yml @@ -1,7 +1,7 @@ one: type: FamilyMerchant name: Test - family: empty + family: dylan_family netflix: type: FamilyMerchant diff --git a/test/fixtures/tags.yml b/test/fixtures/tags.yml index 1c76d6cce..56d1c43b1 100644 --- a/test/fixtures/tags.yml +++ b/test/fixtures/tags.yml @@ -8,4 +8,4 @@ two: three: name: Test - family: empty \ No newline at end of file + family: dylan_family \ No newline at end of file diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb new file mode 100644 index 000000000..29f4f5bb1 --- /dev/null +++ b/test/models/family/data_importer_test.rb @@ -0,0 +1,574 @@ +require "test_helper" + +class Family::DataImporterTest < ActiveSupport::TestCase + setup do + @family = families(:empty) + end + + test "imports accounts with accountable data" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "old-account-1", + name: "Test Checking", + balance: "1500.00", + currency: "USD", + accountable_type: "Depository", + accountable: { subtype: "checking" } + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + result = importer.import! + + assert_equal 1, result[:accounts].count + account = result[:accounts].first + assert_equal "Test Checking", account.name + assert_equal 1500.0, account.balance.to_f + assert_equal "USD", account.currency + assert_equal "Depository", account.accountable_type + end + + test "imports categories with parent relationships" do + ndjson = build_ndjson([ + { + type: "Category", + data: { + id: "cat-parent", + name: "Shopping", + color: "#FF5733", + classification: "expense" + } + }, + { + type: "Category", + data: { + id: "cat-child", + name: "Groceries", + color: "#33FF57", + classification: "expense", + parent_id: "cat-parent" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + parent = @family.categories.find_by(name: "Shopping") + child = @family.categories.find_by(name: "Groceries") + + assert_not_nil parent + assert_not_nil child + assert_equal parent.id, child.parent_id + end + + test "imports tags" do + ndjson = build_ndjson([ + { + type: "Tag", + data: { + id: "tag-1", + name: "Important", + color: "#FF0000" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + tag = @family.tags.find_by(name: "Important") + assert_not_nil tag + assert_equal "#FF0000", tag.color + end + + test "imports merchants" do + ndjson = build_ndjson([ + { + type: "Merchant", + data: { + id: "merchant-1", + name: "Amazon" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + merchant = @family.merchants.find_by(name: "Amazon") + assert_not_nil merchant + end + + test "imports transactions with references" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Main Account", + balance: "5000", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Category", + data: { + id: "cat-1", + name: "Food", + color: "#FF0000", + classification: "expense" + } + }, + { + type: "Tag", + data: { + id: "tag-1", + name: "Essential" + } + }, + { + type: "Transaction", + data: { + id: "txn-1", + account_id: "acct-1", + date: "2024-01-15", + amount: "-50.00", + name: "Grocery Store", + currency: "USD", + category_id: "cat-1", + tag_ids: [ "tag-1" ], + notes: "Weekly groceries" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + result = importer.import! + + assert_equal 1, result[:entries].count + + transaction = @family.transactions.first + assert_not_nil transaction + assert_equal "Grocery Store", transaction.entry.name + assert_equal -50.0, transaction.entry.amount.to_f + assert_equal "Food", transaction.category.name + assert_equal 1, transaction.tags.count + assert_equal "Essential", transaction.tags.first.name + assert_equal "Weekly groceries", transaction.entry.notes + end + + test "imports trades with securities" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "inv-acct-1", + name: "Investment Account", + balance: "10000", + currency: "USD", + accountable_type: "Investment" + } + }, + { + type: "Trade", + data: { + id: "trade-1", + account_id: "inv-acct-1", + date: "2024-01-15", + ticker: "AAPL", + qty: "10", + price: "150.00", + amount: "-1500.00", + currency: "USD" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + result = importer.import! + + # Account + Opening balance + Trade entry + assert_equal 1, result[:entries].count + + trade = @family.trades.first + assert_not_nil trade + assert_equal "AAPL", trade.security.ticker + assert_equal 10.0, trade.qty.to_f + assert_equal 150.0, trade.price.to_f + end + + test "imports valuations" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "prop-acct-1", + name: "Property", + balance: "500000", + currency: "USD", + accountable_type: "Property" + } + }, + { + type: "Valuation", + data: { + id: "val-1", + account_id: "prop-acct-1", + date: "2024-06-15", + amount: "520000", + name: "Updated valuation", + currency: "USD" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + result = importer.import! + + assert_equal 1, result[:entries].count + + account = @family.accounts.find_by(name: "Property") + valuation = account.valuations.joins(:entry).find_by(entries: { name: "Updated valuation" }) + assert_not_nil valuation + assert_equal 520000.0, valuation.entry.amount.to_f + end + + test "imports budgets" do + ndjson = build_ndjson([ + { + type: "Budget", + data: { + id: "budget-1", + start_date: "2024-01-01", + end_date: "2024-01-31", + budgeted_spending: "3000.00", + expected_income: "5000.00", + currency: "USD" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + budget = @family.budgets.first + assert_not_nil budget + assert_equal Date.parse("2024-01-01"), budget.start_date + assert_equal Date.parse("2024-01-31"), budget.end_date + assert_equal 3000.0, budget.budgeted_spending.to_f + assert_equal 5000.0, budget.expected_income.to_f + end + + test "imports budget_categories" do + ndjson = build_ndjson([ + { + type: "Category", + data: { + id: "cat-groceries", + name: "Groceries", + color: "#00FF00", + classification: "expense" + } + }, + { + type: "Budget", + data: { + id: "budget-1", + start_date: "2024-01-01", + end_date: "2024-01-31", + budgeted_spending: "3000.00", + expected_income: "5000.00", + currency: "USD" + } + }, + { + type: "BudgetCategory", + data: { + id: "bc-1", + budget_id: "budget-1", + category_id: "cat-groceries", + budgeted_spending: "500.00", + currency: "USD" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + budget = @family.budgets.first + budget_category = budget.budget_categories.first + assert_not_nil budget_category + assert_equal "Groceries", budget_category.category.name + assert_equal 500.0, budget_category.budgeted_spending.to_f + end + + test "imports rules with conditions and actions" do + ndjson = build_ndjson([ + { + type: "Rule", + version: 1, + data: { + name: "Categorize Coffee", + resource_type: "transaction", + active: true, + conditions: [ + { + condition_type: "transaction_name", + operator: "like", + value: "starbucks" + } + ], + actions: [ + { + action_type: "set_transaction_category", + value: "Coffee" + } + ] + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + rule = @family.rules.find_by(name: "Categorize Coffee") + assert_not_nil rule + assert rule.active + assert_equal "transaction", rule.resource_type + + assert_equal 1, rule.conditions.count + condition = rule.conditions.first + assert_equal "transaction_name", condition.condition_type + assert_equal "like", condition.operator + assert_equal "starbucks", condition.value + + assert_equal 1, rule.actions.count + action = rule.actions.first + assert_equal "set_transaction_category", action.action_type + + # Category should be created + category = @family.categories.find_by(name: "Coffee") + assert_not_nil category + assert_equal category.id, action.value + end + + test "imports rules with compound conditions" do + ndjson = build_ndjson([ + { + type: "Rule", + version: 1, + data: { + name: "Compound Rule", + resource_type: "transaction", + active: true, + conditions: [ + { + condition_type: "compound", + operator: "or", + sub_conditions: [ + { + condition_type: "transaction_name", + operator: "like", + value: "walmart" + }, + { + condition_type: "transaction_name", + operator: "like", + value: "target" + } + ] + } + ], + actions: [ + { + action_type: "auto_categorize" + } + ] + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + rule = @family.rules.find_by(name: "Compound Rule") + assert_not_nil rule + + parent_condition = rule.conditions.first + assert_equal "compound", parent_condition.condition_type + assert_equal "or", parent_condition.operator + assert_equal 2, parent_condition.sub_conditions.count + end + + test "skips invalid records gracefully" do + ndjson = "not valid json\n" + build_ndjson([ + { + type: "Account", + data: { + id: "valid-acct", + name: "Valid Account", + balance: "1000", + currency: "USD", + accountable_type: "Depository" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + result = importer.import! + + assert_equal 1, result[:accounts].count + assert_equal "Valid Account", result[:accounts].first.name + end + + test "skips unsupported record types" do + ndjson = build_ndjson([ + { + type: "UnsupportedType", + data: { id: "unknown" } + }, + { + type: "Account", + data: { + id: "valid-acct", + name: "Known Account", + balance: "1000", + currency: "USD", + accountable_type: "Depository" + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + result = importer.import! + + assert_equal 1, result[:accounts].count + end + + test "full import scenario with all entity types" do + ndjson = build_ndjson([ + # Account + { + type: "Account", + data: { + id: "acct-main", + name: "Main Checking", + balance: "5000", + currency: "USD", + accountable_type: "Depository" + } + }, + # Category + { + type: "Category", + data: { + id: "cat-food", + name: "Food", + color: "#FF5733", + classification: "expense" + } + }, + # Tag + { + type: "Tag", + data: { + id: "tag-weekly", + name: "Weekly" + } + }, + # Merchant + { + type: "Merchant", + data: { + id: "merchant-1", + name: "Local Grocery" + } + }, + # Transaction + { + type: "Transaction", + data: { + id: "txn-1", + account_id: "acct-main", + date: "2024-01-15", + amount: "-75.50", + name: "Weekly groceries", + currency: "USD", + category_id: "cat-food", + merchant_id: "merchant-1", + tag_ids: [ "tag-weekly" ] + } + }, + # Budget + { + type: "Budget", + data: { + id: "budget-jan", + start_date: "2024-01-01", + end_date: "2024-01-31", + budgeted_spending: "2000", + expected_income: "4000", + currency: "USD" + } + }, + # BudgetCategory + { + type: "BudgetCategory", + data: { + id: "bc-food", + budget_id: "budget-jan", + category_id: "cat-food", + budgeted_spending: "500", + currency: "USD" + } + }, + # Rule + { + type: "Rule", + version: 1, + data: { + name: "Auto-tag groceries", + resource_type: "transaction", + active: true, + conditions: [ + { condition_type: "transaction_name", operator: "like", value: "grocery" } + ], + actions: [ + { action_type: "set_transaction_tags", value: "Weekly" } + ] + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + result = importer.import! + + # Verify all entities were created + assert_equal 1, result[:accounts].count + assert_equal 1, @family.categories.count + assert_equal 1, @family.tags.count + assert_equal 1, @family.merchants.count + assert_equal 1, @family.transactions.count + assert_equal 1, @family.budgets.count + assert_equal 1, @family.budget_categories.count + assert_equal 1, @family.rules.count + + # Verify relationships + transaction = @family.transactions.first + assert_equal "Food", transaction.category.name + assert_equal "Local Grocery", transaction.merchant.name + assert_equal "Weekly", transaction.tags.first.name + end + + private + + def build_ndjson(records) + records.map(&:to_json).join("\n") + end +end diff --git a/test/models/sure_import_test.rb b/test/models/sure_import_test.rb new file mode 100644 index 000000000..87d70db8c --- /dev/null +++ b/test/models/sure_import_test.rb @@ -0,0 +1,187 @@ +require "test_helper" + +class SureImportTest < ActiveSupport::TestCase + include ActiveJob::TestHelper + + setup do + @family = families(:dylan_family) + @import = @family.imports.create!(type: "SureImport") + end + + test "dry_run reflects attached ndjson content" do + ndjson = [ + { type: "Account", data: { id: "uuid-1", name: "Test", balance: "1000", currency: "USD", accountable_type: "Depository" } }, + { type: "Transaction", data: { id: "uuid-2" } } + ].map(&:to_json).join("\n") + + attach_ndjson(ndjson) + + dry_run = @import.dry_run + + assert_equal 1, dry_run[:accounts] + assert_equal 1, dry_run[:transactions] + end + + test "publishable? is false when attached file has no supported records" do + ndjson = { type: "UnknownType", data: {} }.to_json + attach_ndjson(ndjson) + + assert @import.uploaded? + assert_not @import.publishable? + end + + test "column_keys required_column_keys and mapping_steps are empty" do + assert_equal [], @import.column_keys + assert_equal [], @import.required_column_keys + assert_equal [], @import.mapping_steps + end + + test "max_row_count is higher than standard imports" do + assert_equal 100_000, @import.max_row_count + end + + test "csv_template returns nil" do + assert_nil @import.csv_template + end + + test "uploaded? returns false without ndjson attachment" do + assert_not @import.uploaded? + end + + test "uploaded? returns true with valid ndjson attachment" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { id: "uuid-1", name: "Test", balance: "1000", currency: "USD", accountable_type: "Depository" } } + ])) + + assert @import.uploaded? + end + + test "uploaded? returns false with invalid ndjson attachment" do + attach_ndjson("not valid json") + + assert_not @import.uploaded? + end + + test "configured? and cleaned? follow uploaded?" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { id: "uuid-1", name: "Test", balance: "1000", currency: "USD", accountable_type: "Depository" } } + ])) + + assert @import.configured? + assert @import.cleaned? + end + + test "publishable? returns true when uploaded and valid" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { id: "uuid-1", name: "Test", balance: "1000", currency: "USD", accountable_type: "Depository" } } + ])) + + assert @import.publishable? + end + + test "dry_run returns counts by type" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { id: "uuid-1" } }, + { type: "Account", data: { id: "uuid-2" } }, + { type: "Category", data: { id: "uuid-3" } }, + { type: "Transaction", data: { id: "uuid-4" } }, + { type: "Transaction", data: { id: "uuid-5" } }, + { type: "Transaction", data: { id: "uuid-6" } } + ])) + + dry_run = @import.dry_run + + assert_equal 2, dry_run[:accounts] + assert_equal 1, dry_run[:categories] + assert_equal 3, dry_run[:transactions] + assert_equal 0, dry_run[:tags] + end + + test "sync_ndjson_rows_count! sets total row count" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { id: "uuid-1" } }, + { type: "Category", data: { id: "uuid-2" } }, + { type: "Transaction", data: { id: "uuid-3" } } + ])) + + @import.sync_ndjson_rows_count! + + assert_equal 3, @import.rows_count + end + + test "publishes import successfully" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { + id: "uuid-1", + name: "Import Test Account", + balance: "1000.00", + currency: "USD", + accountable_type: "Depository", + accountable: { subtype: "checking" } + } } + ])) + + initial_account_count = @family.accounts.count + + @import.publish + + assert_equal "complete", @import.status + assert_equal initial_account_count + 1, @family.accounts.count + + account = @family.accounts.find_by(name: "Import Test Account") + assert_not_nil account + assert_equal 1000.0, account.balance.to_f + assert_equal "USD", account.currency + assert_equal "Depository", account.accountable_type + end + + test "import tracks created accounts for revert" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { + id: "uuid-1", + name: "Revertable Account", + balance: "500.00", + currency: "USD", + accountable_type: "Depository" + } } + ])) + + @import.publish + + assert_equal 1, @import.accounts.count + assert_equal "Revertable Account", @import.accounts.first.name + end + + test "publishes later enqueues job" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { + id: "uuid-1", + name: "Async Account", + balance: "100", + currency: "USD", + accountable_type: "Depository" + } } + ])) + + assert_enqueued_with job: ImportJob, args: [ @import ] do + @import.publish_later + end + + assert_equal "importing", @import.status + end + + private + + def attach_ndjson(ndjson) + @import.ndjson_file.attach( + io: StringIO.new(ndjson), + filename: "all.ndjson", + content_type: "application/x-ndjson" + ) + @import.sync_ndjson_rows_count! + end + + def build_ndjson(records) + records.map(&:to_json).join("\n") + end +end diff --git a/test/system/imports_test.rb b/test/system/imports_test.rb index c39477ac3..15b146e24 100644 --- a/test/system/imports_test.rb +++ b/test/system/imports_test.rb @@ -13,6 +13,7 @@ class ImportsTest < ApplicationSystemTestCase test "transaction import" do visit new_import_path + click_on "Raw Data" click_on "Import transactions" within_testid("import-tabs") do @@ -63,6 +64,7 @@ class ImportsTest < ApplicationSystemTestCase test "trade import" do visit new_import_path + click_on "Raw Data" click_on "Import investments" within_testid("import-tabs") do @@ -105,6 +107,7 @@ class ImportsTest < ApplicationSystemTestCase test "account import" do visit new_import_path + click_on "Raw Data" click_on "Import accounts" within_testid("import-tabs") do @@ -153,6 +156,8 @@ class ImportsTest < ApplicationSystemTestCase test "mint import" do visit new_import_path + # Pending CSV-style imports default the dialog to the Raw Data tab; Mint lives under Financial Tools. + click_on "Financial Tools" click_on "Import from Mint" within_testid("import-tabs") do