From e59235fdc5a80cc588f284800834fa447778e000 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Wed, 13 May 2026 12:05:11 -0700 Subject: [PATCH] feat(statements): add account statement vault (#1753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(statements): add account statement vault Add web-only statement uploads, account linking, duplicate detection, and per-account coverage/reconciliation checks without mutating transactions. Extend ActiveStorage authorization and targeted tests for family/account scoping. * fix(statements): return deleted account statements to inbox Preserve linked statement records when an account is deleted by moving them back to the unmatched inbox, then expand coverage for upload validation, sanitized parser metadata, unavailable reconciliation, and missing-month coverage. * fix(statements): harden vault upload review flows Address review and security findings in the statement vault by preserving sanitized parser metadata, failing closed on orphaned statement blobs, avoiding account_id mass assignment permits, and adding regression coverage for link/delete edge cases. * fix(statements): harden vault upload and access controls * fix(statements): address vault hardening review * fix(statements): address vault review feedback Prioritize SHA-256 duplicate detection while preserving MD5 fallback for legacy rows. Remove free-form account notes from statement matching, document direct account-destroy unlinking, and add year-selectable historical coverage with muted out-of-range months. * fix(statements): harden vault review follow-ups Clarify legacy MD5 checksum use, whitelist statement balance helper dispatch, and preserve sanitized parser metadata. Hide statement management controls from read-only viewers while keeping server-side authorization unchanged. * fix(statements): repair settings system coverage Allow the changelog provider lookup in the self-hosting settings system test, include Statement Vault in settings navigation coverage, and align the feature title casing. Update the devcontainer so ActiveStorage and parallel system tests can run in the documented environment. * fix(statements): move vault beside accounts Place Statement Vault with account settings instead of between Imports and Exports. Keep settings footer ordering and system navigation coverage aligned, including the non-admin visibility guard. * fix(statements): address vault review cleanup Resolve CodeRabbit review feedback for statement upload validation, duplicate race handling, account statement matching semantics, metadata detection, ActiveStorage authorization tests, and small UI/style cleanups. * fix(statements): address vault cleanup review * fix(statements): deduplicate vault style helpers * fix(statements): close vault review follow-ups * fix(statements): refresh schema after upstream rebase * fix(statements): process vault uploads sequentially * fix(statements): close vault review follow-ups * fix(statements): scope vault index to accessible accounts * fix(statements): harden statement vault readiness Squash the statement vault migration hardening into the feature migration, tighten Active Storage authorization edge cases, bound CSV metadata detection, and add real PDF fixture coverage for stored statements. Validation: targeted statement/auth/controller/provider tests, full Rails suite, system tests, RuboCop, Biome, Brakeman, Zeitwerk, importmap audit, npm audit, ERB lint, CodeRabbit, and Codex Security all passed locally. * fix(statements): close vault review follow-ups Move statement unlinking to after account destroy commit, keep Kraken account creation on the shared crypto helper, and add statement metadata length limits with DB checks. Validation: fresh devcontainer with fresh DB via db:prepare, focused account/statement/Kraken/Binance tests, RuboCop, Brakeman, Zeitwerk, git diff --check, CodeRabbit, and Codex Security passed before commit. * fix(statements): address vault scan follow-ups Move statement tab data setup out of the ERB partial, harden reconciliation labels and coverage initialization, and tighten statement schema constraints. Validation: CodeRabbit and Codex Security reviewed the current PR diff; Rails focused tests, full Rails tests, system tests, RuboCop, Brakeman, Zeitwerk, ERB lint, npm lint, importmap audit, npm audit, and git diff --check passed. * fix(statements): defer vault tab loading --------- Signed-off-by: Juan José Mata Co-authored-by: Juan José Mata --- .devcontainer/Dockerfile | 3 + .devcontainer/docker-compose.yml | 3 + app/components/UI/account_page.html.erb | 2 +- app/components/UI/account_page.rb | 40 +- .../account_statements_controller.rb | 211 +++++ app/controllers/accounts_controller.rb | 37 + app/helpers/account_statements_helper.rb | 91 +++ app/helpers/settings_helper.rb | 50 +- app/models/account.rb | 72 +- app/models/account_statement.rb | 462 +++++++++++ .../account_statement/account_matcher.rb | 73 ++ app/models/account_statement/coverage.rb | 194 +++++ .../account_statement/metadata_detector.rb | 221 +++++ app/models/family.rb | 1 + app/views/account_statements/index.html.erb | 170 ++++ app/views/account_statements/show.html.erb | 180 ++++ app/views/accounts/show.html.erb | 6 +- app/views/accounts/show/_menu.html.erb | 1 + app/views/accounts/show/_statements.html.erb | 124 +++ .../accounts/show/_statements_frame.html.erb | 10 + app/views/settings/_settings_nav.html.erb | 3 +- config/brakeman.ignore | 23 - .../active_storage_authorization.rb | 92 ++- .../locales/models/account_statement/en.yml | 30 + .../locales/views/account_statements/en.yml | 116 +++ config/locales/views/accounts/en.yml | 6 + config/locales/views/settings/en.yml | 3 + config/locales/views/settings/fr.yml | 3 + config/routes.rb | 8 + ...0260505120000_create_account_statements.rb | 91 +++ db/schema.rb | 54 ++ .../account_statements_controller_test.rb | 483 +++++++++++ test/controllers/accounts_controller_test.rb | 77 ++ test/fixtures/account_statements.yml | 1 + .../helpers/account_statements_helper_test.rb | 15 + .../active_storage_authorization_test.rb | 246 ++++++ test/models/account_statement_test.rb | 770 ++++++++++++++++++ test/models/account_test.rb | 36 + test/system/settings_test.rb | 15 +- test/test_helper.rb | 23 + 40 files changed, 3943 insertions(+), 103 deletions(-) create mode 100644 app/controllers/account_statements_controller.rb create mode 100644 app/helpers/account_statements_helper.rb create mode 100644 app/models/account_statement.rb create mode 100644 app/models/account_statement/account_matcher.rb create mode 100644 app/models/account_statement/coverage.rb create mode 100644 app/models/account_statement/metadata_detector.rb create mode 100644 app/views/account_statements/index.html.erb create mode 100644 app/views/account_statements/show.html.erb create mode 100644 app/views/accounts/show/_statements.html.erb create mode 100644 app/views/accounts/show/_statements_frame.html.erb create mode 100644 config/locales/models/account_statement/en.yml create mode 100644 config/locales/views/account_statements/en.yml create mode 100644 db/migrate/20260505120000_create_account_statements.rb create mode 100644 test/controllers/account_statements_controller_test.rb create mode 100644 test/fixtures/account_statements.yml create mode 100644 test/helpers/account_statements_helper_test.rb create mode 100644 test/models/account_statement_test.rb diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e024ef870..119f883aa 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,6 +3,9 @@ FROM ruby:${RUBY_VERSION}-slim-bookworm ENV DEBIAN_FRONTEND=noninteractive +# libvips42 supports ActiveStorage image variants through image_processing/ruby-vips +# for existing profile/avatar images. AccountStatement.original_file only stores +# PDF/CSV/XLSX originals, but the devcontainer needs this broader image stack. RUN apt-get update -qq \ && apt-get -y install --no-install-recommends \ apt-utils \ diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index d1825c041..12b837168 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -70,6 +70,9 @@ services: - "7900:7900" shm_size: 2gb restart: unless-stopped + environment: + SE_NODE_MAX_SESSIONS: 4 + SE_NODE_OVERRIDE_MAX_SESSIONS: "true" volumes: postgres-data: diff --git a/app/components/UI/account_page.html.erb b/app/components/UI/account_page.html.erb index b9befa684..820d3a521 100644 --- a/app/components/UI/account_page.html.erb +++ b/app/components/UI/account_page.html.erb @@ -20,7 +20,7 @@ <%= render DS::Tabs.new(active_tab: active_tab, url_param_key: "tab") do |tabs_container| %> <% tabs_container.with_nav(classes: "max-w-fit") do |nav| %> <% tabs.each do |tab| %> - <% nav.with_btn(id: tab, label: tab.to_s.humanize, classes: "px-6") %> + <% nav.with_btn(id: tab, label: t("accounts.show.tabs.#{tab}", default: tab.to_s.humanize), classes: "px-6") %> <% end %> <% end %> diff --git a/app/components/UI/account_page.rb b/app/components/UI/account_page.rb index 0fc5f0e53..36be3c2b2 100644 --- a/app/components/UI/account_page.rb +++ b/app/components/UI/account_page.rb @@ -1,13 +1,19 @@ class UI::AccountPage < ApplicationComponent - attr_reader :account, :chart_view, :chart_period + attr_reader :account, :chart_view, :chart_period, :statement_coverage, :statements, :reconciliation_statuses, + :can_manage_statements renders_one :activity_feed, ->(feed_data:, pagy:, search:) { UI::Account::ActivityFeed.new(feed_data: feed_data, pagy: pagy, search: search) } - def initialize(account:, chart_view: nil, chart_period: nil, active_tab: nil) + def initialize(account:, chart_view: nil, chart_period: nil, active_tab: nil, statement_coverage: nil, statements: [], + reconciliation_statuses: {}, can_manage_statements: false) @account = account @chart_view = chart_view @chart_period = chart_period @active_tab = active_tab + @statement_coverage = statement_coverage + @statements = statements + @reconciliation_statuses = reconciliation_statuses + @can_manage_statements = can_manage_statements end def id @@ -37,7 +43,7 @@ class UI::AccountPage < ApplicationComponent end def tabs - case account.accountable_type + base_tabs = case account.accountable_type when "Investment", "Crypto" [ :activity, :holdings ] when "Property", "Vehicle", "Loan" @@ -45,6 +51,8 @@ class UI::AccountPage < ApplicationComponent else [ :activity ] end + + base_tabs + [ :statements ] end def fx_coverage_start_date @@ -71,6 +79,32 @@ class UI::AccountPage < ApplicationComponent when :holdings, :overview # Accountable is responsible for implementing the partial in the correct folder render "#{account.accountable_type.downcase.pluralize}/tabs/#{tab}", account: account + when :statements + render_statement_tab end end + + def render_statement_tab + return render "accounts/show/statements_frame", **statement_tab_locals if statement_tab_loaded? + + turbo_frame_tag statement_tab_frame_id, src: helpers.account_path(account, tab: "statements"), loading: :lazy + end + + def statement_tab_loaded? + statement_coverage.present? + end + + def statement_tab_frame_id + dom_id(account, :statements_tab) + end + + def statement_tab_locals + { + account: account, + coverage: statement_coverage, + statements: statements, + reconciliation_statuses: reconciliation_statuses, + can_manage_statements: can_manage_statements + } + end end diff --git a/app/controllers/account_statements_controller.rb b/app/controllers/account_statements_controller.rb new file mode 100644 index 000000000..27793c4b6 --- /dev/null +++ b/app/controllers/account_statements_controller.rb @@ -0,0 +1,211 @@ +# frozen_string_literal: true + +class AccountStatementsController < ApplicationController + before_action :set_statement, only: %i[show update destroy link unlink reject] + before_action :ensure_statement_manager!, only: %i[index create update destroy link unlink reject] + + def index + accessible_account_ids = Current.user.accessible_accounts.select(:id) + account_statements = Current.family.account_statements + .with_attached_original_file + .includes(:account, :suggested_account) + .ordered + visible_storage_scope = Current.family.account_statements + .where(account_id: nil) + .or(Current.family.account_statements.where(account_id: accessible_account_ids)) + linked_statement_scope = account_statements.with_account.where(account_id: accessible_account_ids) + + @unmatched_pagy, @unmatched_statements = pagy(account_statements.unmatched, limit: safe_per_page, page_param: :unmatched_page) + @linked_pagy, @linked_statements = pagy(linked_statement_scope, limit: safe_per_page, page_param: :linked_page) + @total_storage_bytes = visible_storage_scope.sum(:byte_size) + @accounts = Current.user.accessible_accounts.visible.alphabetically + @breadcrumbs = [ + [ t("breadcrumbs.home"), root_path ], + [ t("account_statements.index.title"), account_statements_path ] + ] + render layout: "settings" + end + + def show + @accounts = Current.user.accessible_accounts.visible.alphabetically + @can_manage_statement = @statement.manageable_by?(Current.user) + @reconciliation_checks = @statement.reconciliation_checks + @breadcrumbs = [ + [ t("breadcrumbs.home"), root_path ], + [ t("account_statements.index.title"), account_statements_path ], + [ @statement.filename, nil ] + ] + render layout: "settings" + end + + def create + files = Array(statement_upload_params[:files]).reject(&:blank?).select { |file| file.respond_to?(:read) } + account = target_account + + if files.empty? + redirect_back_or_to account_statements_path, alert: t("account_statements.create.no_files") + return + end + + return if account && !require_account_permission!(account) + + created = [] + duplicates = [] + validation_errors = [] + + files.each do |file| + prepared_upload = AccountStatement.prepare_upload!(file) + created << AccountStatement.create_from_prepared_upload!(family: Current.family, account: account, prepared_upload: prepared_upload) + rescue AccountStatement::InvalidUploadError + validation_errors << t("account_statements.create.invalid_file_type") + rescue AccountStatement::DuplicateUploadError => e + duplicates << e.statement + rescue ActiveRecord::RecordInvalid => e + validation_errors << e.record.errors.full_messages.to_sentence + end + + redirect_to redirect_after_create(account, created.first || duplicates.first), + flash_for_upload(created:, duplicates:, validation_errors:) + end + + def update + return if @statement.account && !require_account_permission!(@statement.account) + + target = statement_account_id.present? ? Current.user.accessible_accounts.find(statement_account_id) : nil + return if target && !require_account_permission!(target) + + attrs = statement_params.to_h + attrs[:account] = target if statement_account_id_provided? + + @statement.assign_attributes(attrs) + @statement.assign_account_match if @statement.account.nil? && !@statement.rejected? + + if @statement.save + redirect_to account_statement_path(@statement), notice: t("account_statements.update.success") + else + @accounts = Current.user.accessible_accounts.visible.alphabetically + @can_manage_statement = @statement.manageable_by?(Current.user) + @reconciliation_checks = @statement.reconciliation_checks + flash.now[:alert] = @statement.errors.full_messages.to_sentence + render :show, status: :unprocessable_entity, layout: "settings" + end + end + + def link + return if @statement.account && !require_account_permission!(@statement.account) + + account_id = params[:account_id].presence || @statement.suggested_account_id + if account_id.blank? + redirect_to account_statement_path(@statement), alert: t("account_statements.link.no_account") + return + end + + account = Current.user.accessible_accounts.find(account_id) + return unless require_account_permission!(account) + + @statement.link_to_account!(account) + redirect_to post_link_path(@statement), notice: t("account_statements.link.success", account: account.name) + end + + def unlink + return if @statement.account && !require_account_permission!(@statement.account) + + @statement.unlink! + redirect_to account_statement_path(@statement), notice: t("account_statements.unlink.success") + end + + def reject + return if @statement.account && !require_account_permission!(@statement.account) + + @statement.reject_match! + redirect_to account_statements_path, notice: t("account_statements.reject.success") + end + + def destroy + return if @statement.account && !require_account_permission!(@statement.account) + + redirect_path = @statement.account ? account_path(@statement.account, tab: "statements") : account_statements_path + if @statement.destroy + redirect_to redirect_path, notice: t("account_statements.destroy.success") + else + redirect_back_or_to redirect_path, alert: t("account_statements.destroy.failure") + end + end + + private + + def set_statement + @statement = Current.family.account_statements + .with_attached_original_file + .includes(:account, :suggested_account) + .find(params[:id]) + + raise ActiveRecord::RecordNotFound unless @statement.viewable_by?(Current.user) + end + + def ensure_statement_manager! + return if AccountStatement.statement_manager?(Current.user) + + redirect_to accounts_path, alert: t("accounts.not_authorized") + end + + def statement_upload_params + params.fetch(:account_statement, ActionController::Parameters.new).permit(files: []) + end + + def statement_params + params.require(:account_statement).permit( + :institution_name_hint, + :account_name_hint, + :account_last4_hint, + :period_start_on, + :period_end_on, + :opening_balance, + :closing_balance, + :currency + ) + end + + def target_account + account_id = statement_account_id.presence + return nil if account_id.blank? + + Current.user.accessible_accounts.find(account_id) + end + + def statement_account_id + params.fetch(:account_statement, ActionController::Parameters.new)[:account_id] + end + + def statement_account_id_provided? + params.fetch(:account_statement, ActionController::Parameters.new).key?(:account_id) + end + + def redirect_after_create(account, statement = nil) + if account + account_path(account, tab: "statements") + elsif statement + account_statement_path(statement) + else + account_statements_path + end + end + + def post_link_path(statement) + statement.account ? account_path(statement.account, tab: "statements") : account_statement_path(statement) + end + + def flash_for_upload(created:, duplicates:, validation_errors: []) + alerts = [] + alerts << t("account_statements.create.duplicates", count: duplicates.size) if duplicates.any? + alerts.concat(validation_errors.compact_blank) + + if created.any? + flash = { notice: t("account_statements.create.success", count: created.size) } + flash[:alert] = alerts.to_sentence if alerts.any? + flash + else + { alert: alerts.to_sentence } + end + end +end diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 084a37217..2becdd3b3 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -50,6 +50,10 @@ class AccountsController < ApplicationController @tab = params[:tab] @q = params.fetch(:q, {}).permit(:search, status: []) entries = @account.entries.where(excluded: false).search(@q).reverse_chronological.includes(:entryable) + if statement_tab_active? + build_statement_tab_data + return render_statement_tab_frame if statement_tab_frame_request? + end @pagy, @entries = pagy( entries, @@ -237,6 +241,39 @@ class AccountsController < ApplicationController end end + def build_statement_tab_data + return unless statement_tab_active? + + @statement_coverage = AccountStatement::Coverage.for_year(@account, params[:statement_year]) + @account_statements = @account.account_statements.with_attached_original_file.ordered.to_a + @statement_reconciliation_statuses = AccountStatement.reconciliation_statuses_for(@account_statements, account: @account) + permission = @account.permission_for(Current.user) + @can_manage_statements = AccountStatement.statement_manager?(Current.user) && + permission.in?([ :owner, :full_control ]) + end + + def statement_tab_frame_request? + turbo_frame_request? && request.headers["Turbo-Frame"] == helpers.dom_id(@account, :statements_tab) + end + + def render_statement_tab_frame + render partial: "accounts/show/statements_frame", locals: statement_tab_locals, layout: false + end + + def statement_tab_locals + { + account: @account, + coverage: @statement_coverage, + statements: @account_statements, + reconciliation_statuses: @statement_reconciliation_statuses, + can_manage_statements: @can_manage_statements + } + end + + def statement_tab_active? + @tab == "statements" + end + # Builds sync stats maps for all provider types to avoid N+1 queries in views def build_sync_stats_maps # SimpleFIN sync stats diff --git a/app/helpers/account_statements_helper.rb b/app/helpers/account_statements_helper.rb new file mode 100644 index 000000000..fa736b5ae --- /dev/null +++ b/app/helpers/account_statements_helper.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module AccountStatementsHelper + ACCOUNT_STATEMENT_BALANCE_FIELDS = %w[opening_balance closing_balance].freeze + + def account_statement_status_badge(statement) + case statement.review_status + when "linked" + render("shared/badge", color: "success") { t("account_statements.status.linked") } + when "rejected" + render("shared/badge", color: "warning") { t("account_statements.status.rejected") } + else + render("shared/badge") { t("account_statements.status.unmatched") } + end + end + + def account_statement_coverage_classes(status) + case status.to_s + when "not_expected" + "bg-container-inset text-subdued ring-alpha-black-25" + when "covered" + "bg-success/10 text-success ring-success/20" + when "duplicate", "ambiguous" + "bg-warning/10 text-warning ring-warning/20" + when "mismatched" + "bg-destructive/10 text-destructive ring-destructive/20" + else + "bg-gray-tint-5 text-secondary ring-alpha-black-50" + end + end + + def account_statement_period(statement) + if statement.period_start_on.present? && statement.period_end_on.present? + "#{format_date(statement.period_start_on)} - #{format_date(statement.period_end_on)}" + else + t("account_statements.period.unknown") + end + end + + def account_statement_coverage_label(month) + account_statement_month_label(month.date) + end + + def account_statement_month_label(date) + l(date, format: "%b %Y") + end + + def account_statement_coverage_range(coverage) + t( + "account_statements.account_tab.coverage_range", + start: account_statement_month_label(coverage.expected_start_month), + end: account_statement_month_label(coverage.expected_end_month) + ) + end + + def account_statement_reconciliation_label(check) + key = check[:key] if check.respond_to?(:key?) && check.key?(:key) + key ||= check["key"] if check.respond_to?(:key?) && check.key?("key") + fallback = t("account_statements.reconciliation.checks.unknown_check") + return fallback if key.blank? + + t( + "account_statements.reconciliation.checks.#{key}", + default: fallback + ) + end + + def account_statement_balance_label(statement, field) + return t("account_statements.balance.unknown") unless field.to_s.in?(ACCOUNT_STATEMENT_BALANCE_FIELDS) + + money = statement.public_send("#{field}_money") + money ? money.format : t("account_statements.balance.unknown") + end + + def account_statement_currency_options(statement) + currency_picker_options_for_family(Current.family, extra: [ statement.currency ]).map do |code| + currency = Money::Currency.new(code) + [ "#{currency.name} (#{currency.iso_code})", currency.iso_code ] + end + end + + def account_statement_file_icon(statement) + if statement.pdf? + "file-text" + elsif statement.xlsx? + "sheet" + else + "file-spreadsheet" + end + end +end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index 1cc3f32e2..e3cdfb3d9 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -1,30 +1,31 @@ module SettingsHelper SETTINGS_ORDER = [ # General section - { name: "Accounts", path: :accounts_path }, - { name: "Bank Sync", path: :settings_providers_path, condition: :admin_user? }, - { name: "Preferences", path: :settings_preferences_path }, - { name: "Appearance", path: :settings_appearance_path }, - { name: "Profile Info", path: :settings_profile_path }, - { name: "Security", path: :settings_security_path }, - { name: "Payment", path: :settings_payment_path, condition: :not_self_hosted? }, + { name: -> { t("settings.settings_nav.accounts_label") }, path: :accounts_path }, + { name: -> { t("settings.settings_nav.bank_sync_label") }, path: :settings_providers_path, condition: :admin_user? }, + { name: -> { t("settings.settings_nav.preferences_label") }, path: :settings_preferences_path }, + { name: -> { t("settings.settings_nav.appearance_label") }, path: :settings_appearance_path }, + { name: -> { t("settings.settings_nav.profile_label") }, path: :settings_profile_path }, + { name: -> { t("settings.settings_nav.security_label") }, path: :settings_security_path }, + { name: -> { t("settings.settings_nav.payment_label") }, path: :settings_payment_path, condition: :not_self_hosted? }, # Transactions section - { name: "Categories", path: :categories_path }, - { name: "Tags", path: :tags_path }, - { name: "Rules", path: :rules_path }, - { name: "Merchants", path: :family_merchants_path }, - { name: "Recurring", path: :recurring_transactions_path }, + { name: -> { t("settings.settings_nav.categories_label") }, path: :categories_path }, + { name: -> { t("settings.settings_nav.tags_label") }, path: :tags_path }, + { name: -> { t("settings.settings_nav.rules_label") }, path: :rules_path }, + { name: -> { t("settings.settings_nav.merchants_label") }, path: :family_merchants_path }, + { name: -> { t("settings.settings_nav.recurring_transactions_label") }, path: :recurring_transactions_path }, + { name: -> { t("settings.settings_nav.statement_vault_label") }, path: :account_statements_path, condition: :admin_user? }, # Advanced section - { name: "AI Prompts", path: :settings_ai_prompts_path, condition: :admin_user? }, - { name: "LLM Usage", path: :settings_llm_usage_path, condition: :admin_user? }, - { name: "API Key", path: :settings_api_key_path, condition: :admin_user? }, - { name: "Self-Hosting", path: :settings_hosting_path, condition: :self_hosted_and_admin? }, - { name: "Imports", path: :imports_path, condition: :admin_user? }, - { name: "Exports", path: :family_exports_path, condition: :admin_user? }, + { name: -> { t("settings.settings_nav.ai_prompts_label") }, path: :settings_ai_prompts_path, condition: :admin_user? }, + { name: -> { t("settings.settings_nav.llm_usage_label") }, path: :settings_llm_usage_path, condition: :admin_user? }, + { name: -> { t("settings.settings_nav.api_key_label") }, path: :settings_api_key_path, condition: :admin_user? }, + { name: -> { t("settings.settings_nav.self_hosting_label") }, path: :settings_hosting_path, condition: :self_hosted_and_admin? }, + { name: -> { t("settings.settings_nav.imports_label") }, path: :imports_path, condition: :admin_user? }, + { name: -> { t("settings.settings_nav.exports_label") }, path: :family_exports_path, condition: :admin_user? }, # More section - { name: "Guides", path: :settings_guides_path }, - { name: "What's new", path: :changelog_path }, - { name: "Feedback", path: :feedback_path } + { name: -> { t("settings.settings_nav.guides_label") }, path: :settings_guides_path }, + { name: -> { t("settings.settings_nav.whats_new_label") }, path: :changelog_path }, + { name: -> { t("settings.settings_nav.feedback_label") }, path: :feedback_path } ] def adjacent_setting(current_path, offset) @@ -40,7 +41,7 @@ module SettingsHelper render partial: "settings/settings_nav_link_large", locals: { path: send(adjacent[:path]), direction: offset > 0 ? "next" : "previous", - title: adjacent[:name] + title: setting_name(adjacent) } end @@ -222,6 +223,11 @@ module SettingsHelper !self_hosted? end + def setting_name(setting) + name = setting[:name] + name.respond_to?(:call) ? instance_exec(&name) : name + end + # Helper used by SETTINGS_ORDER conditions def admin_user? Current.user&.admin? diff --git a/app/models/account.rb b/app/models/account.rb index 13c06c58e..b0595d308 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -2,6 +2,8 @@ class Account < ApplicationRecord include AASM, Syncable, Monetizable, Chartable, Linkable, Enrichable, Anchorable, Reconcileable, TaxTreatable before_validation :assign_default_owner, if: -> { owner_id.blank? } + before_destroy :capture_account_statement_ids_to_move + after_destroy_commit :move_account_statements_to_inbox validates :name, :balance, :currency, presence: true validate :owner_belongs_to_family, if: -> { owner_id.present? && family_id.present? } @@ -77,6 +79,8 @@ class Account < ApplicationRecord } has_one_attached :logo, dependent: :purge_later + # No dependent: option; before_destroy captures IDs, after_destroy_commit moves statements back to inbox. + has_many :account_statements delegated_type :accountable, types: Accountable::TYPES, dependent: :destroy delegate :subtype, to: :accountable, allow_nil: true @@ -256,22 +260,7 @@ class Account < ApplicationRecord end def create_from_binance_account(binance_account) - family = binance_account.binance_item.family - - attributes = { - family: family, - name: binance_account.name, - balance: (binance_account.current_balance || 0).to_d, - cash_balance: 0, - currency: binance_account.currency.presence || family.currency, - accountable_type: "Crypto", - accountable_attributes: { - subtype: "exchange", - tax_treatment: "taxable" - } - } - - create_and_sync(attributes, skip_initial_sync: true) + create_from_crypto_exchange_account(binance_account, family: binance_account.binance_item.family) end def create_from_ibkr_account(ibkr_account) @@ -298,26 +287,28 @@ class Account < ApplicationRecord end def create_from_kraken_account(kraken_account) - family = kraken_account.kraken_item.family - - attributes = { - family: family, - name: kraken_account.name, - balance: (kraken_account.current_balance || 0).to_d, - cash_balance: 0, - currency: kraken_account.currency.presence || family.currency, - accountable_type: "Crypto", - accountable_attributes: { - subtype: "exchange", - tax_treatment: "taxable" - } - } - - create_and_sync(attributes, skip_initial_sync: true) + create_from_crypto_exchange_account(kraken_account, family: kraken_account.kraken_item.family) end private + def create_from_crypto_exchange_account(provider_account, family:) + attributes = { + family: family, + name: provider_account.name, + balance: (provider_account.current_balance || 0).to_d, + cash_balance: 0, + currency: provider_account.currency.presence || family.currency, + accountable_type: "Crypto", + accountable_attributes: { + subtype: "exchange", + tax_treatment: "taxable" + } + } + + create_and_sync(attributes, skip_initial_sync: true) + end + def build_simplefin_accountable_attributes(simplefin_account, account_type, subtype) attributes = {} attributes[:subtype] = subtype if subtype.present? @@ -535,4 +526,21 @@ class Account < ApplicationRecord return if User.where(id: owner_id, family_id: family_id).exists? errors.add(:owner, :invalid, message: "must belong to the same family as the account") end + + def capture_account_statement_ids_to_move + @statement_ids_to_move = account_statements.ids + end + + def move_account_statements_to_inbox + statement_ids = Array(@statement_ids_to_move).compact + return if statement_ids.empty? + + # Bypass callbacks deliberately: the account was destroyed, so linked statements need a direct inbox move. + AccountStatement.where(id: statement_ids).update_all( + account_id: nil, + review_status: "unmatched", + match_confidence: nil, + updated_at: Time.current + ) + end end diff --git a/app/models/account_statement.rb b/app/models/account_statement.rb new file mode 100644 index 000000000..07cc80c86 --- /dev/null +++ b/app/models/account_statement.rb @@ -0,0 +1,462 @@ +# frozen_string_literal: true + +require "digest/md5" +require "digest/sha2" +require "stringio" + +class AccountStatement < ApplicationRecord + include Monetizable + + DuplicateUploadError = Class.new(StandardError) do + attr_reader :statement + + def initialize(statement) + @statement = statement + super("Statement file has already been uploaded") + end + end + InvalidUploadError = Class.new(StandardError) + + PreparedUpload = Data.define(:content, :filename, :content_type, :byte_size, :checksum, :content_sha256) + + MAX_FILE_SIZE = 25.megabytes + READ_CHUNK_SIZE = 1.megabyte + ALLOWED_EXTENSION_CONTENT_TYPES = { + ".pdf" => %w[application/pdf], + ".csv" => %w[text/csv text/plain application/csv application/vnd.ms-excel], + ".xlsx" => %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet] + }.freeze + ALLOWED_CONTENT_TYPES = ALLOWED_EXTENSION_CONTENT_TYPES.values.flatten.uniq.freeze + ACCEPTED_FILE_EXTENSIONS = ALLOWED_EXTENSION_CONTENT_TYPES.keys.freeze + + belongs_to :family + belongs_to :account, optional: true + belongs_to :suggested_account, class_name: "Account", optional: true + + has_one_attached :original_file, dependent: :purge_later + + enum :source, { manual_upload: "manual_upload" }, validate: true, default: "manual_upload" + enum :upload_status, { stored: "stored", failed: "failed" }, validate: true, default: "stored" + enum :review_status, { unmatched: "unmatched", linked: "linked", rejected: "rejected" }, validate: true, default: "unmatched", scopes: false + + monetize :opening_balance, :closing_balance + + before_validation :sync_file_metadata, if: -> { original_file.attached? } + before_validation :normalize_currency + before_validation :sync_review_status + + validates :filename, :content_type, :checksum, presence: true + validates :byte_size, presence: true, numericality: { greater_than: 0, less_than_or_equal_to: MAX_FILE_SIZE } + validates :content_type, inclusion: { in: ALLOWED_CONTENT_TYPES } + validates :content_sha256, + format: { with: /\A[0-9a-f]{64}\z/ }, + uniqueness: { scope: :family_id, allow_nil: true, message: :duplicate_statement_file }, + allow_nil: true + validates :parser_confidence, :match_confidence, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true + validate :account_belongs_to_family + validate :suggested_account_belongs_to_family + validate :period_order + validate :currency_is_valid + validate :filename_extension_matches_content_type + validate :original_file_attached + validate :original_file_constraints, if: -> { original_file.attached? } + + scope :ordered, -> { order(created_at: :desc) } + scope :with_account, -> { where.not(account_id: nil) } + scope :unmatched, -> { where(account_id: nil).where(review_status: "unmatched") } + scope :for_month, ->(month) { + month_start = month.to_date.beginning_of_month + month_end = month_start.end_of_month + where("period_start_on <= ? AND period_end_on >= ?", month_end, month_start) + } + + class << self + def statement_manager?(user) + user&.admin? || user&.member? + end + + def create_from_upload!(family:, account:, file:) + prepared_upload = prepare_upload!(file) + create_from_prepared_upload!(family: family, account: account, prepared_upload: prepared_upload) + end + + def create_from_prepared_upload!(family:, account:, prepared_upload:) + statement = nil + duplicate = duplicate_for(family, prepared_upload) + raise DuplicateUploadError, duplicate if duplicate + + statement = family.account_statements.build( + account: account, + filename: prepared_upload.filename, + content_type: prepared_upload.content_type, + byte_size: prepared_upload.byte_size, + checksum: prepared_upload.checksum, + content_sha256: prepared_upload.content_sha256, + source: :manual_upload, + upload_status: :stored, + review_status: account.present? ? :linked : :unmatched, + currency: account&.currency || family.currency + ) + + statement.original_file.attach( + io: StringIO.new(prepared_upload.content), + filename: prepared_upload.filename, + content_type: prepared_upload.content_type + ) + + MetadataDetector.new(statement, content: prepared_upload.content).apply + statement.assign_account_match unless account.present? + statement.save! + statement + rescue ActiveRecord::RecordNotUnique + duplicate = duplicate_for(family, prepared_upload) + purge_original_file(statement) + + if duplicate + raise DuplicateUploadError, duplicate + end + + raise + rescue StandardError + purge_original_file(statement) + raise + end + + def reconciliation_statuses_for(statements, account:) + statement_list = statements.to_a + balance_lookup = balance_lookup_for(account, statement_list) + + statement_list.to_h do |statement| + [ statement.id, statement.reconciliation_status(balance_lookup: balance_lookup) ] + end + end + + def prepare_upload!(file) + filename = file.original_filename.to_s + content = read_upload_content!(file) + byte_size = content.bytesize + raise InvalidUploadError if byte_size.zero? + + content_type = detected_content_type(content:, filename:, declared_content_type: file.content_type) + raise InvalidUploadError unless allowed_upload?(filename:, content_type:) + raise InvalidUploadError if content_type == "application/pdf" && !valid_pdf_content?(content) + + PreparedUpload.new( + content: content, + filename: filename, + content_type: content_type, + byte_size: byte_size, + checksum: Digest::MD5.base64digest(content), + content_sha256: Digest::SHA256.hexdigest(content) + ) + end + + def detected_content_type(content:, filename:, declared_content_type:) + Marcel::MimeType.for( + StringIO.new(content), + name: filename, + declared_type: declared_content_type.presence + ) + end + + def allowed_upload?(filename:, content_type:) + allowed_content_types_for_filename(filename).include?(content_type) + end + + def allowed_content_types_for_filename(filename) + ALLOWED_EXTENSION_CONTENT_TYPES.fetch(File.extname(filename.to_s).downcase, []) + end + + def valid_pdf_content?(content) + content.start_with?("%PDF-") + end + + def purge_original_file(statement) + return unless statement&.original_file&.attached? + + statement.original_file.purge + rescue StandardError => e + Rails.logger.warn("AccountStatement staged blob cleanup failed: #{e.class}: #{e.message}") + end + + def balance_lookup_for(account, statements) + currencies = statements.map(&:statement_currency).compact.uniq + dates = statements.flat_map { |statement| [ statement.period_start_on, statement.period_end_on ] }.compact.uniq + balances = if currencies.any? && dates.any? + account.balances.where(currency: currencies, date: dates).to_a + else + [] + end + balances_by_key = balances.index_by { |balance| [ balance.date, balance.currency ] } + + ->(date, currency) { balances_by_key[[ date, currency ]] } + end + + def read_upload_content!(file) + declared_size = declared_upload_size(file) + raise InvalidUploadError if declared_size.present? && declared_size > MAX_FILE_SIZE + + content = +"".b + loop do + chunk = file.read(READ_CHUNK_SIZE) + break if chunk.nil? || chunk.empty? + + content << chunk + raise InvalidUploadError if content.bytesize > MAX_FILE_SIZE + end + + file.rewind if file.respond_to?(:rewind) + content + end + + def declared_upload_size(file) + if file.respond_to?(:size) + file.size + elsif file.respond_to?(:length) + file.length + end + end + + def duplicate_for(family, prepared_upload) + scope = family.account_statements + sha_duplicate = scope.find_by(content_sha256: prepared_upload.content_sha256) if prepared_upload.content_sha256.present? + return sha_duplicate if sha_duplicate + + # Active Storage's MD5 checksum is retained only to catch legacy rows that predate content_sha256. + legacy_scope = prepared_upload.content_sha256.present? ? scope.where(content_sha256: nil) : scope + legacy_scope.find_by(checksum: prepared_upload.checksum) + end + end + + def viewable_by?(user) + return false unless user&.family_id == family_id + + account.present? ? account.shared_with?(user) : self.class.statement_manager?(user) + end + + def manageable_by?(user) + return false unless user&.family_id == family_id + + return self.class.statement_manager?(user) if account.blank? + + account.permission_for(user).in?([ :owner, :full_control ]) && self.class.statement_manager?(user) + end + + def link_to_account!(target_account, confidence: 1.0) + update!( + account: target_account, + suggested_account: nil, + match_confidence: confidence, + review_status: :linked, + currency: currency.presence || target_account.currency + ) + end + + def unlink! + transaction do + update!( + account: nil, + review_status: :unmatched, + match_confidence: nil + ) + assign_account_match + save! + end + end + + def reject_match! + update!( + suggested_account: nil, + match_confidence: nil, + review_status: :rejected + ) + end + + def assign_account_match + match = AccountMatcher.new(self).best_match + + self.suggested_account = match&.account + self.match_confidence = match&.confidence + clear_invalid_suggested_account + end + + def covered_months + return [] unless period_start_on.present? && period_end_on.present? + + current = period_start_on.beginning_of_month + last = period_end_on.beginning_of_month + months = [] + + while current <= last + months << current + current = current.next_month + end + + months + end + + def covers_month?(month) + covered_months.include?(month.to_date.beginning_of_month) + end + + def reconciliation_status(balance_lookup: nil) + checks = reconciliation_checks(balance_lookup: balance_lookup) + return "unavailable" if checks.empty? + + checks.any? { |check| check[:status] == "mismatched" } ? "mismatched" : "matched" + end + + def reconciliation_mismatched?(balance_lookup: nil) + reconciliation_status(balance_lookup: balance_lookup) == "mismatched" + end + + def reconciliation_checks(balance_lookup: nil) + return [] unless account.present? && period_start_on.present? && period_end_on.present? + + checks = [] + opening_balance_record = balance_record_for(period_start_on, statement_currency, balance_lookup) + closing_balance_record = balance_record_for(period_end_on, statement_currency, balance_lookup) + + if opening_balance.present? && opening_balance_record.present? + checks << reconciliation_check( + key: "opening_balance", + statement_amount: opening_balance, + ledger_amount: opening_balance_record.start_balance + ) + end + + if closing_balance.present? && closing_balance_record.present? + checks << reconciliation_check( + key: "closing_balance", + statement_amount: closing_balance, + ledger_amount: closing_balance_record.end_balance + ) + end + + if opening_balance.present? && closing_balance.present? && opening_balance_record.present? && closing_balance_record.present? + checks << reconciliation_check( + key: "period_movement", + statement_amount: closing_balance - opening_balance, + ledger_amount: closing_balance_record.end_balance - opening_balance_record.start_balance + ) + end + + checks + end + + def statement_currency + currency.presence || account&.currency || family.currency + end + + def pdf? + content_type.in?(ALLOWED_EXTENSION_CONTENT_TYPES[".pdf"]) + end + + def csv? + content_type.in?(ALLOWED_EXTENSION_CONTENT_TYPES[".csv"]) + end + + def xlsx? + content_type.in?(ALLOWED_EXTENSION_CONTENT_TYPES[".xlsx"]) + end + + private + + def reconciliation_check(key:, statement_amount:, ledger_amount:) + difference = statement_amount.to_d - ledger_amount.to_d + { + key: key, + statement_amount: statement_amount.to_d, + ledger_amount: ledger_amount.to_d, + difference: difference, + status: difference.abs <= 0.01.to_d ? "matched" : "mismatched" + } + end + + def balance_record_for(date, currency, balance_lookup) + return balance_lookup.call(date, currency) if balance_lookup + + account.balances.find_by(date: date, currency: currency) + end + + def sync_file_metadata + blob = original_file.blob + self.filename ||= blob.filename.to_s + self.content_type ||= blob.content_type + self.byte_size ||= blob.byte_size + self.checksum ||= blob.checksum + end + + def normalize_currency + self.currency = currency.to_s.upcase.presence if currency.present? + end + + def sync_review_status + return if rejected? + + self.review_status = "linked" if account.present? && !linked? + self.review_status = "unmatched" if account.blank? && linked? + end + + def account_belongs_to_family + return if account.nil? + return if account.family_id == family_id + + errors.add(:account, :invalid) + end + + def suggested_account_belongs_to_family + return if suggested_account_valid_for_family? + + errors.add(:suggested_account, :invalid) + end + + def clear_invalid_suggested_account + return if suggested_account_valid_for_family? + + self.suggested_account = nil + self.match_confidence = nil + end + + def suggested_account_valid_for_family? + suggested_account.nil? || suggested_account.family_id == family_id + end + + def period_order + return if period_start_on.blank? || period_end_on.blank? + return if period_start_on <= period_end_on + + errors.add(:period_end_on, :on_or_after_start) + end + + def currency_is_valid + return if currency.blank? + + Money::Currency.new(currency) + rescue Money::Currency::UnknownCurrencyError, ArgumentError + errors.add(:currency, :invalid) + end + + def filename_extension_matches_content_type + return if filename.blank? || content_type.blank? + return if self.class.allowed_upload?(filename: filename, content_type: content_type) + + errors.add(:content_type, :invalid) + end + + def original_file_constraints + if original_file.byte_size.zero? + errors.add(:original_file, :blank) + elsif original_file.byte_size > MAX_FILE_SIZE + errors.add(:original_file, :too_large, max_mb: MAX_FILE_SIZE / 1.megabyte) + end + + unless self.class.allowed_upload?(filename: original_file.filename.to_s, content_type: original_file.content_type) + errors.add(:original_file, :invalid_format, file_format: original_file.content_type) + end + end + + def original_file_attached + errors.add(:original_file, :blank) unless original_file.attached? + end +end diff --git a/app/models/account_statement/account_matcher.rb b/app/models/account_statement/account_matcher.rb new file mode 100644 index 000000000..6e8c5c1c2 --- /dev/null +++ b/app/models/account_statement/account_matcher.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +class AccountStatement::AccountMatcher + Match = Struct.new(:account, :confidence, keyword_init: true) + + attr_reader :statement + + def initialize(statement) + @statement = statement + end + + def best_match + candidates = statement.family.accounts.visible.to_a.filter_map do |account| + confidence = confidence_for(account) + next if confidence < 0.35 + + Match.new(account: account, confidence: confidence.round(4)) + end + + candidates.max_by(&:confidence) + end + + private + + def confidence_for(account) + score = 0.to_d + + if institution_hint.present? + score += 0.45.to_d if account_text(account).include?(institution_hint) + end + + if account_name_hint.present? + score += 0.25.to_d if account.name.to_s.downcase.include?(account_name_hint) + end + + if account_last4_hint.present? + score += 0.25.to_d if account_sensitive_match_text(account).include?(account_last4_hint) + end + + score += 0.05.to_d if statement.statement_currency == account.currency + [ score, 1.to_d ].min + end + + def institution_hint + @institution_hint ||= statement.institution_name_hint.to_s.downcase.squish.presence + end + + def account_name_hint + @account_name_hint ||= statement.account_name_hint.to_s.downcase.squish.presence + end + + def account_last4_hint + @account_last4_hint ||= statement.account_last4_hint.to_s.downcase.squish.presence + end + + def account_text(account) + [ + account.name, + account.institution_name, + account.institution_domain + ].compact.join(" ").downcase + end + + def account_sensitive_match_text(account) + # Exclude user-controlled account notes from matching hints. Statement + # matching should use conservative account metadata, not free-form prose + # that can accidentally manufacture a last-four match. + [ + account.name, + account.institution_name + ].compact.join(" ").downcase + end +end diff --git a/app/models/account_statement/coverage.rb b/app/models/account_statement/coverage.rb new file mode 100644 index 000000000..225df183c --- /dev/null +++ b/app/models/account_statement/coverage.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +class AccountStatement::Coverage + Month = Struct.new(:date, :status, :statements, :ambiguous_statements, keyword_init: true) do + def expected? + status != "not_expected" + end + + def covered? + status == "covered" + end + + def missing? + status == "missing" + end + + def duplicate? + status == "duplicate" + end + + def ambiguous? + status == "ambiguous" + end + + def mismatched? + status == "mismatched" + end + + def not_expected? + status == "not_expected" + end + end + + attr_reader :account, :start_month, :end_month, :expected_start_month, :expected_end_month, :selected_year, :available_years + + class << self + def for_year(account, year) + expected_end_month = default_expected_end_month + expected_start_month = default_expected_start_month(account, fallback_end_month: expected_end_month) + available_years = years_between(expected_start_month, expected_end_month) + selected_year = resolve_year_value(year, available_years) + + new( + account, + start_month: Date.new(selected_year, 1, 1), + end_month: Date.new(selected_year, 12, 1), + expected_start_month: expected_start_month, + expected_end_month: expected_end_month, + selected_year: selected_year, + available_years: available_years + ) + end + + def years_for(account) + expected_end_month = default_expected_end_month + expected_start_month = default_expected_start_month(account, fallback_end_month: expected_end_month) + + years_between(expected_start_month, expected_end_month) + end + + def resolve_year(account, year) + resolve_year_value(year, years_for(account)) + end + + def default_expected_end_month + Date.current.prev_month.beginning_of_month + end + + def default_expected_start_month(account, fallback_end_month: default_expected_end_month) + candidates = [ + account.entries.minimum(:date), + account.balances.minimum(:date), + account.account_statements.where.not(period_start_on: nil).minimum(:period_start_on), + account.family.account_statements.unmatched.where(suggested_account: account).where.not(period_start_on: nil).minimum(:period_start_on) + ].compact + + start_month = (candidates.min || fallback_end_month.advance(months: -11)).to_date.beginning_of_month + start_month > fallback_end_month ? fallback_end_month : start_month + end + + private + + def years_between(start_month, end_month) + (start_month.year..end_month.year).to_a.reverse + end + + def resolve_year_value(year, available_years) + requested_year = year.to_i if year.present? + + available_years.include?(requested_year) ? requested_year : available_years.first + end + end + + def initialize(account, start_month: nil, end_month: nil, expected_start_month: nil, expected_end_month: nil, selected_year: nil, available_years: nil) + raise ArgumentError, "account is required" if account.nil? + + @account = account + @expected_end_month = (expected_end_month || end_month || self.class.default_expected_end_month).to_date.beginning_of_month + resolved_expected_start_month = (expected_start_month || start_month || self.class.default_expected_start_month(account, fallback_end_month: @expected_end_month)).to_date.beginning_of_month + @expected_start_month = resolved_expected_start_month > @expected_end_month ? @expected_end_month : resolved_expected_start_month + @start_month = (start_month || @expected_start_month).to_date.beginning_of_month + @end_month = (end_month || @expected_end_month).to_date.beginning_of_month + @selected_year = selected_year + @available_years = available_years || self.class.years_for(account) + end + + def months + @months ||= begin + current = start_month + result = [] + + while current <= end_month + result << build_month(current) + current = current.next_month + end + + result + end + end + + def summary_counts + months.group_by(&:status).transform_values(&:count) + end + + private + + def build_month(month) + return Month.new(date: month, status: "not_expected", statements: [], ambiguous_statements: []) unless expected_month?(month) + + linked_statements = statements_covering(linked_statement_scope, month) + ambiguous_statements = statements_covering(ambiguous_statement_scope, month) + + status = if linked_statements.size > 1 + "duplicate" + elsif linked_statements.any? { |statement| statement.reconciliation_mismatched?(balance_lookup: balance_lookup) } + "mismatched" + elsif linked_statements.one? + "covered" + elsif ambiguous_statements.any? + "ambiguous" + else + "missing" + end + + Month.new(date: month, status: status, statements: linked_statements, ambiguous_statements: ambiguous_statements) + end + + def expected_month?(month) + month >= expected_start_month && month <= expected_end_month + end + + def linked_statement_scope + @linked_statement_scope ||= account.account_statements + .where("period_start_on <= ? AND period_end_on >= ?", end_month.end_of_month, start_month) + .ordered + .to_a + end + + def ambiguous_statement_scope + @ambiguous_statement_scope ||= account.family.account_statements + .unmatched + .where(suggested_account: account) + .where("period_start_on <= ? AND period_end_on >= ?", end_month.end_of_month, start_month) + .ordered + .to_a + end + + def statements_covering(statements, month) + month_start = month.to_date.beginning_of_month + month_end = month_start.end_of_month + + statements.select do |statement| + statement.period_start_on.present? && + statement.period_end_on.present? && + statement.period_start_on <= month_end && + statement.period_end_on >= month_start + end + end + + def balance_lookup + @balance_lookup ||= begin + currencies = linked_statement_scope.map(&:statement_currency).compact.uniq + dates = linked_statement_scope.flat_map { |statement| [ statement.period_start_on, statement.period_end_on ] }.compact.uniq + balances = if currencies.any? && dates.any? + account.balances.where(currency: currencies, date: dates).to_a + else + [] + end + by_key = balances.index_by { |balance| [ balance.date, balance.currency ] } + + ->(date, currency) { by_key[[ date, currency ]] } + end + end +end diff --git a/app/models/account_statement/metadata_detector.rb b/app/models/account_statement/metadata_detector.rb new file mode 100644 index 000000000..e8079d5b8 --- /dev/null +++ b/app/models/account_statement/metadata_detector.rb @@ -0,0 +1,221 @@ +# frozen_string_literal: true + +require "csv" +require "stringio" + +class AccountStatement::MetadataDetector + DATE_PATTERNS = [ + /(?\d{4})[-_\.](?0?[1-9]|1[0-2]) + | + (?jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?) + [-_\s\.]+(?\d{4}) + ) + (?![a-z0-9]) + /ix.freeze + + LAST4_PATTERN = /(?:^|[^a-z0-9])(?:x{2,}|ending|last\s*4|acct|account|card)[^\d]*(\d{4})(?=\D|$)/i.freeze + GENERIC_FILENAME_HINTS = [ + "statement", + "statements", + "bank statement", + "bank statements", + "account statement", + "account statements", + "credit card statement", + "card statement" + ].freeze + MAX_CSV_COLUMNS = 100 + MAX_CSV_DATE_SAMPLES = 250 + MAX_CSV_SAMPLE_BYTES = 256 + + attr_reader :statement, :content + + def initialize(statement, content:) + @statement = statement + @content = content + end + + def apply + output = statement.sanitized_parser_output || {} + metadata_sources = [] + + if detect_from_filename + metadata_sources << "filename" + end + + if statement.csv? && detect_from_csv(output) + metadata_sources << "csv_dates" + elsif statement.xlsx? + output["spreadsheet_detection"] = "filename_only" + elsif statement.pdf? + output["pdf_detection"] = "filename_only" + end + + output["metadata_sources"] = metadata_sources + statement.sanitized_parser_output = output + statement.parser_confidence ||= if metadata_sources.include?("csv_dates") + 0.65 + elsif metadata_sources.any? + 0.45 + else + 0.1 + end + end + + private + + def detect_from_filename + basename = File.basename(statement.filename.to_s, ".*") + return false if basename.blank? + + detected = false + + if (last4 = basename.match(LAST4_PATTERN)&.captures&.first) + statement.account_last4_hint ||= last4 + detected = true + end + + dates = DATE_PATTERNS.flat_map { |pattern| basename.scan(pattern) } + .map { |match| Array(match).first } + .filter_map { |value| parse_date(value) } + .uniq + .sort + + if dates.size >= 2 + statement.period_start_on ||= dates.first + statement.period_end_on ||= dates.last + detected = true + elsif dates.size == 1 + statement.period_start_on ||= dates.first.beginning_of_month + statement.period_end_on ||= dates.first.end_of_month + detected = true + elsif (month_date = parse_month_from_filename(basename)) + statement.period_start_on ||= month_date.beginning_of_month + statement.period_end_on ||= month_date.end_of_month + detected = true + end + + hint = basename + .gsub(LAST4_PATTERN, "") + .gsub(/\b\d{4}[-_\.]\d{1,2}(?:[-_\.]\d{1,2})?\b/, "") + .gsub(/\b\d{8}\b/, "") + .tr("_-", " ") + .gsub(/\b(?:19|20)\d{2}\b/, "") + .gsub(/\b(?:0?[1-9]|1[0-2])\b/, "") + .squish + .presence + + if (meaningful_hint = meaningful_filename_hint(hint)) + statement.institution_name_hint ||= meaningful_hint + statement.account_name_hint ||= meaningful_hint + detected = true + end + + detected + end + + def detect_from_csv(output) + csv = CSV.new(StringIO.new(content.to_s), headers: true, liberal_parsing: true) + first_row = csv.shift + return false if first_row.blank? + + headers = first_row.headers.compact.map(&:to_s) + return false if headers.size > MAX_CSV_COLUMNS + + date_header = headers.find { |header| csv_sample_text(header).to_s.match?(/date|posted|transaction/i) } + return false if date_header.blank? + + samples = [ csv_sample_text(first_row[date_header]) ].compact_blank + csv.each do |row| + break if samples.size >= MAX_CSV_DATE_SAMPLES + + sample = csv_sample_text(row[date_header]) + samples << sample if sample.present? + end + return false if samples.blank? + + date_format = Import.detect_date_format(samples) + dates = samples.filter_map { |sample| parse_date_with_format(sample, date_format) }.uniq.sort + return false if dates.blank? + + statement.period_start_on ||= dates.first + statement.period_end_on ||= dates.last + output["csv"] = { + "date_header" => date_header.to_s, + "date_format" => date_format, + "rows_sampled" => samples.size + } + true + rescue CSV::MalformedCSVError + false + end + + def csv_sample_text(value) + text = value.to_s + return nil if text.bytesize > MAX_CSV_SAMPLE_BYTES + + text + end + + def meaningful_filename_hint(hint) + return nil if hint.blank? + + normalized = hint.downcase.gsub(/[^a-z0-9]+/, " ").squish + without_generic_words = normalized + .gsub(/\b(?:bank|account|card|credit|debit|statement|statements)\b/, "") + .squish + + return nil if GENERIC_FILENAME_HINTS.include?(normalized) || without_generic_words.blank? + + hint + end + + def parse_date(value) + text = value.to_s.tr("_", "-") + date = if text.match?(/\A\d{8}\z/) + Date.strptime(text, "%Y%m%d") + else + Date.parse(text) + end + + AccountStatement::MetadataDetector.reasonable_date?(date) ? date : nil + rescue Date::Error, ArgumentError + nil + end + + def parse_date_with_format(value, format) + date = Date.strptime(value.to_s, format) + AccountStatement::MetadataDetector.reasonable_date?(date) ? date : nil + rescue Date::Error, ArgumentError + nil + end + + def parse_month_from_filename(basename) + match = basename.match(MONTH_PATTERN) + return nil unless match + + year = (match[:year_first] || match[:year_second]).to_i + month = if match[:month_first] + match[:month_first].to_i + else + Date::ABBR_MONTHNAMES.index(match[:month_name][0, 3].capitalize) + end + + date = Date.new(year, month, 1) + AccountStatement::MetadataDetector.reasonable_date?(date) ? date : nil + rescue Date::Error, NoMethodError + nil + end + + def self.reasonable_date?(date) + Import.reasonable_date_range.cover?(date) + end +end diff --git a/app/models/family.rb b/app/models/family.rb index fb211d78c..168773701 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -28,6 +28,7 @@ class Family < ApplicationRecord has_many :imports, dependent: :destroy has_many :family_exports, dependent: :destroy + has_many :account_statements, dependent: :destroy has_many :entries, through: :accounts has_many :transactions, through: :accounts diff --git a/app/views/account_statements/index.html.erb b/app/views/account_statements/index.html.erb new file mode 100644 index 000000000..dba3f1677 --- /dev/null +++ b/app/views/account_statements/index.html.erb @@ -0,0 +1,170 @@ +<%= content_for :page_title, t(".title") %> + +<%= settings_section title: t(".title") do %> +
+
+
+
+

<%= t(".upload_title") %>

+

<%= t(".upload_description") %>

+
+
+

<%= t(".storage_used") %>

+

<%= number_to_human_size(@total_storage_bytes) %>

+
+
+ + <%= styled_form_with url: account_statements_path, scope: :account_statement, multipart: true, class: "grid gap-3 lg:grid-cols-[1fr_16rem_auto] lg:items-end" do |form| %> +
+ <%= form.label :files, t("account_statements.form.files_label"), class: "form-field__label" %> + <%= form.file_field :files, + multiple: true, + accept: AccountStatement::ACCEPTED_FILE_EXTENSIONS.join(","), + class: "form-field__input" %> +

<%= t("account_statements.form.files_hint", max_size: AccountStatement::MAX_FILE_SIZE / 1.megabyte) %>

+
+ <%= form.collection_select :account_id, + @accounts, + :id, + :name, + { include_blank: t(".leave_unmatched"), label: t(".account_label") }, + { data: { testid: "account-statement-account-select" } } %> +
+ <%= form.submit t("account_statements.form.inbox_upload") %> +
+ <% end %> +
+ +
+
+

<%= t(".unmatched_title") %>

+ · +

<%= @unmatched_pagy.count %>

+
+ +
+ <% if @unmatched_statements.any? %> + + + + + + + + + + + <% @unmatched_statements.each do |statement| %> + + + + + + + <% end %> + +
<%= t("account_statements.table.file") %><%= t("account_statements.table.period") %><%= t("account_statements.table.suggestion") %><%= t("account_statements.table.actions") %>
+ <%= link_to account_statement_path(statement), class: "flex items-center gap-2 min-w-0 text-sm font-medium text-primary hover:underline" do %> + <%= icon(account_statement_file_icon(statement), size: "sm") %> + <%= statement.filename %> + <% end %> +

<%= number_to_human_size(statement.byte_size) %>

+
<%= account_statement_period(statement) %> + <% if statement.suggested_account.present? %> +
+

<%= statement.suggested_account.name %>

+

<%= t(".confidence", confidence: number_to_percentage(statement.match_confidence.to_d * 100, precision: 0)) %>

+
+ <% else %> + <%= t(".no_suggestion") %> + <% end %> +
+
+ <% if statement.suggested_account.present? %> + <%= button_to link_account_statement_path(statement), + method: :patch, + params: { account_id: statement.suggested_account_id }, + class: "flex items-center", + aria: { label: t("account_statements.table.link_suggestion") } do %> + <%= icon("link", class: "w-5 h-5 text-primary") %> + <% end %> + <%= button_to reject_account_statement_path(statement), + method: :patch, + class: "flex items-center", + aria: { label: t("account_statements.table.reject") } do %> + <%= icon("x", class: "w-5 h-5 text-secondary") %> + <% end %> + <% end %> + <%= link_to account_statement_path(statement), aria: { label: t("account_statements.table.view") } do %> + <%= icon("eye", class: "w-5 h-5 text-primary") %> + <% end %> +
+
+ <% else %> +

<%= t(".empty_unmatched") %>

+ <% end %> +
+ <% if @unmatched_pagy.pages > 1 %> +
+ <%= render "shared/pagination", pagy: @unmatched_pagy %> +
+ <% end %> +
+ +
+
+

<%= t(".linked_title") %>

+ · +

<%= @linked_pagy.count %>

+
+ +
+ <% if @linked_statements.any? %> + + + + + + + + + + + <% @linked_statements.each do |statement| %> + + + + + + + <% end %> + +
<%= t("account_statements.table.file") %><%= t("account_statements.table.account") %><%= t("account_statements.table.period") %><%= t("account_statements.table.actions") %>
+ <%= link_to account_statement_path(statement), class: "flex items-center gap-2 min-w-0 text-sm font-medium text-primary hover:underline" do %> + <%= icon(account_statement_file_icon(statement), size: "sm") %> + <%= statement.filename %> + <% end %> + <%= statement.account&.name %><%= account_statement_period(statement) %> +
+ <%= link_to account_statement_path(statement), aria: { label: t("account_statements.table.view") } do %> + <%= icon("eye", class: "w-5 h-5 text-primary") %> + <% end %> + <% if statement.original_file.attached? %> + <%= link_to rails_blob_path(statement.original_file, disposition: "attachment"), aria: { label: t("account_statements.table.download") } do %> + <%= icon("download", class: "w-5 h-5 text-primary") %> + <% end %> + <% end %> +
+
+ <% else %> +

<%= t(".empty_linked") %>

+ <% end %> +
+ <% if @linked_pagy.pages > 1 %> +
+ <%= render "shared/pagination", pagy: @linked_pagy %> +
+ <% end %> +
+
+<% end %> diff --git a/app/views/account_statements/show.html.erb b/app/views/account_statements/show.html.erb new file mode 100644 index 000000000..e320fc79a --- /dev/null +++ b/app/views/account_statements/show.html.erb @@ -0,0 +1,180 @@ +<%= content_for :page_title, @statement.filename %> + +<%= settings_section title: t(".title") do %> +
+
+
+
+
+ <%= icon(account_statement_file_icon(@statement), size: "sm") %> +

<%= @statement.filename %>

+
+
+ <%= account_statement_status_badge(@statement) %> + <%= number_to_human_size(@statement.byte_size) %> + <%= @statement.content_type %> +
+
+ +
+ <% if @statement.original_file.attached? %> + <%= link_to rails_blob_path(@statement.original_file, disposition: "attachment"), + class: "inline-flex items-center gap-2 text-sm font-medium text-primary hover:text-primary-hover" do %> + <%= icon("download", size: "sm") %> + <%= t(".download") %> + <% end %> + <% end %> + <% if @can_manage_statement %> + <%= button_to account_statement_path(@statement), + method: :delete, + class: "inline-flex items-center gap-2 text-sm font-medium text-destructive", + data: { turbo_confirm: CustomConfirm.for_resource_deletion("statement") } do %> + <%= icon("trash-2", size: "sm", color: "destructive") %> + <%= t(".delete") %> + <% end %> + <% end %> +
+
+
+ +
+
+

<%= t(".metadata_title") %>

+ + <% if @can_manage_statement %> + <%= styled_form_with model: @statement, url: account_statement_path(@statement), method: :patch, class: "space-y-4" do |form| %> +
+ <%= form.text_field :institution_name_hint, label: t(".institution_name_hint") %> + <%= form.text_field :account_name_hint, label: t(".account_name_hint") %> + <%= form.text_field :account_last4_hint, label: t(".account_last4_hint") %> + <%= form.select :currency, + account_statement_currency_options(@statement), + { label: t(".currency"), selected: @statement.statement_currency } %> + <%= form.date_field :period_start_on, label: t(".period_start_on") %> + <%= form.date_field :period_end_on, label: t(".period_end_on") %> + <%= form.number_field :opening_balance, label: t(".opening_balance"), step: "0.01" %> + <%= form.number_field :closing_balance, label: t(".closing_balance"), step: "0.01" %> +
+ + <%= form.submit t(".save") %> + <% end %> + <% else %> +
+
+
<%= t(".account_label") %>
+
<%= @statement.account&.name || t(".unmatched_account") %>
+
+
+
<%= t(".institution_name_hint") %>
+
<%= @statement.institution_name_hint.presence || t(".unknown_value") %>
+
+
+
<%= t(".account_name_hint") %>
+
<%= @statement.account_name_hint.presence || t(".unknown_value") %>
+
+
+
<%= t(".account_last4_hint") %>
+
<%= @statement.account_last4_hint.presence || t(".unknown_value") %>
+
+
+
<%= t(".currency") %>
+
<%= @statement.statement_currency %>
+
+
+
<%= t("account_statements.table.period") %>
+
<%= account_statement_period(@statement) %>
+
+
+
<%= t(".opening_balance") %>
+
<%= account_statement_balance_label(@statement, :opening_balance) %>
+
+
+
<%= t(".closing_balance") %>
+
<%= account_statement_balance_label(@statement, :closing_balance) %>
+
+
+ <% end %> +
+ +
+
+

<%= t(".linking_title") %>

+ + <% if @statement.account.present? %> +

+ <%= t(".linked_to", account: @statement.account.name) %> +

+ <% if @can_manage_statement %> + <%= button_to unlink_account_statement_path(@statement), + method: :patch, + class: "inline-flex items-center gap-2 text-sm font-medium text-primary" do %> + <%= icon("unlink", size: "sm") %> + <%= t(".unlink") %> + <% end %> + <% end %> + <% elsif @statement.suggested_account.present? %> +

+ <%= t(".suggested_account", account: @statement.suggested_account.name, confidence: number_to_percentage(@statement.match_confidence.to_d * 100, precision: 0)) %> +

+ <% if @can_manage_statement %> +
+ <%= button_to link_account_statement_path(@statement), + method: :patch, + params: { account_id: @statement.suggested_account_id }, + class: "inline-flex items-center gap-2 text-sm font-medium text-primary" do %> + <%= icon("link", size: "sm") %> + <%= t(".link_suggestion") %> + <% end %> + <%= button_to reject_account_statement_path(@statement), + method: :patch, + class: "inline-flex items-center gap-2 text-sm font-medium text-secondary" do %> + <%= icon("x", size: "sm") %> + <%= t(".reject") %> + <% end %> +
+ <% end %> + <% else %> +

<%= t(".no_suggestion") %>

+ <% end %> +
+ +
+

<%= t(".reconciliation_title") %>

+ + <% if @reconciliation_checks.any? %> +
+ <% @reconciliation_checks.each do |check| %> +
+
+

<%= account_statement_reconciliation_label(check) %>

+ <% if check[:status] == "matched" %> + <%= render "shared/badge", color: "success" do %><%= t("account_statements.reconciliation.matched") %><% end %> + <% else %> + <%= render "shared/badge", color: "error" do %><%= t("account_statements.reconciliation.mismatched") %><% end %> + <% end %> +
+
+
+
<%= t(".statement_amount") %>
+
<%= Money.new(check[:statement_amount], @statement.statement_currency).format %>
+
+
+
<%= t(".ledger_amount") %>
+
<%= Money.new(check[:ledger_amount], @statement.statement_currency).format %>
+
+
+
<%= t(".difference") %>
+
<%= Money.new(check[:difference], @statement.statement_currency).format %>
+
+
+
+ <% end %> +
+ <% else %> +

<%= t(".reconciliation_unavailable") %>

+ <% end %> +
+
+
+
+<% end %> diff --git a/app/views/accounts/show.html.erb b/app/views/accounts/show.html.erb index 170f12986..a2c663b6b 100644 --- a/app/views/accounts/show.html.erb +++ b/app/views/accounts/show.html.erb @@ -2,7 +2,11 @@ account: @account, chart_view: @chart_view, chart_period: @period, - active_tab: @tab + active_tab: @tab, + statement_coverage: @statement_coverage, + statements: @account_statements, + reconciliation_statuses: @statement_reconciliation_statuses, + can_manage_statements: @can_manage_statements ) do |account_page| %> <%= account_page.with_activity_feed(feed_data: @activity_feed_data, pagy: @pagy, search: @q[:search]) %> <% end %> diff --git a/app/views/accounts/show/_menu.html.erb b/app/views/accounts/show/_menu.html.erb index 45a39fe18..cb577369e 100644 --- a/app/views/accounts/show/_menu.html.erb +++ b/app/views/accounts/show/_menu.html.erb @@ -6,6 +6,7 @@ <% menu.with_item(variant: "link", text: "Edit", href: edit_account_path(account), icon: "pencil-line", data: { turbo_frame: :modal }) %> <% end %> <% menu.with_item(variant: "link", text: "Sharing", href: account_sharing_path(account), icon: "users", data: { turbo_frame: :modal }) %> + <% menu.with_item(variant: "link", text: t(".statements"), href: account_path(account, tab: "statements"), icon: "archive") %> <% if permission.in?([ :owner, :full_control ]) %> <% if account.supports_trades? %> diff --git a/app/views/accounts/show/_statements.html.erb b/app/views/accounts/show/_statements.html.erb new file mode 100644 index 000000000..3a5d4a11a --- /dev/null +++ b/app/views/accounts/show/_statements.html.erb @@ -0,0 +1,124 @@ +<%# locals: (account:, coverage:, statements:, reconciliation_statuses:, can_manage_statements:) %> + +
+
+
+
+

<%= t("account_statements.account_tab.coverage_title") %>

+

<%= t("account_statements.account_tab.coverage_description") %>

+

<%= account_statement_coverage_range(coverage) %>

+
+
+ <%= form_with url: account_path(account), method: :get, data: { controller: "auto-submit-form" } do |form| %> + <%= form.hidden_field :tab, value: "statements" %> + <%= form.select :statement_year, + coverage.available_years.map { |year| [ year, year ] }, + { selected: coverage.selected_year }, + class: "bg-container border border-secondary rounded-lg px-3 py-2 text-sm pr-7 cursor-pointer text-primary focus:outline-hidden focus:ring-0", + data: { "auto-submit-form-target": "auto" }, + aria: { label: t("account_statements.account_tab.year_label") } %> + <% end %> + <%= link_to account_statements_path, + class: "inline-flex items-center gap-2 text-sm font-medium text-primary hover:text-primary-hover" do %> + <%= icon("inbox", size: "sm") %> + <%= t("account_statements.account_tab.open_inbox") %> + <% end %> +
+
+ +
+ <% coverage.months.each do |month| %> + + <% end %> +
+
+ + <% if can_manage_statements %> +
+ <%= styled_form_with url: account_statements_path, scope: :account_statement, multipart: true, class: "space-y-3" do |form| %> + <%= form.hidden_field :account_id, value: account.id %> +
+ <%= form.label :files, t("account_statements.form.files_label"), class: "form-field__label" %> + <%= form.file_field :files, + multiple: true, + accept: AccountStatement::ACCEPTED_FILE_EXTENSIONS.join(","), + class: "form-field__input" %> +

<%= t("account_statements.form.files_hint", max_size: AccountStatement::MAX_FILE_SIZE / 1.megabyte) %>

+
+ <%= form.submit t("account_statements.form.account_upload") %> + <% end %> +
+ <% end %> + +
+
+

<%= t("account_statements.account_tab.statements_title") %>

+ · +

<%= statements.size %>

+
+ +
+ <% if statements.any? %> + + + + + + + + + + + <% statements.each do |statement| %> + + + + + + + <% end %> + +
<%= t("account_statements.table.file") %><%= t("account_statements.table.period") %><%= t("account_statements.table.reconciliation") %><%= t("account_statements.table.actions") %>
+ <%= link_to account_statement_path(statement), class: "flex items-center gap-2 min-w-0 text-sm font-medium text-primary hover:underline" do %> + <%= icon(account_statement_file_icon(statement), size: "sm") %> + <%= statement.filename %> + <% end %> +

<%= number_to_human_size(statement.byte_size) %>

+
<%= account_statement_period(statement) %> + <% case reconciliation_statuses[statement.id] %> + <% when "matched" %> + <%= render "shared/badge", color: "success" do %><%= t("account_statements.reconciliation.matched") %><% end %> + <% when "mismatched" %> + <%= render "shared/badge", color: "error" do %><%= t("account_statements.reconciliation.mismatched") %><% end %> + <% else %> + <%= render "shared/badge" do %><%= t("account_statements.reconciliation.unavailable") %><% end %> + <% end %> + +
+ <%= link_to account_statement_path(statement), aria: { label: t("account_statements.table.view") } do %> + <%= icon("eye", class: "w-5 h-5 text-primary") %> + <% end %> + <% if statement.original_file.attached? %> + <%= link_to rails_blob_path(statement.original_file, disposition: "attachment"), aria: { label: t("account_statements.table.download") } do %> + <%= icon("download", class: "w-5 h-5 text-primary") %> + <% end %> + <% end %> + <% if can_manage_statements %> + <%= button_to unlink_account_statement_path(statement), + method: :patch, + class: "flex items-center", + aria: { label: t("account_statements.table.unlink") } do %> + <%= icon("unlink", class: "w-5 h-5 text-secondary") %> + <% end %> + <% end %> +
+
+ <% else %> +

<%= t("account_statements.account_tab.empty") %>

+ <% end %> +
+
+
diff --git a/app/views/accounts/show/_statements_frame.html.erb b/app/views/accounts/show/_statements_frame.html.erb new file mode 100644 index 000000000..acccc4190 --- /dev/null +++ b/app/views/accounts/show/_statements_frame.html.erb @@ -0,0 +1,10 @@ +<%# locals: (account:, coverage:, statements:, reconciliation_statuses:, can_manage_statements:) %> + +<%= turbo_frame_tag dom_id(account, :statements_tab) do %> + <%= render "accounts/show/statements", + account: account, + coverage: coverage, + statements: statements, + reconciliation_statuses: reconciliation_statuses, + can_manage_statements: can_manage_statements %> +<% end %> diff --git a/app/views/settings/_settings_nav.html.erb b/app/views/settings/_settings_nav.html.erb index 94df67ae2..19bd4a241 100644 --- a/app/views/settings/_settings_nav.html.erb +++ b/app/views/settings/_settings_nav.html.erb @@ -19,7 +19,8 @@ nav_sections = [ { label: t(".tags_label"), path: tags_path, icon: "tags" }, { label: t(".rules_label"), path: rules_path, icon: "git-branch" }, { label: t(".merchants_label"), path: family_merchants_path, icon: "store" }, - { label: t(".recurring_transactions_label"), path: recurring_transactions_path, icon: "repeat" } + { label: t(".recurring_transactions_label"), path: recurring_transactions_path, icon: "repeat" }, + { label: t(".statement_vault_label"), path: account_statements_path, icon: "archive", if: Current.user&.admin? } ] }, ( diff --git a/config/brakeman.ignore b/config/brakeman.ignore index ca044eb6c..d6b23c8f8 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -46,29 +46,6 @@ ], "note": "" }, - { - "warning_type": "Mass Assignment", - "warning_code": 105, - "fingerprint": "85e2c11853dd6c69b1953a6ec3ad661cd0ce3df55e4e5beff92365b6ed601171", - "check_name": "PermitAttributes", - "message": "Potentially dangerous key allowed for mass assignment", - "file": "app/controllers/api/v1/transactions_controller.rb", - "line": 255, - "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", - "code": "params.require(:transaction).permit(:account_id, :date, :amount, :name, :description, :notes, :currency, :category_id, :merchant_id, :nature, :tag_ids => ([]))", - "render_path": null, - "location": { - "type": "method", - "class": "Api::V1::TransactionsController", - "method": "transaction_params" - }, - "user_input": ":account_id", - "confidence": "High", - "cwe_id": [ - 915 - ], - "note": "account_id is properly validated in create action - line 79 ensures account belongs to user's family: family.accounts.find(transaction_params[:account_id])" - }, { "warning_type": "Mass Assignment", "warning_code": 105, diff --git a/config/initializers/active_storage_authorization.rb b/config/initializers/active_storage_authorization.rb index 7766dc3c3..c856faa08 100644 --- a/config/initializers/active_storage_authorization.rb +++ b/config/initializers/active_storage_authorization.rb @@ -2,35 +2,95 @@ Rails.application.config.to_prepare do module ActiveStorageAttachmentAuthorization extend ActiveSupport::Concern + PROTECTED_RECORD_TYPES = %w[Transaction AccountStatement].freeze included do include Authentication - before_action :authorize_transaction_attachment, if: :transaction_attachment? + before_action :authorize_protected_attachment end private - def authorize_transaction_attachment - attachment = ActiveStorage::Attachment.find_by(blob: authorized_blob) - return unless attachment&.record_type == "Transaction" + def authorize_protected_attachment + # Direct uploads create unattached blobs; model/controller code authorizes the later attachment. + return if is_a?(ActiveStorage::DirectUploadsController) + return unless authorized_blob - transaction = attachment.record + attachments = authorized_attachments + raise ActiveRecord::RecordNotFound if attachments.empty? - # Check if current user has access to this transaction's family - unless Current.family == transaction.entry.account.family - raise ActiveRecord::RecordNotFound + protected_attachments = attachments.select { |attachment| attachment.record_type.in?(PROTECTED_RECORD_TYPES) } + return if protected_attachments.empty? + return if protected_attachments.all? { |attachment| protected_attachment_authorized?(attachment) } + + raise ActiveRecord::RecordNotFound + end + + def protected_attachment_authorized?(attachment) + case attachment.record_type + when "Transaction" + transaction_attachment_authorized?(attachment) + when "AccountStatement" + account_statement_attachment_authorized?(attachment) + else + false end end - def transaction_attachment? - return false unless authorized_blob + def transaction_attachment_authorized?(attachment) + transaction = attachment.record + return false if transaction.nil? - attachment = ActiveStorage::Attachment.find_by(blob: authorized_blob) - attachment&.record_type == "Transaction" + Current.family == transaction.entry.account.family + rescue ActiveRecord::RecordNotFound, NoMethodError + false + end + + def account_statement_attachment_authorized?(attachment) + statement = attachment.record + return false if statement.nil? + + statement.viewable_by?(Current.user) + rescue ActiveRecord::RecordNotFound + false + end + + def authorized_attachments + return nil unless authorized_blob + + @authorized_attachments ||= ActiveStorage::Attachment.where(blob: authorized_blob).to_a end def authorized_blob - @blob || @representation&.blob + @blob || @representation&.blob || disk_service_blob + end + + def disk_service_blob + return nil unless is_a?(ActiveStorage::DiskController) && action_name == "show" + + key = decode_verified_key&.fetch(:key, nil) + return nil if key.blank? + + blob_key = key.to_s[%r{\Avariants/([^/]+)/}, 1] || key + ActiveStorage::Blob.find_by(key: blob_key) + rescue ActiveStorage::InvalidKeyError + nil + end + + def new_session_url + Rails.application.routes.url_helpers.new_session_url(active_storage_auth_url_options) + end + + def new_registration_url + Rails.application.routes.url_helpers.new_registration_url(active_storage_auth_url_options) + end + + def active_storage_auth_url_options + { + protocol: request.protocol, + host: request.host, + port: request.optional_port + }.compact end end @@ -38,8 +98,10 @@ Rails.application.config.to_prepare do ActiveStorage::Blobs::RedirectController, ActiveStorage::Blobs::ProxyController, ActiveStorage::Representations::RedirectController, - ActiveStorage::Representations::ProxyController - ].each do |controller| + ActiveStorage::Representations::ProxyController, + (ActiveStorage::DiskController if defined?(ActiveStorage::DiskController)), + (ActiveStorage::DirectUploadsController if defined?(ActiveStorage::DirectUploadsController)) + ].compact.each do |controller| controller.include ActiveStorageAttachmentAuthorization end end diff --git a/config/locales/models/account_statement/en.yml b/config/locales/models/account_statement/en.yml new file mode 100644 index 000000000..d801052d4 --- /dev/null +++ b/config/locales/models/account_statement/en.yml @@ -0,0 +1,30 @@ +--- +en: + activerecord: + attributes: + account_statement: + account: Account + account_last4_hint: Account last four + account_name_hint: Account name hint + closing_balance: Closing balance + content_sha256: Content digest + currency: Currency + filename: Filename + institution_name_hint: Institution hint + opening_balance: Opening balance + original_file: Statement file + period_end_on: Period end + period_start_on: Period start + errors: + models: + account_statement: + attributes: + checksum: + duplicate_statement_file: has already been uploaded for this family + content_sha256: + duplicate_statement_file: has already been uploaded for this family + original_file: + invalid_format: must be a PDF, CSV, or XLSX file + too_large: is too large. Maximum size is %{max_mb}MB + period_end_on: + on_or_after_start: must be on or after the period start diff --git a/config/locales/views/account_statements/en.yml b/config/locales/views/account_statements/en.yml new file mode 100644 index 000000000..78e21ae57 --- /dev/null +++ b/config/locales/views/account_statements/en.yml @@ -0,0 +1,116 @@ +--- +en: + account_statements: + account_tab: + coverage_title: Statement coverage + coverage_description: Historical months backed by uploaded statements and balance checks. + coverage_range: "%{start} - %{end}" + empty: No statements linked to this account yet. + open_inbox: Inbox + statements_title: Statements + year_label: Coverage year + balance: + unknown: Unknown + coverage: + status: + ambiguous: Ambiguous + covered: Covered + duplicate: Duplicate + mismatched: Mismatched + missing: Missing + not_expected: Not expected + create: + duplicates: + one: 1 duplicate statement was skipped. + other: "%{count} duplicate statements were skipped." + invalid_file_type: Upload a PDF, CSV, or XLSX statement under the size limit. + no_files: Select at least one statement file. + success: + one: 1 statement uploaded. + other: "%{count} statements uploaded." + destroy: + failure: Statement could not be deleted. + success: Statement deleted. + form: + account_upload: Upload statement + files_hint: PDF, CSV, or XLSX. %{max_size}MB max per file. + files_label: Statement files + inbox_upload: Upload + index: + account_label: Account + confidence: "%{confidence} match" + empty_linked: No linked statements yet. + empty_unmatched: The statement inbox is clear. + leave_unmatched: Leave unmatched + linked_title: Linked statements + no_suggestion: No suggestion + storage_used: Storage used + title: Statement Vault + unmatched_title: Unmatched inbox + upload_description: Upload statements to the inbox, or choose an account to link immediately. + upload_title: Upload statements + link: + no_account: Choose an account before linking this statement. + success: Linked statement to %{account}. + period: + unknown: Period unknown + reconciliation: + checks: + closing_balance: Closing balance + opening_balance: Opening balance + period_movement: Period movement + unknown_check: Unknown check + matched: Matched + mismatched: Mismatched + unavailable: Not checked + reject: + success: Statement match rejected. + show: + account_label: Account + account_last4_hint: Account last four + account_name_hint: Account name hint + closing_balance: Closing balance + currency: Currency + delete: Delete + difference: Difference + download: Download + institution_name_hint: Institution hint + ledger_amount: Sure ledger + linked_to: Linked to %{account}. + linking_title: Account link + link_suggestion: Link suggestion + metadata_title: Statement metadata + no_suggestion: No account suggestion yet. + opening_balance: Opening balance + period_end_on: Period end + period_start_on: Period start + reconciliation_title: Reconciliation + reconciliation_unavailable: Add a statement period and opening or closing balance, then make sure Sure has balance history for those dates. + reject: Reject + save: Save statement + statement_amount: Statement + suggested_account: Suggested account is %{account} (%{confidence} confidence). + title: Statement + unlink: Unlink + unmatched_account: Unmatched inbox + unknown_value: Unknown + status: + linked: Linked + rejected: Rejected + unmatched: Unmatched + table: + account: Account + actions: Actions + download: Download + file: File + link_suggestion: Link suggestion + period: Period + reconciliation: Reconciliation + reject: Reject suggestion + suggestion: Suggestion + unlink: Unlink + view: View + unlink: + success: Statement moved back to the unmatched inbox. + update: + success: Statement updated. diff --git a/config/locales/views/accounts/en.yml b/config/locales/views/accounts/en.yml index 852a2bc28..70ba87ec0 100644 --- a/config/locales/views/accounts/en.yml +++ b/config/locales/views/accounts/en.yml @@ -62,6 +62,11 @@ en: title: What would you like to add? show: limited_fx_history_warning: "Exchange rate history is only available from %{date} onwards. Transactions before this date use approximate currency conversions — this can happen when the FX provider only offers a limited historical window." + tabs: + activity: Activity + holdings: Holdings + overview: Overview + statements: Statements activity: amount: Amount balance: Balance @@ -98,6 +103,7 @@ en: import_trades: Import trades import_transactions: Import transactions manage: Manage accounts + statements: Statements update: success: "%{type} account updated" sidebar: diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index 4ac65a9f3..2904c9a99 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -161,8 +161,10 @@ en: general_section_title: General imports_label: Imports exports_label: Exports + llm_usage_label: LLM Usage logout: Logout merchants_label: Merchants + providers_label: Providers guides_label: Guides other_section_title: More preferences_label: Preferences @@ -171,6 +173,7 @@ en: rules_label: Rules security_label: Security self_hosting_label: Self-Hosting + statement_vault_label: Statement Vault tags_label: Tags transactions_section_title: Transactions whats_new_label: What's new diff --git a/config/locales/views/settings/fr.yml b/config/locales/views/settings/fr.yml index a3cb0aec7..6faca441d 100644 --- a/config/locales/views/settings/fr.yml +++ b/config/locales/views/settings/fr.yml @@ -159,8 +159,10 @@ fr: general_section_title: Général imports_label: Importations exports_label: Exportations + llm_usage_label: Utilisation LLM logout: Se déconnecter merchants_label: Marchands + providers_label: Fournisseurs guides_label: Guides other_section_title: Plus preferences_label: Préférences @@ -169,6 +171,7 @@ fr: rules_label: Règles security_label: Sécurité self_hosting_label: Auto-hébergement + statement_vault_label: Coffre des relevés tags_label: Étiquettes transactions_section_title: Transactions whats_new_label: Dernières nouvelles diff --git a/config/routes.rb b/config/routes.rb index ac61dcecb..36673f125 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -434,6 +434,14 @@ Rails.application.routes.draw do resource :sharing, only: [ :show, :update ], controller: "account_sharings" end + resources :account_statements, only: %i[index show create update destroy] do + member do + patch :link + patch :unlink + patch :reject + end + end + # Convenience routes for polymorphic paths # Example: account_path(Account.new(accountable: Depository.new)) => /depositories/123 direct :edit_account do |model, options| diff --git a/db/migrate/20260505120000_create_account_statements.rb b/db/migrate/20260505120000_create_account_statements.rb new file mode 100644 index 000000000..d31f73cfc --- /dev/null +++ b/db/migrate/20260505120000_create_account_statements.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +class CreateAccountStatements < ActiveRecord::Migration[7.2] + def change + create_table :account_statements, id: :uuid do |t| + t.references :family, null: false, foreign_key: { on_delete: :cascade }, type: :uuid + t.references :account, null: true, type: :uuid, foreign_key: { to_table: :accounts, on_delete: :nullify } + t.references :suggested_account, null: true, type: :uuid, foreign_key: { to_table: :accounts, on_delete: :nullify } + + t.string :filename, null: false, limit: 255 + t.string :content_type, null: false, limit: 100 + t.bigint :byte_size, null: false + t.string :checksum, null: false, limit: 64 + t.string :content_sha256 + t.string :source, null: false, default: "manual_upload" + t.string :upload_status, null: false, default: "stored" + + t.string :institution_name_hint, limit: 200 + t.string :account_name_hint, limit: 200 + t.string :account_last4_hint, limit: 4 + t.date :period_start_on + t.date :period_end_on + t.decimal :opening_balance, precision: 19, scale: 4 + t.decimal :closing_balance, precision: 19, scale: 4 + t.string :currency, limit: 3 + + t.decimal :parser_confidence, precision: 5, scale: 4 + t.decimal :match_confidence, precision: 5, scale: 4 + t.string :review_status, null: false, default: "unmatched" + t.jsonb :sanitized_parser_output, null: false, default: {} + + t.timestamps + + t.index [ :family_id, :checksum ], name: "index_account_statements_on_family_checksum" + t.index [ :family_id, :content_sha256 ], + unique: true, + where: "content_sha256 IS NOT NULL", + name: "index_account_statements_on_family_content_sha256" + t.index [ :family_id, :review_status ], name: "index_account_statements_on_family_review_status" + t.index [ :account_id, :period_start_on, :period_end_on ], name: "index_account_statements_on_account_period" + t.index [ :suggested_account_id, :review_status ], name: "index_account_statements_on_suggested_account_review" + end + + add_check_constraint :account_statements, "byte_size > 0", name: "chk_account_statements_byte_size_positive" + add_check_constraint :account_statements, + "char_length(filename) <= 255", + name: "chk_account_statements_filename_length" + add_check_constraint :account_statements, + "char_length(content_type) <= 100", + name: "chk_account_statements_content_type_length" + add_check_constraint :account_statements, + "char_length(checksum) <= 64", + name: "chk_account_statements_checksum_length" + add_check_constraint :account_statements, + "institution_name_hint IS NULL OR char_length(institution_name_hint) <= 200", + name: "chk_account_statements_institution_hint_length" + add_check_constraint :account_statements, + "account_name_hint IS NULL OR char_length(account_name_hint) <= 200", + name: "chk_account_statements_account_name_hint_length" + add_check_constraint :account_statements, + "account_last4_hint IS NULL OR char_length(account_last4_hint) <= 4", + name: "chk_account_statements_account_last4_hint_length" + add_check_constraint :account_statements, + "currency IS NULL OR char_length(currency) <= 3", + name: "chk_account_statements_currency_length" + add_check_constraint :account_statements, + "period_start_on IS NULL OR period_end_on IS NULL OR period_start_on <= period_end_on", + name: "chk_account_statements_period_order" + add_check_constraint :account_statements, + "parser_confidence IS NULL OR (parser_confidence >= 0 AND parser_confidence <= 1)", + name: "chk_account_statements_parser_confidence" + add_check_constraint :account_statements, + "match_confidence IS NULL OR (match_confidence >= 0 AND match_confidence <= 1)", + name: "chk_account_statements_match_confidence" + add_check_constraint :account_statements, + "byte_size <= 26214400", + name: "chk_account_statements_byte_size_max" + add_check_constraint :account_statements, + "source IN ('manual_upload')", + name: "chk_account_statements_source" + add_check_constraint :account_statements, + "upload_status IN ('stored', 'failed')", + name: "chk_account_statements_upload_status" + add_check_constraint :account_statements, + "review_status IN ('unmatched', 'linked', 'rejected')", + name: "chk_account_statements_review_status" + add_check_constraint :account_statements, + "content_sha256 IS NULL OR content_sha256 ~ '^[0-9a-f]{64}$'", + name: "chk_account_statements_content_sha256" + end +end diff --git a/db/schema.rb b/db/schema.rb index 8b3cef737..943e4dfe3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -43,6 +43,57 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_12_211200) do t.check_constraint "permission::text = ANY (ARRAY['full_control'::character varying::text, 'read_write'::character varying::text, 'read_only'::character varying::text])", name: "chk_account_shares_permission" end + create_table "account_statements", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "family_id", null: false + t.uuid "account_id" + t.uuid "suggested_account_id" + t.string "filename", limit: 255, null: false + t.string "content_type", limit: 100, null: false + t.bigint "byte_size", null: false + t.string "checksum", limit: 64, null: false + t.string "source", default: "manual_upload", null: false + t.string "upload_status", default: "stored", null: false + t.string "institution_name_hint", limit: 200 + t.string "account_name_hint", limit: 200 + t.string "account_last4_hint", limit: 4 + t.date "period_start_on" + t.date "period_end_on" + t.decimal "opening_balance", precision: 19, scale: 4 + t.decimal "closing_balance", precision: 19, scale: 4 + t.string "currency", limit: 3 + t.decimal "parser_confidence", precision: 5, scale: 4 + t.decimal "match_confidence", precision: 5, scale: 4 + t.string "review_status", default: "unmatched", null: false + t.jsonb "sanitized_parser_output", default: {}, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.string "content_sha256" + t.index ["account_id", "period_start_on", "period_end_on"], name: "index_account_statements_on_account_period" + t.index ["account_id"], name: "index_account_statements_on_account_id" + t.index ["family_id", "checksum"], name: "index_account_statements_on_family_checksum" + t.index ["family_id", "content_sha256"], name: "index_account_statements_on_family_content_sha256", unique: true, where: "(content_sha256 IS NOT NULL)" + t.index ["family_id", "review_status"], name: "index_account_statements_on_family_review_status" + t.index ["family_id"], name: "index_account_statements_on_family_id" + t.index ["suggested_account_id", "review_status"], name: "index_account_statements_on_suggested_account_review" + t.index ["suggested_account_id"], name: "index_account_statements_on_suggested_account_id" + t.check_constraint "byte_size <= 26214400", name: "chk_account_statements_byte_size_max" + t.check_constraint "byte_size > 0", name: "chk_account_statements_byte_size_positive" + t.check_constraint "account_last4_hint IS NULL OR char_length(account_last4_hint::text) <= 4", name: "chk_account_statements_account_last4_hint_length" + t.check_constraint "account_name_hint IS NULL OR char_length(account_name_hint::text) <= 200", name: "chk_account_statements_account_name_hint_length" + t.check_constraint "char_length(checksum::text) <= 64", name: "chk_account_statements_checksum_length" + t.check_constraint "char_length(content_type::text) <= 100", name: "chk_account_statements_content_type_length" + t.check_constraint "content_sha256 IS NULL OR content_sha256::text ~ '^[0-9a-f]{64}$'::text", name: "chk_account_statements_content_sha256" + t.check_constraint "currency IS NULL OR char_length(currency::text) <= 3", name: "chk_account_statements_currency_length" + t.check_constraint "char_length(filename::text) <= 255", name: "chk_account_statements_filename_length" + t.check_constraint "institution_name_hint IS NULL OR char_length(institution_name_hint::text) <= 200", name: "chk_account_statements_institution_hint_length" + t.check_constraint "match_confidence IS NULL OR match_confidence >= 0::numeric AND match_confidence <= 1::numeric", name: "chk_account_statements_match_confidence" + t.check_constraint "parser_confidence IS NULL OR parser_confidence >= 0::numeric AND parser_confidence <= 1::numeric", name: "chk_account_statements_parser_confidence" + t.check_constraint "period_start_on IS NULL OR period_end_on IS NULL OR period_start_on <= period_end_on", name: "chk_account_statements_period_order" + t.check_constraint "review_status::text = ANY (ARRAY['unmatched'::character varying::text, 'linked'::character varying::text, 'rejected'::character varying::text])", name: "chk_account_statements_review_status" + t.check_constraint "source::text = 'manual_upload'::text", name: "chk_account_statements_source" + t.check_constraint "upload_status::text = ANY (ARRAY['stored'::character varying::text, 'failed'::character varying::text])", name: "chk_account_statements_upload_status" + end + create_table "accounts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.string "subtype" t.uuid "family_id", null: false @@ -1798,6 +1849,9 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_12_211200) do add_foreign_key "account_providers", "accounts", on_delete: :cascade add_foreign_key "account_shares", "accounts" add_foreign_key "account_shares", "users" + add_foreign_key "account_statements", "accounts", column: "suggested_account_id", on_delete: :nullify + add_foreign_key "account_statements", "accounts", on_delete: :nullify + add_foreign_key "account_statements", "families", on_delete: :cascade add_foreign_key "accounts", "families" add_foreign_key "accounts", "imports" add_foreign_key "accounts", "plaid_accounts" diff --git a/test/controllers/account_statements_controller_test.rb b/test/controllers/account_statements_controller_test.rb new file mode 100644 index 000000000..01739f0dc --- /dev/null +++ b/test/controllers/account_statements_controller_test.rb @@ -0,0 +1,483 @@ +require "test_helper" + +class AccountStatementsControllerTest < ActionDispatch::IntegrationTest + setup do + ensure_tailwind_build + sign_in @user = users(:family_admin) + @account = accounts(:depository) + end + + test "shows statement vault" do + get account_statements_url + assert_response :success + assert_select "h1", text: I18n.t("account_statements.index.title") + end + + test "statement vault only lists linked statements for accessible accounts" do + accessible_statement = AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "accessible_statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + private_account = accounts(:other_asset) + private_statement = AccountStatement.create_from_upload!( + family: private_account.family, + account: private_account, + file: uploaded_file(filename: "private_statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-02,2\n") + ) + sign_in users(:family_member) + + get account_statements_url + + assert_response :success + assert_includes response.body, accessible_statement.filename + refute_includes response.body, private_statement.filename + refute_includes response.body, private_account.name + end + + test "non manager cannot open statement vault" do + sign_in family_guest + + get account_statements_url + + assert_redirected_to accounts_url + assert_equal I18n.t("accounts.not_authorized"), flash[:alert] + end + + test "non manager cannot view unmatched statement" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: nil, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv") + ) + sign_in family_guest + + get account_statement_url(statement) + + assert_response :not_found + end + + test "uploads statement to account without importing transactions" do + assert_difference "AccountStatement.count", 1 do + assert_no_difference [ "Import.count", "Entry.count", "Transaction.count" ] do + post account_statements_url, params: { + account_statement: { + account_id: @account.id, + files: [ uploaded_file(filename: "Checking_2024-01.csv", content_type: "text/csv") ] + } + } + end + end + + statement = AccountStatement.order(:created_at).last + assert_equal @account, statement.account + assert statement.linked? + assert_redirected_to account_url(@account, tab: "statements") + end + + test "member with writable account access can upload linked statement" do + sign_in users(:family_member) + + assert_difference "AccountStatement.count", 1 do + post account_statements_url, params: { + account_statement: { + account_id: @account.id, + files: [ uploaded_file(filename: "member_statement.csv", content_type: "text/csv") ] + } + } + end + + statement = AccountStatement.order(:created_at).last + assert_equal @account, statement.account + assert_redirected_to account_url(@account, tab: "statements") + end + + test "uploads unmatched statement to inbox" do + assert_difference "AccountStatement.count", 1 do + post account_statements_url, params: { + account_statement: { + files: [ uploaded_file(filename: "Unknown_2024-01.csv", content_type: "text/csv") ] + } + } + end + + statement = AccountStatement.order(:created_at).last + assert_nil statement.account + assert statement.unmatched? + assert_redirected_to account_statement_url(statement) + end + + test "skips duplicate statement upload" do + AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + assert_no_difference "AccountStatement.count" do + post account_statements_url, params: { + account_statement: { + account_id: @account.id, + files: [ uploaded_file(filename: "duplicate.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") ] + } + } + end + + assert_redirected_to account_url(@account, tab: "statements") + assert_equal I18n.t("account_statements.create.duplicates", count: 1), flash[:alert] + end + + test "continues upload loop after a validation error" do + invalid_record = AccountStatement.new + invalid_record.errors.add(:filename, "is invalid") + + assert_difference "AccountStatement.count", 1 do + created_statement = AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "valid-result.csv", content_type: "text/csv", content: "date,amount\n2024-01-02,2\n") + ) + upload_sequence = sequence("statement upload processing") + AccountStatement.expects(:create_from_prepared_upload!).in_sequence(upload_sequence).raises(ActiveRecord::RecordInvalid.new(invalid_record)) + AccountStatement.expects(:create_from_prepared_upload!).in_sequence(upload_sequence).returns(created_statement) + + post account_statements_url, params: { + account_statement: { + account_id: @account.id, + files: [ + uploaded_file(filename: "invalid.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n"), + uploaded_file(filename: "valid.csv", content_type: "text/csv", content: "date,amount\n2024-01-02,2\n") + ] + } + } + end + + assert_redirected_to account_url(@account, tab: "statements") + assert_equal I18n.t("account_statements.create.success", count: 1), flash[:notice] + assert_includes flash[:alert], invalid_record.errors.full_messages.to_sentence + end + + test "rejects invalid statement file type" do + assert_no_difference "AccountStatement.count" do + post account_statements_url, params: { + account_statement: { + files: [ uploaded_file(filename: "statement.bin", content_type: "application/octet-stream", content: "\x00\x01\x02".b) ] + } + } + end + + assert_redirected_to account_statements_url + assert_equal I18n.t("account_statements.create.invalid_file_type"), flash[:alert] + end + + test "continues upload loop after an invalid file type" do + assert_difference "AccountStatement.count", 1 do + post account_statements_url, params: { + account_statement: { + files: [ + uploaded_file(filename: "statement.bin", content_type: "application/octet-stream", content: "\x00\x01\x02".b), + uploaded_file(filename: "valid.csv", content_type: "text/csv", content: "date,amount\n2024-01-02,2\n") + ] + } + } + end + + statement = AccountStatement.order(:created_at).last + assert_redirected_to account_statement_url(statement) + assert_equal I18n.t("account_statements.create.success", count: 1), flash[:notice] + assert_includes flash[:alert], I18n.t("account_statements.create.invalid_file_type") + end + + test "rejects txt and xls statement uploads" do + [ + uploaded_file(filename: "statement.txt", content_type: "text/plain"), + uploaded_file(filename: "statement.xls", content_type: "application/vnd.ms-excel") + ].each do |file| + assert_no_difference "AccountStatement.count" do + post account_statements_url, params: { + account_statement: { + files: [ file ] + } + } + end + + assert_redirected_to account_statements_url + assert_equal I18n.t("account_statements.create.invalid_file_type"), flash[:alert] + end + end + + test "rejects empty csv and xlsx statement uploads" do + [ + uploaded_file(filename: "empty.csv", content_type: "text/csv", content: ""), + uploaded_file( + filename: "empty.xlsx", + content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + content: "" + ) + ].each do |file| + assert_no_difference "AccountStatement.count" do + post account_statements_url, params: { + account_statement: { + files: [ file ] + } + } + end + + assert_redirected_to account_statements_url + assert_equal I18n.t("account_statements.create.invalid_file_type"), flash[:alert] + end + end + + test "rejects oversized statement upload" do + original_max_file_size = AccountStatement::MAX_FILE_SIZE + silence_warnings { AccountStatement.const_set(:MAX_FILE_SIZE, 16) } + + begin + assert_no_difference "AccountStatement.count" do + post account_statements_url, params: { + account_statement: { + files: [ + uploaded_file( + filename: "oversized.csv", + content_type: "text/csv", + content: "x" * (AccountStatement::MAX_FILE_SIZE + 1) + ) + ] + } + } + end + ensure + silence_warnings { AccountStatement.const_set(:MAX_FILE_SIZE, original_max_file_size) } + end + + assert_redirected_to account_statements_url + assert_equal I18n.t("account_statements.create.invalid_file_type"), flash[:alert] + end + + test "rejects cross-family account id" do + other_account = Account.create!( + family: families(:empty), + owner: users(:empty), + name: "Other family account", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + + assert_no_difference "AccountStatement.count" do + post account_statements_url, params: { + account_statement: { + account_id: other_account.id, + files: [ uploaded_file(filename: "statement.csv", content_type: "text/csv") ] + } + } + end + assert_response :not_found + end + + test "read only shared user cannot upload to account" do + sign_in users(:family_member) + account = accounts(:credit_card) + + assert_no_difference "AccountStatement.count" do + post account_statements_url, params: { + account_statement: { + account_id: account.id, + files: [ uploaded_file(filename: "statement.csv", content_type: "text/csv") ] + } + } + end + + assert_redirected_to account_url(account) + assert_equal I18n.t("accounts.not_authorized"), flash[:alert] + end + + test "read only shared user sees statement detail without edit controls" do + account = accounts(:credit_card) + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: account, + file: uploaded_file(filename: "readonly_statement.csv", content_type: "text/csv") + ) + sign_in users(:family_member) + + get account_statement_url(statement) + + assert_response :success + assert_select "input[name='account_statement[period_start_on]']", 0 + assert_select "select[name='account_statement[account_id]']", 0 + assert_select "button", text: I18n.t("account_statements.show.delete"), count: 0 + assert_select "button", text: I18n.t("account_statements.show.save"), count: 0 + assert_select "button", text: I18n.t("account_statements.show.unlink"), count: 0 + end + + test "metadata form does not expose account select for managers" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "manager_statement.csv", content_type: "text/csv") + ) + + get account_statement_url(statement) + + assert_response :success + assert_select "input[name='account_statement[period_start_on]']", 1 + assert_select "input[name='account_statement[currency]']", 0 + assert_select "select[name='account_statement[currency]'] option[value='USD']" + assert_select "select[name='account_statement[account_id]']", 0 + end + + test "links suggested statement" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: nil, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.update!(suggested_account: @account, match_confidence: 0.9) + + patch link_account_statement_url(statement), params: { account_id: @account.id } + + assert_redirected_to account_url(@account, tab: "statements") + statement.reload + assert_equal @account, statement.account + assert statement.linked? + end + + test "read only shared user cannot relink linked statement to writable account" do + source_account = accounts(:credit_card) + target_account = accounts(:depository) + statement = AccountStatement.create_from_upload!( + family: source_account.family, + account: source_account, + file: uploaded_file(filename: "readonly_relink.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + sign_in users(:family_member) + + patch link_account_statement_url(statement), params: { account_id: target_account.id } + + assert_redirected_to account_url(source_account) + assert_equal I18n.t("accounts.not_authorized"), flash[:alert] + assert_equal source_account, statement.reload.account + end + + test "link shows friendly error when no target account is available" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: nil, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + patch link_account_statement_url(statement) + + assert_redirected_to account_statement_url(statement) + assert_equal I18n.t("account_statements.link.no_account"), flash[:alert] + statement.reload + assert_nil statement.account + assert statement.unmatched? + end + + test "unlinks statement back to inbox" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + patch unlink_account_statement_url(statement) + + assert_redirected_to account_statement_url(statement) + statement.reload + assert_nil statement.account + assert statement.unmatched? + end + + test "rejects suggestion" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: nil, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.update!(suggested_account: @account, match_confidence: 0.9) + + patch reject_account_statement_url(statement) + + assert_redirected_to account_statements_url + statement.reload + assert statement.rejected? + assert_nil statement.suggested_account + end + + test "updates metadata" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + patch account_statement_url(statement), params: { + account_statement: { + period_start_on: "2024-01-01", + period_end_on: "2024-01-31", + closing_balance: "123.45", + currency: "usd" + } + } + + assert_redirected_to account_statement_url(statement) + statement.reload + assert_equal Date.new(2024, 1, 31), statement.period_end_on + assert_equal 123.45.to_d, statement.closing_balance + assert_equal "USD", statement.currency + end + + test "metadata update links selected account" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: nil, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + patch account_statement_url(statement), params: { + account_statement: { + account_id: @account.id, + period_start_on: "2024-01-01", + period_end_on: "2024-01-31" + } + } + + assert_redirected_to account_statement_url(statement) + statement.reload + assert_equal @account, statement.account + assert statement.linked? + end + + test "deletes statement" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + assert_difference "AccountStatement.count", -1 do + delete account_statement_url(statement) + end + + assert_redirected_to account_url(@account, tab: "statements") + end + + test "destroy reports failure when statement cannot be deleted" do + statement = AccountStatement.create_from_upload!( + family: @account.family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + AccountStatement.any_instance.stubs(:destroy).returns(false) + + assert_no_difference "AccountStatement.count" do + delete account_statement_url(statement) + end + + assert_redirected_to account_url(@account, tab: "statements") + assert_equal I18n.t("account_statements.destroy.failure"), flash[:alert] + end +end diff --git a/test/controllers/accounts_controller_test.rb b/test/controllers/accounts_controller_test.rb index cda1e81e7..4b7604c3e 100644 --- a/test/controllers/accounts_controller_test.rb +++ b/test/controllers/accounts_controller_test.rb @@ -19,6 +19,83 @@ class AccountsControllerTest < ActionDispatch::IntegrationTest assert_response :success end + test "show lazily loads statement tab data unless statements tab is active" do + AccountStatement::Coverage.expects(:for_year).never + AccountStatement.expects(:reconciliation_statuses_for).never + + get account_url(@account) + + assert_response :success + assert_select "select[name='statement_year']", count: 0 + statements_path = account_path(@account, tab: "statements") + assert_select "turbo-frame[src='#{statements_path}']" + end + + test "statements tab shows coverage and upload for statement managers with account write access" do + get account_url(@account, tab: "statements") + + assert_response :success + assert_select "input[type=file][accept='.pdf,.csv,.xlsx']" + assert_select "select[name='statement_year']" + assert_select "p", text: I18n.l(Date.current.prev_month.beginning_of_month, format: "%b %Y") + end + + test "statements tab lazy frame returns matching frame content" do + frame_id = dom_id(@account, :statements_tab) + + get account_url(@account, tab: "statements"), headers: { "Turbo-Frame" => frame_id } + + assert_response :success + assert_select "turbo-frame##{frame_id}", count: 1 + assert_select "select[name='statement_year']" + assert_select "turbo-frame##{dom_id(@account, :container)}", count: 0 + end + + test "statements tab filters historical coverage by year" do + account = Account.create!( + family: @user.family, + owner: @user, + name: "Historical Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + statement = AccountStatement.create_from_upload!( + family: @user.family, + account: account, + file: uploaded_file(filename: "historical.csv", content_type: "text/csv") + ) + statement.update!(period_start_on: Date.new(2024, 2, 1), period_end_on: Date.new(2024, 2, 29)) + + travel_to Date.new(2026, 5, 6) do + get account_url(account, tab: "statements") + + assert_response :success + assert_select "select[name='statement_year'] option[selected='selected']", text: "2026" + assert_select "p", text: "May 2026" + assert_select "p", text: "Not expected" + + get account_url(account, tab: "statements", statement_year: 2024) + + assert_response :success + assert_select "select[name='statement_year'] option[selected='selected']", text: "2024" + assert_select "p", text: "Jan 2024" + assert_select "p", text: "Feb 2024" + assert_select "p", text: "Covered" + assert_select "p", text: "Missing" + assert_select "p", text: "Not expected" + end + end + + test "statements tab hides upload for read only account access" do + sign_in users(:family_member) + + get account_url(accounts(:credit_card), tab: "statements") + + assert_response :success + assert_select "input[type=file]", count: 0 + end + test "account activity marks trade amounts as privacy-sensitive" do trade_entry = entries(:trade) expected_amount = ApplicationController.helpers.format_money(-trade_entry.amount_money) diff --git a/test/fixtures/account_statements.yml b/test/fixtures/account_statements.yml new file mode 100644 index 000000000..868eb8b97 --- /dev/null +++ b/test/fixtures/account_statements.yml @@ -0,0 +1 @@ +# Empty fixture file so account_statements is cleaned with the rest of the test fixture set. diff --git a/test/helpers/account_statements_helper_test.rb b/test/helpers/account_statements_helper_test.rb new file mode 100644 index 000000000..567468bda --- /dev/null +++ b/test/helpers/account_statements_helper_test.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class AccountStatementsHelperTest < ActionView::TestCase + test "reconciliation label falls back for invalid checks" do + opening_balance = I18n.t("account_statements.reconciliation.checks.opening_balance") + closing_balance = I18n.t("account_statements.reconciliation.checks.closing_balance") + unknown_check = I18n.t("account_statements.reconciliation.checks.unknown_check") + + assert_equal opening_balance, account_statement_reconciliation_label({ key: "opening_balance" }) + assert_equal closing_balance, account_statement_reconciliation_label({ "key" => "closing_balance" }) + assert_equal unknown_check, account_statement_reconciliation_label({}) + assert_equal unknown_check, account_statement_reconciliation_label(nil) + assert_equal unknown_check, account_statement_reconciliation_label([]) + end +end diff --git a/test/integration/active_storage_authorization_test.rb b/test/integration/active_storage_authorization_test.rb index 75698a536..9c30d4f91 100644 --- a/test/integration/active_storage_authorization_test.rb +++ b/test/integration/active_storage_authorization_test.rb @@ -12,6 +12,16 @@ class ActiveStorageAuthorizationTest < ActionDispatch::IntegrationTest content_type: "application/pdf" ) @attachment_a = @transaction_a.attachments.first + + @statement_a = AccountStatement.create_from_upload!( + family: @user_a.family, + account: @transaction_a.entry.account, + file: uploaded_file( + filename: "statement.pdf", + content_type: "application/pdf", + content: "%PDF-1.4 Family A Secret Statement" + ) + ) end test "user can access attachments within their own family" do @@ -29,6 +39,33 @@ class ActiveStorageAuthorizationTest < ActionDispatch::IntegrationTest assert_match(/rails\/active_storage\/disk/, response.header["Location"]) end + test "disk service urls require authentication" do + sign_in @user_a + + get rails_blob_path(@statement_a.original_file) + assert_response :redirect + disk_url = response.location + sign_out @user_a + + get disk_url + + assert_redirected_to new_session_url + end + + test "disk service urls enforce statement blob authorization" do + sign_in @user_a + + get rails_blob_path(@statement_a.original_file) + assert_response :redirect + disk_url = response.location + sign_out @user_a + sign_in @user_b + + get disk_url + + assert_response :not_found + end + test "user cannot access attachments from a different family" do sign_in @user_b @@ -55,4 +92,213 @@ class ActiveStorageAuthorizationTest < ActionDispatch::IntegrationTest assert_response :not_found end + + test "user cannot access statement blob from a different family" do + sign_in @user_b + + get rails_blob_path(@statement_a.original_file) + + assert_response :not_found + end + + test "unauthenticated user is redirected before statement blob access" do + get rails_blob_path(@statement_a.original_file) + + assert_redirected_to new_session_url + end + + test "user cannot access linked statement blob for an inaccessible account" do + private_account = accounts(:other_asset) + statement = AccountStatement.create_from_upload!( + family: @user_a.family, + account: private_account, + file: uploaded_file( + filename: "private_statement.pdf", + content_type: "application/pdf", + content: "%PDF-1.4 Private Family Statement" + ) + ) + + sign_in users(:family_member) + + get rails_blob_path(statement.original_file) + + assert_response :not_found + end + + test "user can access linked statement blob for a shared account" do + statement = AccountStatement.create_from_upload!( + family: @user_a.family, + account: accounts(:credit_card), + file: uploaded_file( + filename: "shared_statement.pdf", + content_type: "application/pdf", + content: "%PDF-1.4 Shared Family Statement" + ) + ) + + sign_in users(:family_member) + + get rails_blob_path(statement.original_file) + + assert_response :redirect + follow_redirect! + assert_response :success + assert_match(/rails\/active_storage\/disk/, request.path) + end + + test "guest cannot access unmatched statement blob" do + statement = AccountStatement.create_from_upload!( + family: @user_a.family, + account: nil, + file: uploaded_file( + filename: "unmatched_statement.pdf", + content_type: "application/pdf", + content: "%PDF-1.4 Unmatched Family Statement" + ) + ) + + sign_in family_guest + + get rails_blob_path(statement.original_file) + + assert_response :not_found + end + + test "orphaned statement attachment fails closed" do + attachment = @statement_a.original_file.attachment + attachment.update_columns(record_id: SecureRandom.uuid) + + sign_in @user_a + + get rails_blob_path(attachment) + + assert_response :not_found + end + + test "unattached blobs fail closed" do + blob = ActiveStorage::Blob.create_and_upload!( + io: StringIO.new("unattached statement"), + filename: "unattached.csv", + content_type: "text/csv" + ) + + sign_in @user_a + + get rails_blob_path(blob) + + assert_response :not_found + end + + test "blob authorization checks protected attachments even when blob is also attached elsewhere" do + document = FamilyDocument.create!(family: @user_a.family, filename: "shared.pdf", status: "ready") + document.file.attach(@statement_a.original_file.blob) + + sign_in @user_b + + get rails_blob_path(document.file) + + assert_response :not_found + end + + test "blob authorization denies when any protected attachment is unauthorized" do + statement_b = AccountStatement.new( + family: @user_b.family, + filename: "shared_statement.pdf", + content_type: @statement_a.content_type, + byte_size: @statement_a.byte_size, + checksum: @statement_a.checksum, + content_sha256: @statement_a.content_sha256, + currency: @user_b.family.currency + ) + statement_b.original_file.attach(@statement_a.original_file.blob) + statement_b.save! + + sign_in @user_a + + get rails_blob_path(@statement_a.original_file) + + assert_response :not_found + end + + test "unknown protected attachment types fail closed" do + blob = ActiveStorage::Blob.create_and_upload!( + io: StringIO.new("unknown protected attachment"), + filename: "unknown.csv", + content_type: "text/csv" + ) + ActiveStorage::Attachment.insert!( + { + name: "file", + record_type: "ProtectedAttachmentProbe", + record_id: SecureRandom.uuid, + blob_id: blob.id, + created_at: Time.current + } + ) + + with_protected_record_types("Transaction", "AccountStatement", "ProtectedAttachmentProbe") do + sign_in @user_a + + get rails_blob_path(blob) + + assert_response :not_found + end + end + + test "direct uploads require authentication" do + post rails_direct_uploads_path, params: { + blob: { + filename: "statement.csv", + byte_size: 1, + checksum: Digest::MD5.base64digest("1"), + content_type: "text/csv" + } + }, as: :json + + assert_redirected_to new_session_url + end + + test "authenticated direct uploads can create unattached blobs" do + sign_in @user_a + + post rails_direct_uploads_path, params: { + blob: { + filename: "statement.csv", + byte_size: 1, + checksum: Digest::MD5.base64digest("1"), + content_type: "text/csv" + } + }, as: :json + + assert_response :success + assert response.parsed_body["signed_id"].present? + end + + test "orphaned transaction attachment fails closed" do + @attachment_a.update_columns(record_id: SecureRandom.uuid) + + sign_in @user_a + + get rails_blob_path(@attachment_a) + + assert_response :not_found + end + + private + + def sign_out(user) + user.sessions.each { |session| delete session_path(session) } + end + + def with_protected_record_types(*types) + previous_types = ActiveStorageAttachmentAuthorization::PROTECTED_RECORD_TYPES + ActiveStorageAttachmentAuthorization.send(:remove_const, :PROTECTED_RECORD_TYPES) + ActiveStorageAttachmentAuthorization.const_set(:PROTECTED_RECORD_TYPES, types.flatten.freeze) + + yield + ensure + ActiveStorageAttachmentAuthorization.send(:remove_const, :PROTECTED_RECORD_TYPES) + ActiveStorageAttachmentAuthorization.const_set(:PROTECTED_RECORD_TYPES, previous_types) + end end diff --git a/test/models/account_statement_test.rb b/test/models/account_statement_test.rb new file mode 100644 index 000000000..a2829a0c9 --- /dev/null +++ b/test/models/account_statement_test.rb @@ -0,0 +1,770 @@ +require "test_helper" + +class AccountStatementTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @account = accounts(:depository) + end + + OversizedDeclaredUpload = Struct.new(:original_filename, keyword_init: true) do + def size + AccountStatement::MAX_FILE_SIZE + 1 + end + + def read(*) + raise "oversized upload should be rejected before reading" + end + end + + class UploadWithoutDeclaredSize + attr_reader :original_filename, :content_type + + def initialize(filename:, content_type:, content:) + @original_filename = filename + @content_type = content_type + @io = StringIO.new(content) + end + + def read(length) + @io.read(length) + end + + def rewind + @io.rewind + end + end + + test "creates linked statement from upload without importing transactions" do + assert_no_difference [ "Import.count", "Entry.count", "Transaction.count" ] do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file( + filename: "Chase_2024-01_account_6789.csv", + content_type: "text/csv", + content: "date,description,amount\n2024-01-01,Coffee,-5.00\n2024-01-31,Deposit,100.00\n" + ) + ) + + assert statement.linked? + assert_equal @account, statement.account + assert_equal Date.new(2024, 1, 1), statement.period_start_on + assert_equal Date.new(2024, 1, 31), statement.period_end_on + assert_equal "USD", statement.currency + assert_equal Digest::SHA256.hexdigest("date,description,amount\n2024-01-01,Coffee,-5.00\n2024-01-31,Deposit,100.00\n"), statement.content_sha256 + assert statement.original_file.attached? + end + end + + test "suggests obvious account match without linking inbox upload" do + @account.update!(institution_name: "Chase Bank 6789", notes: "Private note") + + statement = AccountStatement.create_from_upload!( + family: @family, + account: nil, + file: uploaded_file( + filename: "Chase_Bank_2024-01_account_6789.pdf", + content_type: "application/pdf", + content: "%PDF-1.4 statement" + ) + ) + + assert statement.unmatched? + assert_nil statement.account + assert_equal @account, statement.suggested_account + assert_operator statement.match_confidence, :>=, 0.7 + end + + test "rejects duplicate sha256 within family" do + file_content = "date,description,amount\n2024-01-01,Coffee,-5.00\n" + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: file_content) + ) + + error = assert_raises(AccountStatement::DuplicateUploadError) do + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement-copy.csv", content_type: "text/csv", content: file_content) + ) + end + + assert_equal "statement.csv", error.statement.filename + end + + test "allows distinct files with same md5 checksum and different sha256" do + Digest::MD5.stubs(:base64digest).returns("same-md5-checksum") + + assert_difference "AccountStatement.count", 2 do + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement-a.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement-b.csv", content_type: "text/csv", content: "date,amount\n2024-01-02,2\n") + ) + end + end + + test "uses md5 checksum fallback for legacy statements without sha256" do + Digest::MD5.stubs(:base64digest).returns("legacy-md5-checksum") + existing = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "legacy.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + existing.update_columns(content_sha256: nil) + + error = assert_raises(AccountStatement::DuplicateUploadError) do + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "legacy-copy.csv", content_type: "text/csv", content: "date,amount\n2024-01-02,2\n") + ) + end + + assert_equal existing, error.statement + end + + test "reports duplicate upload after database uniqueness race" do + file_content = "date,description,amount\n2024-01-01,Coffee,-5.00\n" + existing = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: file_content) + ) + prepared_upload = AccountStatement.prepare_upload!( + uploaded_file(filename: "statement-copy.csv", content_type: "text/csv", content: file_content) + ) + + AccountStatement.stubs(:duplicate_for).returns(nil, existing) + AccountStatement.any_instance.stubs(:save!).raises(ActiveRecord::RecordNotUnique.new("duplicate")) + + error = assert_raises(AccountStatement::DuplicateUploadError) do + AccountStatement.create_from_prepared_upload!( + family: @family, + account: @account, + prepared_upload: prepared_upload + ) + end + + assert_equal existing, error.statement + end + + test "purges staged blob when database uniqueness race is re-raised" do + prepared_upload = AccountStatement.prepare_upload!( + uploaded_file(filename: "statement-copy.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + AccountStatement.stubs(:duplicate_for).returns(nil) + AccountStatement.any_instance.stubs(:save!).raises(ActiveRecord::RecordNotUnique.new("duplicate")) + + assert_no_difference [ "ActiveStorage::Blob.count", "ActiveStorage::Attachment.count" ] do + assert_raises(ActiveRecord::RecordNotUnique) do + AccountStatement.create_from_prepared_upload!( + family: @family, + account: @account, + prepared_upload: prepared_upload + ) + end + end + end + + test "purges staged blob when metadata detection fails after attach" do + AccountStatement::MetadataDetector.any_instance.stubs(:apply).raises(StandardError, "parser failed") + + assert_no_difference [ "ActiveStorage::Blob.count", "ActiveStorage::Attachment.count" ] do + assert_raises(StandardError) do + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + end + end + end + + test "with_account scope keeps account linkage semantics while enum predicate follows review status" do + linked_statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "linked.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + accountless_statement = AccountStatement.create_from_upload!( + family: @family, + account: nil, + file: uploaded_file(filename: "accountless.csv", content_type: "text/csv", content: "date,amount\n2024-01-02,2\n") + ) + accountless_statement.update_columns(review_status: "linked") + + assert accountless_statement.reload.linked? + assert_includes @family.account_statements.with_account, linked_statement + assert_not_includes @family.account_statements.with_account, accountless_statement + assert_not_includes @family.account_statements.unmatched, accountless_statement + end + + test "allows same checksum in different families" do + file_content = "date,description,amount\n2024-01-01,Coffee,-5.00\n" + + assert_difference "AccountStatement.count", 2 do + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: file_content) + ) + + AccountStatement.create_from_upload!( + family: families(:empty), + account: nil, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: file_content) + ) + end + end + + test "validates linked account family" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + statement.account = Account.create!( + family: families(:empty), + owner: users(:empty), + name: "Other family account", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + + assert_not statement.valid? + assert_includes statement.errors[:account], "is invalid" + end + + test "validates statement currency codes" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + statement.currency = "NOPE" + + assert_not statement.valid? + assert_includes statement.errors[:currency], "is invalid" + end + + test "rejects unsupported file extension even when mime type is broadly allowed" do + assert_raises(AccountStatement::InvalidUploadError) do + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.txt", content_type: "text/plain", content: "date,amount\n2024-01-01,1\n") + ) + end + + assert_raises(AccountStatement::InvalidUploadError) do + AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.xls", content_type: "application/vnd.ms-excel", content: "date,amount\n2024-01-01,1\n") + ) + end + end + + test "rejects empty csv and xlsx statement uploads" do + [ + uploaded_file(filename: "empty.csv", content_type: "text/csv", content: ""), + uploaded_file( + filename: "empty.xlsx", + content_type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + content: "" + ) + ].each do |file| + assert_no_difference "AccountStatement.count" do + assert_raises(AccountStatement::InvalidUploadError) do + AccountStatement.create_from_upload!(family: @family, account: @account, file: file) + end + end + end + end + + test "rejects declared oversized upload before reading content" do + assert_raises(AccountStatement::InvalidUploadError) do + AccountStatement.prepare_upload!(OversizedDeclaredUpload.new(original_filename: "oversized.csv")) + end + end + + test "streams unknown-size uploads and rejects when content exceeds size limit" do + file = UploadWithoutDeclaredSize.new( + filename: "oversized.csv", + content_type: "text/csv", + content: "x" * (AccountStatement::MAX_FILE_SIZE + 1) + ) + + assert_raises(AccountStatement::InvalidUploadError) do + AccountStatement.prepare_upload!(file) + end + end + + test "stores sanitized csv parser output without raw rows" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file( + filename: "Checking_2024-01.csv", + content_type: "text/csv", + content: "posted_at,description,amount\n2024-01-01,Coffee Shop,-5.00\n2024-01-31,Payroll,100.00\n" + ) + ) + + assert_equal Date.new(2024, 1, 1), statement.period_start_on + assert_equal Date.new(2024, 1, 31), statement.period_end_on + assert_equal "posted_at", statement.sanitized_parser_output.dig("csv", "date_header") + assert_equal 2, statement.sanitized_parser_output.dig("csv", "rows_sampled") + assert_not_includes statement.sanitized_parser_output.to_json, "Coffee Shop" + assert_not_includes statement.sanitized_parser_output.to_json, "Payroll" + end + + test "detects filename dates separated by underscores" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file( + filename: "statement_2024_01_31.csv", + content_type: "text/csv", + content: "description,amount\nCoffee,-5.00\n" + ) + ) + + assert_equal Date.new(2024, 1, 1), statement.period_start_on + assert_equal Date.new(2024, 1, 31), statement.period_end_on + end + + test "ignores unreasonable filename dates" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file( + filename: "statement_1969_01_31.csv", + content_type: "text/csv", + content: "description,amount\nCoffee,-5.00\n" + ) + ) + + assert_nil statement.period_start_on + assert_nil statement.period_end_on + end + + test "samples csv metadata without parsing raw rows into sanitized output" do + rows = 300.times.map { |index| "2024-01-#{(index % 28) + 1},Row #{index}" }.join("\n") + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file( + filename: "Checking_2024-01.csv", + content_type: "text/csv", + content: "posted_at,description\n#{rows}\n" + ) + ) + + assert_equal 250, statement.sanitized_parser_output.dig("csv", "rows_sampled") + assert_not_includes statement.sanitized_parser_output.to_json, "Row 299" + end + + test "bounds csv metadata detection column count" do + headers = [ "posted_at", *101.times.map { |index| "column_#{index}" } ].join(",") + values = [ "2024-01-01", *101.times.map { "value" } ].join(",") + + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file( + filename: "Checking.csv", + content_type: "text/csv", + content: "#{headers}\n#{values}\n" + ) + ) + + assert_nil statement.sanitized_parser_output["csv"] + end + + test "bounds csv metadata detection sample length" do + oversized_date = "2024-01-01" + ("x" * AccountStatement::MetadataDetector::MAX_CSV_SAMPLE_BYTES) + + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file( + filename: "Checking.csv", + content_type: "text/csv", + content: "posted_at,description\n#{oversized_date},oversized\n" + ) + ) + + assert_nil statement.sanitized_parser_output["csv"] + assert_not_includes statement.sanitized_parser_output.to_json, oversized_date + end + + test "preserves sanitized pdf metadata output" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: nil, + file: uploaded_file( + filename: "Statement.pdf", + content_type: "application/pdf", + content: "%PDF-1.4 statement" + ) + ) + + assert_equal "filename_only", statement.sanitized_parser_output["pdf_detection"] + assert_empty statement.sanitized_parser_output["metadata_sources"] + assert_nil statement.institution_name_hint + assert_nil statement.account_name_hint + assert_equal 0.1.to_d, statement.parser_confidence + end + + test "stores an actual pdf document fixture as a statement" do + fixture_path = file_fixture("imports/sample_bank_statement.pdf") + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: Rack::Test::UploadedFile.new( + fixture_path, + "application/pdf", + true, + original_filename: "sample_bank_statement_2024-01.pdf" + ) + ) + + assert statement.linked? + assert statement.original_file.attached? + assert_equal "application/pdf", statement.content_type + assert_equal fixture_path.size, statement.byte_size + assert_equal Digest::SHA256.file(fixture_path).hexdigest, statement.content_sha256 + assert_equal "filename_only", statement.sanitized_parser_output["pdf_detection"] + assert_equal [ "filename" ], statement.sanitized_parser_output["metadata_sources"] + assert_equal Date.new(2024, 1, 1), statement.period_start_on + assert_equal Date.new(2024, 1, 31), statement.period_end_on + assert statement.original_file.blob.download.start_with?("%PDF-") + end + + test "handles malformed csv metadata detection without raw parser output" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: nil, + file: uploaded_file( + filename: "Unknown 2024-02.csv", + content_type: "text/csv", + content: "date,description\n\"unterminated" + ) + ) + + assert_equal Date.new(2024, 2, 1), statement.period_start_on + assert_equal Date.new(2024, 2, 29), statement.period_end_on + assert_nil statement.sanitized_parser_output["csv"] + assert_not_includes statement.sanitized_parser_output.to_json, "unterminated" + end + + test "reports reconciliation unavailable when balances are missing" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.update!( + period_start_on: Date.new(2024, 1, 1), + period_end_on: Date.new(2024, 1, 31), + closing_balance: 100 + ) + + assert_empty statement.reconciliation_checks + assert_equal "unavailable", statement.reconciliation_status + end + + test "coverage requires account" do + error = assert_raises(ArgumentError) do + AccountStatement::Coverage.new(nil) + end + assert_match(/account is required/, error.message) + end + + test "database constraints reject invalid persisted status values" do + attrs = { + family_id: @family.id, + filename: "statement.csv", + content_type: "text/csv", + byte_size: 1, + checksum: SecureRandom.base64(16), + source: "provider_sync", + upload_status: "stored", + review_status: "unmatched" + } + + assert_raises(ActiveRecord::StatementInvalid) do + AccountStatement.transaction(requires_new: true) do + AccountStatement.insert_all!([ attrs ], record_timestamps: true) + end + end + end + + test "database constraints reject empty persisted statement byte sizes" do + attrs = { + family_id: @family.id, + filename: "empty.csv", + content_type: "text/csv", + byte_size: 0, + checksum: SecureRandom.base64(16), + source: "manual_upload", + upload_status: "stored", + review_status: "unmatched" + } + + assert_raises(ActiveRecord::StatementInvalid) do + AccountStatement.transaction(requires_new: true) do + AccountStatement.insert_all!([ attrs ], record_timestamps: true) + end + end + end + + test "moves linked statements to inbox when account is deleted" do + account = Account.create!( + family: @family, + owner: users(:family_admin), + name: "Temporary Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + statement = AccountStatement.create_from_upload!( + family: @family, + account: account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + account.destroy! + + statement.reload + assert_nil statement.account + assert statement.unmatched? + assert_includes @family.account_statements.unmatched, statement + end + + test "unlink clears invalid recomputed suggestion" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + other_account = Account.create!( + family: families(:empty), + owner: users(:empty), + name: "Other family account", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + invalid_match = AccountStatement::AccountMatcher::Match.new(account: other_account, confidence: 0.9) + AccountStatement::AccountMatcher.any_instance.stubs(:best_match).returns(invalid_match) + + statement.unlink! + + statement.reload + assert_nil statement.account + assert_nil statement.suggested_account + assert_nil statement.match_confidence + assert statement.unmatched? + end + + test "preserves explicit rejected review status" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + + statement.reject_match! + + assert statement.rejected? + assert_equal @account, statement.account + end + + test "preserves rejected review status across unrelated saves" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.reject_match! + + statement.update!(period_start_on: Date.new(2024, 1, 1)) + + assert statement.rejected? + assert_equal @account, statement.account + end + + test "allows intentional review status changes away from rejected" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.reject_match! + + statement.link_to_account!(@account) + + assert statement.linked? + assert_equal @account, statement.account + end + + test "normalizes account last four hint when matching accounts" do + @account.update!(institution_name: "Acme Bank ABCD", notes: "Private note") + + statement = AccountStatement.new( + family: @family, + institution_name_hint: "Acme", + account_last4_hint: "ABCD", + currency: @account.currency + ) + + match = AccountStatement::AccountMatcher.new(statement).best_match + + assert_equal @account, match.account + assert_operator match.confidence, :>=, 0.75.to_d + end + + test "does not match account last four hints from account notes" do + @account.update!(institution_name: "Acme Bank", notes: "Masked statement suffix abcd") + + statement = AccountStatement.new( + family: @family, + account_last4_hint: "ABCD", + currency: @account.currency + ) + + assert_nil AccountStatement::AccountMatcher.new(statement).best_match + end + + test "coverage year selection spans historical account data through last completed month" do + account = Account.create!( + family: @family, + owner: users(:family_admin), + name: "Historical Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + + travel_to Date.new(2026, 5, 6) do + create_statement(account: account, month: Date.new(2024, 2, 1), content: "historical") + + current_year_coverage = AccountStatement::Coverage.for_year(account, nil) + historical_coverage = AccountStatement::Coverage.for_year(account, 2024) + + assert_equal 2026, current_year_coverage.selected_year + assert_equal [ 2026, 2025, 2024 ], current_year_coverage.available_years + + historical_statuses = historical_coverage.months.index_by(&:date).transform_values(&:status) + assert_equal "not_expected", historical_statuses[Date.new(2024, 1, 1)] + assert_equal "covered", historical_statuses[Date.new(2024, 2, 1)] + assert_equal "missing", historical_statuses[Date.new(2024, 3, 1)] + + current_statuses = current_year_coverage.months.index_by(&:date).transform_values(&:status) + assert_equal "missing", current_statuses[Date.new(2026, 4, 1)] + assert_equal "not_expected", current_statuses[Date.new(2026, 5, 1)] + end + end + + test "coverage start can come from balances entries and suggested statements" do + account = Account.create!( + family: @family, + owner: users(:family_admin), + name: "Archive Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + + account.entries.create!( + name: "Old transaction", + date: Date.new(2021, 6, 15), + amount: 10, + currency: "USD", + entryable: Transaction.new + ) + account.balances.create!(date: Date.new(2020, 3, 31), balance: 100, currency: "USD") + create_statement(account: nil, suggested_account: account, month: Date.new(2019, 7, 1), content: "suggested") + + travel_to Date.new(2026, 5, 6) do + coverage = AccountStatement::Coverage.for_year(account, 2019) + statuses = coverage.months.index_by(&:date).transform_values(&:status) + + assert_equal [ 2026, 2025, 2024, 2023, 2022, 2021, 2020, 2019 ], coverage.available_years + assert_equal "not_expected", statuses[Date.new(2019, 6, 1)] + assert_equal "ambiguous", statuses[Date.new(2019, 7, 1)] + end + end + + test "coverage marks covered duplicate ambiguous and mismatched months" do + covered_month = 5.months.ago.to_date.beginning_of_month + missing_month = 4.months.ago.to_date.beginning_of_month + duplicate_month = 3.months.ago.to_date.beginning_of_month + ambiguous_month = 2.months.ago.to_date.beginning_of_month + mismatched_month = 1.month.ago.to_date.beginning_of_month + + create_statement(account: @account, month: covered_month, content: "covered") + create_statement(account: @account, month: duplicate_month, content: "duplicate-a") + create_statement(account: @account, month: duplicate_month, content: "duplicate-b") + create_statement(account: nil, suggested_account: @account, month: ambiguous_month, content: "ambiguous") + create_statement(account: @account, month: mismatched_month, content: "mismatched", closing_balance: 120) + + @account.balances.create!( + date: mismatched_month.end_of_month, + balance: 100, + currency: "USD", + start_cash_balance: 100, + cash_inflows: 0, + cash_outflows: 0 + ) + + coverage = AccountStatement::Coverage.new( + @account, + start_month: covered_month, + end_month: mismatched_month + ) + + statuses = coverage.months.index_by(&:date).transform_values(&:status) + assert_equal "covered", statuses[covered_month] + assert_equal "missing", statuses[missing_month] + assert_equal "duplicate", statuses[duplicate_month] + assert_equal "ambiguous", statuses[ambiguous_month] + assert_equal "mismatched", statuses[mismatched_month] + end + + private + + def create_statement(account:, month:, content:, suggested_account: nil, closing_balance: nil) + statement = AccountStatement.create_from_upload!( + family: @family, + account: account, + file: uploaded_file( + filename: "statement_#{content}_#{month.strftime('%Y-%m')}.csv", + content_type: "text/csv", + content: "date,amount\n#{month},1\n#{month.end_of_month},2\n#{content}\n" + ) + ) + statement.update!( + suggested_account: suggested_account, + period_start_on: month, + period_end_on: month.end_of_month, + closing_balance: closing_balance + ) + statement + end +end diff --git a/test/models/account_test.rb b/test/models/account_test.rb index 766b5ab2f..a903f5d86 100644 --- a/test/models/account_test.rb +++ b/test/models/account_test.rb @@ -205,6 +205,42 @@ class AccountTest < ActiveSupport::TestCase assert_not ActiveStorage::Attachment.exists?(attachment_id) end + test "destroying account moves linked statements to inbox after commit" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.update!(match_confidence: 0.8) + + @account.destroy! + + statement.reload + assert_nil statement.account_id + assert_equal "unmatched", statement.review_status + assert_nil statement.match_confidence + end + + test "rolled back account destroy keeps linked statements unchanged" do + statement = AccountStatement.create_from_upload!( + family: @family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.update!(match_confidence: 0.8) + + Account.transaction do + @account.destroy! + raise ActiveRecord::Rollback + end + + statement.reload + assert Account.exists?(@account.id) + assert_equal @account.id, statement.account_id + assert_equal "linked", statement.review_status + assert_equal 0.8.to_d, statement.match_confidence + end + # Account sharing tests test "owned_by? returns true for account owner" do diff --git a/test/system/settings_test.rb b/test/system/settings_test.rb index 4aef39d0e..1107c2aed 100644 --- a/test/system/settings_test.rb +++ b/test/system/settings_test.rb @@ -26,6 +26,8 @@ class SettingsTest < ApplicationSystemTestCase # Add admin settings if user is admin if @user.admin? + merchants_index = @settings_links.index([ "Merchants", family_merchants_path ]) + @settings_links.insert(merchants_index + 1, [ "Statement Vault", account_statements_path ]) @settings_links += [ [ "AI Prompts", settings_ai_prompts_path ], [ "API Key", settings_api_key_path ] @@ -40,7 +42,7 @@ class SettingsTest < ApplicationSystemTestCase assert_current_path accounts_path, ignore_query: true @settings_links.each do |name, path| - click_link name + click_link name, match: :first assert_selector "h1", text: name assert_current_path path end @@ -52,9 +54,10 @@ class SettingsTest < ApplicationSystemTestCase Rails.application.config.app_mode.stubs(:self_hosted?).returns(true) Provider::Registry.stubs(:get_provider).with(:twelve_data).returns(nil) Provider::Registry.stubs(:get_provider).with(:yahoo_finance).returns(nil) + Provider::Registry.stubs(:get_provider).with(:github).returns(stub(fetch_latest_release_notes: nil)) open_settings_from_sidebar assert_selector "li", text: "Self-Hosting" - click_link "Self-Hosting" + click_link "Self-Hosting", match: :first assert_current_path settings_hosting_path assert_selector "h1", text: "Self-Hosting" find("select#setting_onboarding_state").select("Invite-only") @@ -92,15 +95,17 @@ class SettingsTest < ApplicationSystemTestCase assert_no_selector "li", text: "AI Prompts" assert_no_selector "li", text: "API Key" assert_no_selector "li", text: "Bank sync" + assert_no_selector "li", text: "Statement Vault" end end private def open_settings_from_sidebar - within "div[data-testid=user-menu]" do - find("button").click + user_menu = find("div[data-testid=user-menu]", match: :first, visible: :visible) + within user_menu do + find("[data-DS--menu-target='button']", match: :first).click + click_link "Settings", match: :first end - click_link "Settings" end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 8eb66de7e..461ebe3a6 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -23,6 +23,8 @@ require "minitest/autorun" require "mocha/minitest" require "aasm/minitest" require "webmock/minitest" +require "rack/test" +require "tempfile" require "uri" VCR.configure do |config| @@ -104,6 +106,27 @@ module ActiveSupport "maybetestpassword817983172" end + def uploaded_file(filename:, content_type:, content: "date,amount\n2024-01-01,1\n") + tempfile = Tempfile.new([ File.basename(filename, ".*"), File.extname(filename) ]) + tempfile.binmode + tempfile.write(content) + tempfile.rewind + + Rack::Test::UploadedFile.new(tempfile.path, content_type, true, original_filename: filename) + end + + def family_guest + @family_guest ||= users(:family_admin).family.users.create!( + first_name: "Readonly", + last_name: "Guest", + email: "readonly-guest@example.com", + password: user_password_test, + role: "guest", + onboarded_at: Time.current, + ui_layout: "dashboard" + ) + end + # Ensures the Investment Contributions category exists for a family # Used in transfer tests where this bootstrapped category is required # Uses family locale to ensure consistent naming