` (plus any `extra_params`),
+ # which re-renders `frame` (a Turbo Frame id) with the chosen period. When
+ # `frame` is nil the links fall back to a normal Turbo Drive visit.
+ #
+ # The selected period is marked with a check icon and `aria-current`, and the
+ # trigger button shows its label.
+ #
+ # NOTE: `url` must be a path without a query string; pass query state via
+ # `extra_params` so the picker can compose `?period=…` cleanly.
+ attr_reader :selected_key, :url, :frame, :extra_params, :placement
+
+ def initialize(selected:, url:, frame: nil, extra_params: {}, placement: "bottom-end")
+ @selected_key = selected.respond_to?(:key) ? selected.key : selected.to_s
+ @url = url
+ @frame = frame
+ @extra_params = (extra_params || {}).symbolize_keys
+ @placement = placement
+ end
+
+ def periods
+ Period.all
+ end
+
+ def selected_label
+ period_for(selected_key).label_short
+ end
+
+ def selected?(key)
+ key == selected_key
+ end
+
+ def href_for(key)
+ "#{url}?#{extra_params.merge(period: key).to_query}"
+ end
+
+ private
+ def period_for(key)
+ Period.from_key(key)
+ rescue Period::InvalidKeyError
+ Period.last_30_days
+ end
+end
diff --git a/app/controllers/api/v1/import_sessions_controller.rb b/app/controllers/api/v1/import_sessions_controller.rb
new file mode 100644
index 000000000..f749124d7
--- /dev/null
+++ b/app/controllers/api/v1/import_sessions_controller.rb
@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+class Api::V1::ImportSessionsController < Api::V1::BaseController
+ before_action :ensure_read_scope, only: [ :show ]
+ before_action :ensure_write_scope, only: [ :create, :create_chunk, :publish ]
+ before_action :set_import_session, only: [ :show, :create_chunk, :publish ]
+
+ def create
+ @import_session = ImportSession.create_or_find_for!(
+ family: Current.family,
+ import_type: params[:type].to_s,
+ client_session_id: params[:client_session_id].presence,
+ expected_chunks: expected_chunks_param
+ )
+
+ render_import_session(status: :created)
+ rescue ImportSession::ConflictError => e
+ render_import_session_conflict(e.message)
+ rescue ActiveRecord::RecordInvalid => e
+ render_error(
+ "validation_failed",
+ "Import session could not be created",
+ :unprocessable_entity,
+ errors: e.record.errors.full_messages
+ )
+ end
+
+ def show
+ render_import_session
+ end
+
+ def create_chunk
+ content, filename, content_type = sure_import_upload_attributes
+ return unless content
+
+ @import_session.attach_chunk!(
+ sequence: sequence_param,
+ client_chunk_id: params[:client_chunk_id].presence,
+ content: content,
+ filename: filename,
+ content_type: content_type
+ )
+
+ @import_session.reload
+ render_import_session(status: :created)
+ rescue ImportSession::ConflictError => e
+ render_import_session_conflict(e.message)
+ rescue ActiveRecord::RecordInvalid => e
+ render_error(
+ "validation_failed",
+ "Import chunk could not be created",
+ :unprocessable_entity,
+ errors: e.record.errors.full_messages
+ )
+ end
+
+ def publish
+ @import_session.publish_later
+ @import_session.reload
+ render_import_session(status: :accepted)
+ rescue Import::MaxRowCountExceededError
+ render_error("max_row_count_exceeded", "Import session has too many rows to publish.", :unprocessable_entity)
+ rescue ImportSession::EnqueueError
+ render_error("import_enqueue_failed", "Import session could not be queued.", :service_unavailable)
+ rescue ImportSession::ConflictError => e
+ render_import_session_conflict(e.message)
+ end
+
+ private
+ def set_import_session
+ @import_session = Current.family.import_sessions.find(params[:id])
+ end
+
+ def ensure_read_scope
+ authorize_scope!(:read)
+ end
+
+ def ensure_write_scope
+ authorize_scope!(:write)
+ end
+
+ def expected_chunks_param
+ return if params[:expected_chunks].blank?
+
+ params[:expected_chunks]
+ end
+
+ def sequence_param
+ raise ActionController::ParameterMissing.new(:sequence) if params[:sequence].blank?
+
+ params[:sequence]
+ end
+
+ def sure_import_upload_attributes
+ if params[:file].present?
+ sure_import_file_upload_attributes(params[:file])
+ elsif params[:raw_file_content].present?
+ sure_import_raw_content_attributes(params[:raw_file_content].to_s)
+ else
+ render_error("missing_content", "Provide a Sure NDJSON file or raw_file_content.", :unprocessable_entity)
+ nil
+ end
+ end
+
+ def sure_import_file_upload_attributes(file)
+ if file.size > SureImport.max_ndjson_size
+ render_error(
+ "file_too_large",
+ "File is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB.",
+ :unprocessable_entity
+ )
+ return
+ end
+
+ extension = File.extname(file.original_filename.to_s).downcase
+ unless SureImport::ALLOWED_NDJSON_CONTENT_TYPES.include?(file.content_type) || extension.in?(%w[.ndjson .json])
+ render_error("invalid_file_type", "Invalid file type. Please upload a Sure NDJSON file.", :unprocessable_entity)
+ return
+ end
+
+ sure_import_validated_attributes(
+ content: file.read,
+ filename: file.original_filename.presence || "sure-import.ndjson",
+ content_type: file.content_type.presence || "application/x-ndjson"
+ )
+ end
+
+ def sure_import_raw_content_attributes(content)
+ if content.bytesize > SureImport.max_ndjson_size
+ render_error(
+ "content_too_large",
+ "Content is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB.",
+ :unprocessable_entity
+ )
+ return
+ end
+
+ sure_import_validated_attributes(
+ content: content,
+ filename: "sure-import.ndjson",
+ content_type: "application/x-ndjson"
+ )
+ end
+
+ def sure_import_validated_attributes(content:, filename:, content_type:)
+ unless SureImport.valid_ndjson_first_line?(content)
+ render_error("invalid_ndjson", "Invalid Sure NDJSON content.", :unprocessable_entity)
+ return
+ end
+
+ [ content, filename, content_type ]
+ end
+
+ def render_import_session_conflict(message)
+ render_error("import_session_conflict", message, :conflict)
+ end
+
+ def render_import_session(status: :ok)
+ chunks = @import_session.imports.ordered_by_sequence.map do |import|
+ {
+ id: import.id,
+ sequence: import.sequence,
+ client_chunk_id: import.client_chunk_id,
+ status: import.status,
+ rows_count: import.rows_count,
+ summary: import.summary || {},
+ error: import.error_details.presence,
+ created_at: import.created_at,
+ updated_at: import.updated_at
+ }
+ end
+
+ render json: {
+ data: {
+ id: @import_session.id,
+ type: @import_session.import_type,
+ status: @import_session.status,
+ client_session_id: @import_session.client_session_id,
+ expected_chunks: @import_session.expected_chunks,
+ chunks_count: chunks.size,
+ summary: @import_session.summary || {},
+ error: @import_session.error_details.presence,
+ created_at: @import_session.created_at,
+ updated_at: @import_session.updated_at,
+ chunks: chunks
+ }
+ }, status: status
+ end
+
+ def render_error(error, message, status, errors: nil)
+ payload = { error: error, message: message }
+ payload[:errors] = errors if errors
+ render json: payload, status: status
+ end
+end
diff --git a/app/controllers/concerns/accountable_resource.rb b/app/controllers/concerns/accountable_resource.rb
index 479ad9efc..18e38b15e 100644
--- a/app/controllers/concerns/accountable_resource.rb
+++ b/app/controllers/concerns/accountable_resource.rb
@@ -48,7 +48,14 @@ module AccountableResource
@account.lock_saved_attributes!
end
- redirect_to account_params[:return_to].presence || @account, notice: t("accounts.create.success", type: accountable_type.name.underscore.humanize)
+ # Prefer the form-carried return_to, then the session value StoreLocation
+ # captured from `?return_to=` (survives multi-step flows where the param
+ # isn't threaded), then the account page. The form param is sanitized here
+ # (the session value is already filtered at store time); the session is
+ # consumed with delete so a stale value can't leak into a later flow.
+ return_path = safe_return_to(account_params[:return_to]) || session.delete(:return_to).presence || @account
+ redirect_to return_path,
+ notice: t("accounts.create.success", type: accountable_type.name.underscore.humanize)
end
def update
diff --git a/app/controllers/concerns/store_location.rb b/app/controllers/concerns/store_location.rb
index e2e8d3181..14f98068d 100644
--- a/app/controllers/concerns/store_location.rb
+++ b/app/controllers/concerns/store_location.rb
@@ -24,9 +24,20 @@ private
end
def store_return_to
- if params[:return_to].present?
- session[:return_to] = params[:return_to]
- end
+ safe = safe_return_to(params[:return_to])
+ session[:return_to] = safe if safe
+ end
+
+ # Only allow internal absolute paths (a single leading "/"). Blocks absolute
+ # URLs, protocol-relative ("//evil"), and backslash tricks ("/\\evil") so a
+ # crafted ?return_to= can't open-redirect — including via a custom
+ # turbo_stream redirect, which Rails' redirect host-guard does NOT cover
+ # (the client `Turbo.visit`es the target and full-navigates cross-origin).
+ def safe_return_to(value)
+ # is_a?(String) first: a crafted `?return_to[]=foo` makes params[:return_to]
+ # an Array, and Array#match? doesn't exist — without this guard the helper
+ # raises NoMethodError before the redirect hardening can reject it.
+ value if value.is_a?(String) && value.present? && value.match?(%r{\A/(?![/\\])})
end
def clear_previous_path
diff --git a/app/controllers/enable_banking_items_controller.rb b/app/controllers/enable_banking_items_controller.rb
index 0bfc0dabb..901d0589c 100644
--- a/app/controllers/enable_banking_items_controller.rb
+++ b/app/controllers/enable_banking_items_controller.rb
@@ -131,39 +131,6 @@ class EnableBankingItemsController < ApplicationController
return
end
- # Re-fetch ASPSP list from provider to avoid session cookie overflow.
- # We do not store full ASPSP metadata in the session to stay within the 4KB limit;
- # instead, we re-query the provider here for the final authorization parameters.
- aspsp_data = nil
- begin
- provider_for_lookup = @enable_banking_item.enable_banking_provider
- if provider_for_lookup
- response = provider_for_lookup.get_aspsps(country: @enable_banking_item.country_code)
- raw_aspsps = response[:aspsps] || response["aspsps"] || []
- found = raw_aspsps.find { |a| a[:name] == aspsp_name || a["name"] == aspsp_name }
- aspsp_data = found&.with_indifferent_access
- end
- rescue Provider::EnableBanking::EnableBankingError => e
- Rails.logger.warn "Enable Banking: could not fetch ASPSP metadata in authorize: #{e.message}"
- end
-
- # Block DECOUPLED banks — our OAuth redirect flow doesn't support them
- if aspsp_data.present?
- # Adjust psu_type if the bank does not support the requested type
- supported_types = Array(aspsp_data[:psu_types]).map(&:to_s)
- if supported_types.any? && !supported_types.include?(psu_type)
- psu_type = supported_types.first
- end
-
- first_method = Array(aspsp_data[:auth_methods]).first
- approach = first_method&.dig(:approach) || first_method&.dig("approach")
- if approach == "DECOUPLED"
- redirect_to settings_providers_path, alert: t(".decoupled_not_supported",
- default: "This bank uses a separate device authentication method which is not yet supported. Please add this account manually.")
- return
- end
- end
-
begin
target_item = if params[:new_connection] == "true"
Current.family.enable_banking_items.create!(
@@ -181,12 +148,14 @@ class EnableBankingItemsController < ApplicationController
language = I18n.locale.to_s.split("-").first
- redirect_url = target_item.start_authorization(
+ # begin_authorization! re-fetches ASPSP metadata and auto-selects the best
+ # auth method (REDIRECT > DECOUPLED > EMBEDDED). Decoupled/MFA banks proceed
+ # through Enable Banking's hosted SCA page rather than being blocked.
+ redirect_url = target_item.begin_authorization!(
aspsp_name: aspsp_name,
redirect_url: enable_banking_callback_url,
state: target_item.id,
psu_type: psu_type,
- aspsp_data: aspsp_data,
language: language
)
@@ -269,11 +238,11 @@ class EnableBankingItemsController < ApplicationController
begin
language = I18n.locale.to_s.split("-").first
- redirect_url = @enable_banking_item.start_authorization(
- aspsp_name: @enable_banking_item.aspsp_name,
+ # Route through the shared path so reauthorization re-selects the same auth
+ # method (decoupled banks included) instead of falling back to a default.
+ redirect_url = @enable_banking_item.begin_authorization!(
redirect_url: enable_banking_callback_url,
state: @enable_banking_item.id,
- psu_type: @enable_banking_item.psu_type || "personal",
language: language
)
diff --git a/app/controllers/goals_controller.rb b/app/controllers/goals_controller.rb
index c54a44371..1bb8f3c8a 100644
--- a/app/controllers/goals_controller.rb
+++ b/app/controllers/goals_controller.rb
@@ -26,7 +26,7 @@ class GoalsController < ApplicationController
# entirely (rendered with filterable: false).
@grid_goals = @active_goals + @completed_goals
- @linkable_account_count = Current.family.accounts.where(accountable_type: "Depository").visible.count
+ @linkable_account_count = Current.user.accessible_accounts.where(accountable_type: "Depository").visible.count
@kpi = kpi_payload(@active_goals)
@any_pending_pledge = @active_goals.any? { |g| g.open_pledges.any? }
@show_search = @grid_goals.size > 6
@@ -61,7 +61,7 @@ class GoalsController < ApplicationController
def create
@goal = Current.family.goals.new(goal_params)
accounts = lookup_accounts(params.dig(:goal, :account_ids))
- @goal.currency = accounts.first.currency if accounts.any? && @goal.currency.blank?
+ @goal.currency = (accounts.first&.currency || Current.family.primary_currency_code) if @goal.currency.blank?
Goal.transaction do
accounts.each { |a| @goal.goal_accounts.build(account: a) }
@@ -169,18 +169,24 @@ class GoalsController < ApplicationController
return [] if ids.blank?
ids = Array(ids).reject(&:blank?)
- Current.family.accounts.where(accountable_type: "Depository").visible.where(id: ids).to_a
+ Current.user.accessible_accounts.where(accountable_type: "Depository").visible.where(id: ids).to_a
end
def linkable_accounts_for_new
- Current.family.accounts.where(accountable_type: "Depository").visible.alphabetically.to_a
+ Current.user.accessible_accounts.where(accountable_type: "Depository").visible.alphabetically.to_a
end
def sync_linked_accounts!(goal, accounts)
desired_ids = accounts.map(&:id).to_set
current_ids = goal.goal_accounts.pluck(:account_id).to_set
- (current_ids - desired_ids).each do |id|
+ # Only unlink accounts the current user can actually see in the picker.
+ # A family goal may be linked to another member's private account, which
+ # never renders as a checkbox — so its absence from the submitted set is
+ # not an intentional removal and must not destroy the link.
+ removable_ids = Current.user.accessible_accounts.where(id: current_ids.to_a).pluck(:id).to_set
+
+ ((current_ids & removable_ids) - desired_ids).each do |id|
goal.goal_accounts.where(account_id: id).destroy_all
end
additions = accounts.reject { |a| current_ids.include?(a.id) }
diff --git a/app/controllers/properties_controller.rb b/app/controllers/properties_controller.rb
index 1de7a5f8c..9b00d4cee 100644
--- a/app/controllers/properties_controller.rb
+++ b/app/controllers/properties_controller.rb
@@ -75,9 +75,18 @@ class PropertiesController < ApplicationController
if @account.draft?
@account.activate!
+ # The property setup wizard (create → balances → address) is multi-step,
+ # so the original `?return_to=` only survives in the session (captured by
+ # StoreLocation), not as a threaded form param. Honor it on completion so
+ # flows like the savings-goals "Add an account" CTA land back where they
+ # started instead of on the account page. Sanitized + consumed: the
+ # turbo_stream branch below isn't covered by Rails' redirect host-guard,
+ # so an unsafe value must not reach stream_redirect_to.
+ return_path = safe_return_to(session.delete(:return_to)) || account_path(@account)
+
respond_to do |format|
- format.html { redirect_to account_path(@account) }
- format.turbo_stream { stream_redirect_to account_path(@account) }
+ format.html { redirect_to return_path }
+ format.turbo_stream { stream_redirect_to return_path }
end
else
@success_message = "Address updated successfully."
diff --git a/app/javascript/controllers/time_series_chart_controller.js b/app/javascript/controllers/time_series_chart_controller.js
index 0628d0315..3c358dd96 100644
--- a/app/javascript/controllers/time_series_chart_controller.js
+++ b/app/javascript/controllers/time_series_chart_controller.js
@@ -140,7 +140,18 @@ export default class extends Controller {
.attr("d", this._d3Line)
.attr("stroke-linejoin", "round")
.attr("stroke-linecap", "round")
- .attr("stroke-width", this.strokeWidthValue);
+ // A flat series (no variation across the period — a single valuation or an
+ // unchanged balance) otherwise renders as a full-bleed near-black rule
+ // bisecting the hero card. Draw it as a faint hairline so it reads as
+ // "no change", consistent across light and dark (#2137).
+ .attr("stroke-width", this._isFlatSeries ? 1 : this.strokeWidthValue)
+ .attr("stroke-opacity", this._isFlatSeries ? 0.4 : 1);
+ }
+
+ get _isFlatSeries() {
+ const min = d3.min(this._normalDataPoints, this._getDatumValue);
+ const max = d3.max(this._normalDataPoints, this._getDatumValue);
+ return min === max;
}
_installTrendlineSplit() {
diff --git a/app/jobs/import_session_job.rb b/app/jobs/import_session_job.rb
new file mode 100644
index 000000000..de7f0e377
--- /dev/null
+++ b/app/jobs/import_session_job.rb
@@ -0,0 +1,12 @@
+class ImportSessionJob < ApplicationJob
+ queue_as :high_priority
+
+ def perform(import_session)
+ raise ArgumentError, "ImportSessionJob requires an import_session" if import_session.nil?
+
+ Rails.logger.info("ImportSessionJob started import_session_id=#{import_session.id}")
+ import_session.publish
+ import_session.reload
+ Rails.logger.info("ImportSessionJob finished import_session_id=#{import_session.id} status=#{import_session.status}")
+ end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index 48a64b796..a2c144b8e 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -41,7 +41,11 @@ class Account < ApplicationRecord
enum :classification, { asset: "asset", liability: "liability" }, validate: { allow_nil: true }
- scope :visible, -> { where(status: [ "draft", "active" ]) }
+ VISIBLE_STATUSES = %w[draft active].freeze
+ HISTORICAL_STATUSES = (VISIBLE_STATUSES + %w[disabled]).freeze
+
+ scope :visible, -> { where(status: VISIBLE_STATUSES) }
+ scope :historical, -> { where(status: HISTORICAL_STATUSES) }
scope :assets, -> { where(classification: "asset") }
scope :liabilities, -> { where(classification: "liability") }
scope :alphabetically, -> { order(:name) }
diff --git a/app/models/assistant/function/import_bank_statement.rb b/app/models/assistant/function/import_bank_statement.rb
index dee54602f..d3bf86dec 100644
--- a/app/models/assistant/function/import_bank_statement.rb
+++ b/app/models/assistant/function/import_bank_statement.rb
@@ -93,6 +93,8 @@ class Assistant::Function::ImportBankStatement < Assistant::Function
end
# Extract transactions from the PDF using provider
+ # TODO(#2113): hardcoded to OpenAI. Provider::Anthropic implements
+ # extract_bank_statement (PR #1985); this should honor Setting.llm_provider.
provider = Provider::Registry.get_provider(:openai)
unless provider
return {
diff --git a/app/models/balance/chart_series_builder.rb b/app/models/balance/chart_series_builder.rb
index c8c733579..b6aa9f7bc 100644
--- a/app/models/balance/chart_series_builder.rb
+++ b/app/models/balance/chart_series_builder.rb
@@ -1,10 +1,14 @@
class Balance::ChartSeriesBuilder
- def initialize(account_ids:, currency:, period: Period.last_30_days, interval: nil, favorable_direction: "up")
+ def initialize(account_ids:, currency:, period: Period.last_30_days, interval: nil,
+ favorable_direction: "up", account_active_until_dates: {})
@account_ids = account_ids
@currency = currency
@period = period
@interval = interval
@favorable_direction = favorable_direction
+ @account_active_until_dates = account_active_until_dates.compact
+ .transform_keys(&:to_s)
+ .transform_values { |date| date.to_date.iso8601 }
end
def balance_series
@@ -29,7 +33,7 @@ class Balance::ChartSeriesBuilder
end
private
- attr_reader :account_ids, :currency, :period, :favorable_direction
+ attr_reader :account_ids, :currency, :period, :favorable_direction, :account_active_until_dates
def interval
@interval || period.interval
@@ -74,7 +78,8 @@ class Balance::ChartSeriesBuilder
start_date: period.start_date,
end_date: period.end_date,
interval: interval,
- sign_multiplier: sign_multiplier
+ sign_multiplier: sign_multiplier,
+ account_active_until_dates_json: account_active_until_dates.to_json
}
])
rescue => e
@@ -96,6 +101,19 @@ class Balance::ChartSeriesBuilder
SELECT generate_series(DATE :start_date, DATE :end_date, :interval::interval)::date AS date
UNION DISTINCT
SELECT :end_date::date -- Ensure end date is included
+ ),
+ account_windows AS (
+ SELECT
+ account_window.account_id::uuid AS account_id,
+ account_window.active_until_date::date AS active_until_date
+ FROM jsonb_each_text(CAST(:account_active_until_dates_json AS jsonb))
+ AS account_window(account_id, active_until_date)
+ ),
+ selected_accounts AS (
+ SELECT accounts.*, account_windows.active_until_date
+ FROM accounts
+ LEFT JOIN account_windows ON account_windows.account_id = accounts.id
+ WHERE accounts.id = ANY(array[:account_ids]::uuid[])
)
SELECT
d.date,
@@ -119,7 +137,8 @@ class Balance::ChartSeriesBuilder
END * COALESCE(er.rate, 1) * :sign_multiplier::integer
), 0) AS start_holdings_balance
FROM dates d
- CROSS JOIN accounts
+ LEFT JOIN selected_accounts accounts
+ ON accounts.active_until_date IS NULL OR d.date <= accounts.active_until_date
LEFT JOIN LATERAL (
SELECT b.end_balance,
b.end_cash_balance,
@@ -153,7 +172,6 @@ class Balance::ChartSeriesBuilder
LIMIT 1)
) AS rate
) er ON TRUE
- WHERE accounts.id = ANY(array[:account_ids]::uuid[])
GROUP BY d.date
ORDER BY d.date
SQL
diff --git a/app/models/balance_sheet/historical_account_scope.rb b/app/models/balance_sheet/historical_account_scope.rb
new file mode 100644
index 000000000..2906908e9
--- /dev/null
+++ b/app/models/balance_sheet/historical_account_scope.rb
@@ -0,0 +1,18 @@
+class BalanceSheet::HistoricalAccountScope
+ def initialize(family, user: nil)
+ @family = family
+ @user = user
+ end
+
+ def account_ids
+ relation.pluck(:id)
+ end
+
+ def relation
+ scope = family.accounts.historical
+ user.present? ? scope.included_in_finances_for(user) : scope
+ end
+
+ private
+ attr_reader :family, :user
+end
diff --git a/app/models/balance_sheet/net_worth_series_builder.rb b/app/models/balance_sheet/net_worth_series_builder.rb
index 7c29a6ece..58f97d321 100644
--- a/app/models/balance_sheet/net_worth_series_builder.rb
+++ b/app/models/balance_sheet/net_worth_series_builder.rb
@@ -7,7 +7,8 @@ class BalanceSheet::NetWorthSeriesBuilder
def net_worth_series(period: Period.last_30_days)
Rails.cache.fetch(cache_key(period)) do
builder = Balance::ChartSeriesBuilder.new(
- account_ids: visible_account_ids,
+ account_ids: historical_account_ids,
+ account_active_until_dates: disabled_account_active_until_dates,
currency: family.currency,
period: period,
favorable_direction: "up"
@@ -20,18 +21,31 @@ class BalanceSheet::NetWorthSeriesBuilder
private
attr_reader :family, :user
- def visible_account_ids
- @visible_account_ids ||= begin
- scope = family.accounts.visible
- scope = scope.included_in_finances_for(user) if user
- scope.pluck(:id)
+ def historical_accounts
+ @historical_accounts ||= historical_account_scope.relation.to_a
+ end
+
+ def historical_account_ids
+ @historical_account_ids ||= historical_accounts.map(&:id)
+ end
+
+ def disabled_account_active_until_dates
+ @disabled_account_active_until_dates ||= historical_accounts.each_with_object({}) do |account, dates|
+ next unless account.disabled?
+
+ disabled_on = (account.disabled_at || account.updated_at).to_date
+ dates[account.id] = disabled_on - 1.day
end
end
+ def historical_account_scope
+ @historical_account_scope ||= BalanceSheet::HistoricalAccountScope.new(family, user: user)
+ end
+
def cache_key(period)
shares_version = user ? AccountShare.where(user: user).maximum(:updated_at)&.to_i : nil
key = [
- "balance_sheet_net_worth_series",
+ "balance_sheet_net_worth_series_historical",
user&.id,
shares_version,
period.start_date,
diff --git a/app/models/enable_banking_item.rb b/app/models/enable_banking_item.rb
index 7f3b6a540..87af6ec78 100644
--- a/app/models/enable_banking_item.rb
+++ b/app/models/enable_banking_item.rb
@@ -73,19 +73,31 @@ class EnableBankingItem < ApplicationRecord
raise StandardError.new("Enable Banking provider is not configured") unless provider
validated_psu_type = psu_type
+ selected_method = nil
# Store ASPSP metadata before calling provider so it's available even if auth fails
if aspsp_data.present?
aspsp_data = aspsp_data.with_indifferent_access
- first_auth_method = aspsp_data.dig(:auth_methods, 0) || aspsp_data.dig("auth_methods", 0)
- aspsp_types = aspsp_data[:psu_types] || []
+ aspsp_types = Array(aspsp_data[:psu_types]).map(&:to_s)
+
+ # If the requested PSU type isn't supported by this ASPSP, fall back to the
+ # first type it advertises rather than failing outright.
+ validated_psu_type = if psu_type.present? && aspsp_types.include?(psu_type)
+ psu_type
+ elsif aspsp_types.any?
+ aspsp_types.first
+ else
+ psu_type
+ end
+
+ selected_method = select_auth_method(aspsp_data, validated_psu_type)
+
update!(
aspsp_required_psu_headers: aspsp_data[:required_psu_headers] || [],
aspsp_maximum_consent_validity: aspsp_data[:maximum_consent_validity],
- aspsp_auth_approach: first_auth_method&.dig(:approach) || first_auth_method&.dig("approach"),
+ aspsp_auth_approach: selected_method&.dig(:approach),
aspsp_psu_types: aspsp_types
)
- validated_psu_type = psu_type.present? && aspsp_types.include?(psu_type) ? psu_type : nil
end
result = provider.start_authorization(
@@ -95,7 +107,8 @@ class EnableBankingItem < ApplicationRecord
state: state,
psu_type: validated_psu_type,
maximum_consent_validity: aspsp_maximum_consent_validity,
- language: language
+ language: language,
+ auth_method: selected_method&.dig(:name)
)
attributes = {
@@ -109,6 +122,26 @@ class EnableBankingItem < ApplicationRecord
result[:url]
end
+ # Shared entry point for both initial authorization and reauthorization.
+ # Re-fetches ASPSP metadata (so the auth method / PSU type selection and the
+ # stored approach stay accurate) and starts the provider authorization. The
+ # re-fetch — rather than caching the full ASPSP object in the session — keeps
+ # us under the 4KB session cookie limit.
+ # @return [String] Redirect URL for the user
+ def begin_authorization!(redirect_url:, state:, language: nil, psu_type: nil, aspsp_name: nil)
+ name = aspsp_name.presence || self.aspsp_name
+ raise StandardError.new("No bank selected for this connection") if name.blank?
+
+ start_authorization(
+ aspsp_name: name,
+ redirect_url: redirect_url,
+ state: state,
+ psu_type: psu_type.presence || self.psu_type || "personal",
+ aspsp_data: fetch_aspsp_data(name),
+ language: language
+ )
+ end
+
# Complete the authorization flow with the code from callback
def complete_authorization(code:)
provider = enable_banking_provider
@@ -130,6 +163,27 @@ class EnableBankingItem < ApplicationRecord
result
end
+ # Reconcile the locally-stored session expiry with what the API reports.
+ # The session info returned by GET /sessions carries the authoritative
+ # access.valid_until; persisting it on every sync keeps session_valid? accurate
+ # and avoids both premature "expired" states and stale "still valid" states.
+ def reconcile_session_expiry!(session_data)
+ return unless session_data.is_a?(Hash)
+
+ valid_until = session_data.dig(:access, :valid_until) || session_data.dig("access", "valid_until")
+ return if valid_until.blank?
+
+ parsed = Time.zone.parse(valid_until.to_s)
+ return if parsed.nil? || parsed == session_expires_at
+
+ update!(session_expires_at: parsed)
+ rescue ArgumentError, TypeError, ActiveRecord::ActiveRecordError => e
+ # Best-effort reconciliation: swallow bad timestamps (ArgumentError/TypeError)
+ # as well as validation/locking failures from update! (RecordInvalid,
+ # StaleObjectError) so a sync is never derailed by expiry bookkeeping.
+ Rails.logger.warn "EnableBankingItem #{id} - Failed to reconcile session expiry: #{e.message}"
+ end
+
def import_latest_enable_banking_data
provider = enable_banking_provider
unless provider
@@ -288,6 +342,51 @@ class EnableBankingItem < ApplicationRecord
private
+ # Authentication approach preference, lowest number wins.
+ # REDIRECT is the smoothest (PSU authenticates entirely on the ASPSP page).
+ # DECOUPLED works through Enable Banking's hosted page (push-to-app / photoTAN
+ # / chipTAN). EMBEDDED is last resort (handled by the hosted page too).
+ AUTH_APPROACH_PRIORITY = { "REDIRECT" => 0, "DECOUPLED" => 1, "EMBEDDED" => 2 }.freeze
+
+ # Choose the best authentication method for the given PSU type.
+ # Returns a hash with :name and :approach, or nil when the ASPSP exposes no
+ # API-selectable methods (Enable Banking then falls back to its default).
+ def select_auth_method(aspsp_data, psu_type)
+ methods = Array(aspsp_data[:auth_methods]).map(&:with_indifferent_access)
+ return nil if methods.empty?
+
+ # Hidden methods aren't surfaced on Enable Banking's hosted page, so we don't
+ # auto-select one (the PSU couldn't complete it). If every method is hidden,
+ # return nil and let /auth fall back to the ASPSP's default rather than
+ # forcing a non-selectable method.
+ methods = methods.reject { |m| ActiveModel::Type::Boolean.new.cast(m[:hidden_method]) }
+ return nil if methods.empty?
+
+ # Prefer methods that match the chosen PSU type; if none declare a psu_type
+ # (or none match), consider all of them.
+ matching = methods.select { |m| m[:psu_type].blank? || m[:psu_type].to_s == psu_type.to_s }
+ candidates = matching.presence || methods
+
+ best = candidates.min_by { |m| AUTH_APPROACH_PRIORITY.fetch(m[:approach].to_s, 99) }
+ return nil unless best
+
+ { name: best[:name], approach: best[:approach] }
+ end
+
+ # Fetch the ASPSP object for a given name from the provider's /aspsps list.
+ # Returns a HashWithIndifferentAccess, or nil if unavailable.
+ def fetch_aspsp_data(aspsp_name)
+ provider = enable_banking_provider
+ return nil unless provider
+
+ response = provider.get_aspsps(country: country_code)
+ raw_aspsps = response[:aspsps] || response["aspsps"] || []
+ raw_aspsps.find { |a| (a[:name] || a["name"]) == aspsp_name }&.with_indifferent_access
+ rescue Provider::EnableBanking::EnableBankingError => e
+ Rails.logger.warn "EnableBankingItem #{id} - could not fetch ASPSP metadata for #{aspsp_name}: #{e.message}"
+ nil
+ end
+
def parse_session_expiry(session_result)
if session_result[:access].present? && session_result[:access][:valid_until].present?
parsed = Time.zone.parse(session_result[:access][:valid_until])
diff --git a/app/models/enable_banking_item/importer.rb b/app/models/enable_banking_item/importer.rb
index 696b4f673..d08f0c5f5 100644
--- a/app/models/enable_banking_item/importer.rb
+++ b/app/models/enable_banking_item/importer.rb
@@ -124,14 +124,20 @@ class EnableBankingItem::Importer
private
- def handle_sync_error(exception)
+ # @param session_level [Boolean] true only for the top-level GET /sessions call.
+ # A session-level 401/404 means the consent is genuinely dead and the user
+ # must re-authorize. Per-account 401/404 (a stale account UID, a transient
+ # hiccup on one account) must NOT mark the whole connection requires_update —
+ # doing so is what made every sync report "session expired". Those are recorded
+ # as ordinary sync errors and retried on the next sync.
+ def handle_sync_error(exception, session_level: false)
# Check the underlying cause first, then the exception itself
exceptions = [ exception.cause, exception ].compact
provider_error = exceptions.find { |ex| ex.is_a?(Provider::EnableBanking::EnableBankingError) }
- # Handle session expiration status update
- if provider_error && [ :unauthorized, :not_found ].include?(provider_error.error_type)
+ # Handle session expiration status update (session-level failures only)
+ if session_level && provider_error && [ :unauthorized, :not_found ].include?(provider_error.error_type)
enable_banking_item.update!(status: :requires_update)
return I18n.t("enable_banking_items.errors.session_invalid")
end
@@ -151,14 +157,18 @@ class EnableBankingItem::Importer
end
def fetch_session_data
- enable_banking_provider.get_session(session_id: enable_banking_item.session_id)
+ session_data = enable_banking_provider.get_session(session_id: enable_banking_item.session_id)
+ # Keep the local expiry in sync with the authoritative value from the API so
+ # session_valid? doesn't drift (premature "expired" or stale "still valid").
+ enable_banking_item.reconcile_session_expiry!(session_data)
+ session_data
rescue Provider::EnableBanking::EnableBankingError => e
Rails.logger.error "EnableBankingItem::Importer - Enable Banking API error: #{e.message}"
- @session_error = handle_sync_error(e)
+ @session_error = handle_sync_error(e, session_level: true)
nil
rescue => e
Rails.logger.error "EnableBankingItem::Importer - Unexpected error fetching session: #{e.class} - #{e.message}"
- @session_error = handle_sync_error(e)
+ @session_error = handle_sync_error(e, session_level: true)
nil
end
diff --git a/app/models/enable_banking_item/syncer.rb b/app/models/enable_banking_item/syncer.rb
index dfe8ce984..3049a05eb 100644
--- a/app/models/enable_banking_item/syncer.rb
+++ b/app/models/enable_banking_item/syncer.rb
@@ -8,11 +8,14 @@ class EnableBankingItem::Syncer
end
def perform_sync(sync)
- # Check if session is valid before syncing
+ # An expired/missing session is an expected state that needs user action, not a
+ # hard failure. Mark the connection requires_update and finish the sync
+ # gracefully so the UI surfaces the "Reconnect" CTA instead of a red sync error.
unless enable_banking_item.session_valid?
sync.update!(status_text: "Session expired - re-authorization required") if sync.respond_to?(:status_text)
enable_banking_item.update!(status: :requires_update)
- raise StandardError.new("Enable Banking session has expired. Please re-authorize.")
+ collect_health_stats(sync, errors: nil)
+ return
end
# Phase 1: Import data from Enable Banking API
@@ -20,6 +23,16 @@ class EnableBankingItem::Syncer
import_result = enable_banking_item.import_latest_enable_banking_data
unless import_result[:success]
+ # A session-level auth failure detected mid-import flips the item to
+ # requires_update — surface that as a graceful reconnect state, not a red
+ # error. Transient/per-account failures leave status good and fall through
+ # to a normal sync error that retries next time.
+ if enable_banking_item.requires_update?
+ sync.update!(status_text: "Re-authorization required") if sync.respond_to?(:status_text)
+ collect_health_stats(sync, errors: nil)
+ return
+ end
+
error_msg = import_result[:error]
if error_msg.blank? && (import_result[:accounts_failed].to_i > 0 || import_result[:transactions_failed].to_i > 0)
parts = []
diff --git a/app/models/family.rb b/app/models/family.rb
index d4c6cf1e4..c5f8f2252 100644
--- a/app/models/family.rb
+++ b/app/models/family.rb
@@ -27,6 +27,8 @@ class Family < ApplicationRecord
has_many :invitations, dependent: :destroy
has_many :imports, dependent: :destroy
+ has_many :import_sessions, dependent: :destroy
+ has_many :import_source_mappings, dependent: :destroy
has_many :family_exports, dependent: :destroy
has_many :account_statements, dependent: :destroy
diff --git a/app/models/family/data_exporter.rb b/app/models/family/data_exporter.rb
index 404218056..20e08a70d 100644
--- a/app/models/family/data_exporter.rb
+++ b/app/models/family/data_exporter.rb
@@ -545,6 +545,7 @@ class Family::DataExporter
def serialize_rule_for_export(rule)
{
+ id: rule.id,
name: rule.name,
resource_type: rule.resource_type,
active: rule.active,
diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb
index 4de55a9bc..7a68419c5 100644
--- a/app/models/family/data_importer.rb
+++ b/app/models/family/data_importer.rb
@@ -1,6 +1,36 @@
require "set"
class Family::DataImporter
+ MissingReferenceError = Class.new(StandardError) do
+ attr_reader :code, :details
+
+ def initialize(record_type:, source_type:, source_id:)
+ @code = "missing_source_reference"
+ @details = {
+ record_type: record_type,
+ source_type: source_type,
+ source_id: source_id
+ }
+
+ super("#{record_type} references missing #{source_type} source id #{source_id}")
+ end
+ end
+
+ InvalidRecordError = Class.new(StandardError) do
+ attr_reader :code, :details
+
+ def initialize(record_type:, field:, value:)
+ @code = "invalid_import_record"
+ @details = {
+ record_type: record_type,
+ field: field,
+ value: value
+ }
+
+ super("#{record_type} has invalid #{field}: #{value.inspect}")
+ end
+ end
+
SUPPORTED_TYPES = %w[Account Balance Category Tag Merchant RecurringTransaction Transaction Transfer RejectedTransfer Trade Holding Valuation Budget BudgetCategory Rule].freeze
ACCOUNTABLE_TYPE_CLASSES = {
"Depository" => Depository, "Investment" => Investment, "Crypto" => Crypto,
@@ -12,9 +42,41 @@ class Family::DataImporter
ACCOUNTABLE_TYPE_CLASSES[type.to_s]
end
- def initialize(family, ndjson_content)
+ MAPPING_TYPES = {
+ accounts: "Account",
+ categories: "Category",
+ tags: "Tag",
+ merchants: "Merchant",
+ recurring_transactions: "RecurringTransaction",
+ transactions: "Transaction",
+ budgets: "Budget",
+ securities: "Security",
+ rules: "Rule"
+ }.freeze
+ SUMMARY_KEYS = {
+ "Account" => "accounts",
+ "Balance" => "balances",
+ "Category" => "categories",
+ "Tag" => "tags",
+ "Merchant" => "merchants",
+ "RecurringTransaction" => "recurring_transactions",
+ "Transaction" => "transactions",
+ "Transfer" => "transfers",
+ "RejectedTransfer" => "rejected_transfers",
+ "Trade" => "trades",
+ "Holding" => "holdings",
+ "Valuation" => "valuations",
+ "Budget" => "budgets",
+ "BudgetCategory" => "budget_categories",
+ "Rule" => "rules"
+ }.freeze
+
+ def initialize(family, ndjson_content, import_session: nil, import: nil)
@family = family
@ndjson_content = ndjson_content
+ @import_session = import_session
+ @import = import
+ @strict_references = import_session.present?
@id_mappings = {
accounts: {},
categories: {},
@@ -23,11 +85,13 @@ class Family::DataImporter
recurring_transactions: {},
transactions: {},
budgets: {},
- securities: {}
+ securities: {},
+ rules: {}
}
@security_cache = {}
@created_accounts = []
@created_entries = []
+ @summary = Hash.new { |hash, key| hash[key] = empty_summary_bucket }
end
def import!
@@ -54,7 +118,7 @@ class Family::DataImporter
import_rules(records["Rule"] || [])
end
- { accounts: @created_accounts, entries: @created_entries }
+ { accounts: @created_accounts, entries: @created_entries, summary: compact_summary }
end
private
@@ -79,6 +143,128 @@ class Family::DataImporter
records
end
+ def empty_summary_bucket
+ { "created" => 0, "updated" => 0, "skipped" => 0, "failed" => 0 }
+ end
+
+ def compact_summary
+ @summary.select { |_entity_type, counts| counts.values.any?(&:positive?) }
+ end
+
+ def increment_summary(record_type, status)
+ @summary[SUMMARY_KEYS.fetch(record_type)].tap do |counts|
+ counts[status.to_s] = counts.fetch(status.to_s, 0) + 1
+ end
+ end
+
+ def map_source!(mapping_key, source_id, target)
+ return if source_id.blank? || target.blank?
+
+ @id_mappings[mapping_key][source_id] = target.id
+ return unless @import_session
+
+ source_type = MAPPING_TYPES.fetch(mapping_key)
+ mapping = @import_session.source_mappings.find_or_initialize_by(
+ family: @family,
+ source_type: source_type,
+ source_id: source_id
+ )
+ mapping.target = target
+ mapping.save!
+ end
+
+ def mapped_id(mapping_key, old_id, record_type:, required: true)
+ if old_id.blank?
+ missing_reference(record_type, mapping_key, "(blank)") if required
+ return
+ end
+
+ return @id_mappings[mapping_key][old_id] if @id_mappings[mapping_key].key?(old_id)
+
+ source_type = MAPPING_TYPES.fetch(mapping_key)
+ mapping = @import_session&.source_mappings&.find_by(
+ family: @family,
+ source_type: source_type,
+ source_id: old_id
+ )
+
+ if mapping
+ @id_mappings[mapping_key][old_id] = mapping.target_id
+ return mapping.target_id
+ end
+
+ if required && @strict_references
+ raise MissingReferenceError.new(
+ record_type: record_type,
+ source_type: source_type,
+ source_id: old_id
+ )
+ end
+
+ nil
+ end
+
+ def mapped_record(mapping_key, old_id, scope, record_type:)
+ target_id = mapped_id(mapping_key, old_id, record_type: record_type, required: false)
+ return if target_id.blank?
+
+ scope.find_by(id: target_id)
+ end
+
+ def missing_reference(record_type, mapping_key, old_id)
+ if @strict_references
+ increment_summary(record_type, :failed)
+ raise MissingReferenceError.new(
+ record_type: record_type,
+ source_type: MAPPING_TYPES.fetch(mapping_key),
+ source_id: old_id
+ )
+ end
+
+ increment_summary(record_type, :skipped)
+ nil
+ end
+
+ def require_source_id!(record_type, source_id)
+ return if source_id.present? || !@strict_references
+
+ increment_summary(record_type, :failed)
+ raise MissingReferenceError.new(
+ record_type: record_type,
+ source_type: record_type,
+ source_id: "(blank)"
+ )
+ end
+
+ def invalid_record!(record_type, field, value)
+ if @strict_references
+ increment_summary(record_type, :failed)
+ raise InvalidRecordError.new(record_type: record_type, field: field, value: value)
+ end
+
+ increment_summary(record_type, :skipped)
+ nil
+ end
+
+ def session_entry_source
+ return unless @import_session
+
+ "sure_import_session:#{@import_session.id}"
+ end
+
+ def session_entry_external_id(record_type, source_id)
+ return if @import_session.blank? || source_id.blank?
+
+ "#{record_type}:#{source_id}"
+ end
+
+ def session_imported_entry(account, record_type, source_id)
+ external_id = session_entry_external_id(record_type, source_id)
+ return if external_id.blank?
+
+ account.entries.find_by(source: session_entry_source, external_id: external_id)
+ end
+
def import_accounts(records)
records.each do |record|
data = record["data"]
@@ -86,26 +272,41 @@ class Family::DataImporter
accountable_data = data["accountable"] || {}
accountable_type = data["accountable_type"]
+ require_source_id!("Account", old_id)
+
accountable_class = self.class.accountable_class_for(accountable_type)
- next unless accountable_class
- accountable = accountable_class.new
- accountable.subtype = accountable_data["subtype"] if accountable.respond_to?(:subtype=) && accountable_data["subtype"]
-
- # Copy any other accountable attributes
- safe_accountable_attrs = %w[subtype locked_attributes]
- safe_accountable_attrs.each do |attr|
- if accountable.respond_to?("#{attr}=") && accountable_data[attr].present?
- accountable.send("#{attr}=", accountable_data[attr])
- end
+ unless accountable_class
+ invalid_record!("Account", "accountable_type", accountable_type)
+ next
end
- account = @family.accounts.build(
+ account = mapped_record(:accounts, old_id, @family.accounts, record_type: "Account")
+ created = account.blank?
+
+ if account
+ accountable = account.accountable
+ else
+ # Build accountable
+ accountable = accountable_class.new
+ accountable.subtype = accountable_data["subtype"] if accountable.respond_to?(:subtype=) && accountable_data["subtype"]
+
+ # Copy any other accountable attributes
+ safe_accountable_attrs = %w[subtype locked_attributes]
+ safe_accountable_attrs.each do |attr|
+ if accountable.respond_to?("#{attr}=") && accountable_data[attr].present?
+ accountable.send("#{attr}=", accountable_data[attr])
+ end
+ end
+
+ account = @family.accounts.build(accountable: accountable)
+ end
+
+ account.assign_attributes(
name: data["name"],
balance: data["balance"].to_d,
cash_balance: data["cash_balance"]&.to_d || data["balance"].to_d,
currency: data["currency"] || @family.currency,
- accountable: accountable,
subtype: data["subtype"],
institution_name: data["institution_name"],
institution_domain: data["institution_domain"],
@@ -118,7 +319,7 @@ class Family::DataImporter
# Set opening balance if we have a historical balance and the import
# does not provide either an explicit opening-anchor valuation or an
# authoritative balance-history stream for this account.
- if data["balance"].present? && !skip_opening_balance_import?(old_id, data)
+ if created && data["balance"].present? && !skip_opening_balance_import?(old_id, data)
manager = Account::OpeningBalanceManager.new(account)
result = manager.set_opening_balance(
balance: data["balance"].to_d,
@@ -127,8 +328,9 @@ class Family::DataImporter
log_failed_opening_balance_import(account, old_id, result) unless result.success?
end
- @id_mappings[:accounts][old_id] = account.id
- @created_accounts << account
+ map_source!(:accounts, old_id, account)
+ @created_accounts << account if created
+ increment_summary("Account", created ? :created : :updated)
end
end
@@ -139,16 +341,23 @@ class Family::DataImporter
def import_balances(records)
records.each do |record|
data = record["data"] || {}
- new_account_id = @id_mappings[:accounts][data["account_id"]]
+ new_account_id = mapped_id(:accounts, data["account_id"], record_type: "Balance")
balance_date = parse_import_date(data["date"])
- next if new_account_id.blank? || balance_date.blank? || data["balance"].blank?
+ next if new_account_id.blank?
+
+ if balance_date.blank? || data["balance"].blank?
+ increment_summary("Balance", :skipped)
+ next
+ end
account = @family.accounts.find(new_account_id)
currency = data["currency"].presence || account.currency
balance = account.balances.find_or_initialize_by(date: balance_date, currency: currency)
+ created = balance.new_record?
balance.assign_attributes(imported_balance_attributes(data))
balance.save!
+ increment_summary("Balance", created ? :created : :updated)
end
end
@@ -188,24 +397,30 @@ class Family::DataImporter
old_id = data["id"]
parent_id = data["parent_id"]
+ require_source_id!("Category", old_id)
+
# Store parent relationship for second pass
parent_mappings[old_id] = parent_id if parent_id.present?
- category = @family.categories.build(
+ category = mapped_record(:categories, old_id, @family.categories, record_type: "Category")
+ created = category.blank?
+ category ||= @family.categories.build
+
+ category.assign_attributes(
name: data["name"],
color: data["color"] || Category::UNCATEGORIZED_COLOR,
classification_unused: data["classification_unused"] || data["classification"] || "expense",
lucide_icon: data["lucide_icon"] || "shapes"
)
category.save!
-
- @id_mappings[:categories][old_id] = category.id
+ map_source!(:categories, old_id, category)
+ increment_summary("Category", created ? :created : :updated)
end
# Second pass: establish parent relationships
parent_mappings.each do |old_id, old_parent_id|
- new_id = @id_mappings[:categories][old_id]
- new_parent_id = @id_mappings[:categories][old_parent_id]
+ new_id = mapped_id(:categories, old_id, record_type: "Category")
+ new_parent_id = mapped_id(:categories, old_parent_id, record_type: "Category")
next unless new_id && new_parent_id
@@ -219,13 +434,22 @@ class Family::DataImporter
data = record["data"]
old_id = data["id"]
- tag = @family.tags.build(
+ require_source_id!("Tag", old_id)
+
+ tag = mapped_record(:tags, old_id, @family.tags, record_type: "Tag")
+ created = tag.blank?
+ tag ||= @family.tags.build
+ color = data["color"] || tag.color
+ # Keep replayed session imports deterministic when the source omits a color.
+ color ||= Tag::COLORS.first if created
+
+ tag.assign_attributes(
name: data["name"],
- color: data["color"] || Tag::COLORS.sample
+ color: color
)
tag.save!
-
- @id_mappings[:tags][old_id] = tag.id
+ map_source!(:tags, old_id, tag)
+ increment_summary("Tag", created ? :created : :updated)
end
end
@@ -234,14 +458,20 @@ class Family::DataImporter
data = record["data"]
old_id = data["id"]
- merchant = @family.merchants.build(
+ require_source_id!("Merchant", old_id)
+
+ merchant = mapped_record(:merchants, old_id, @family.merchants, record_type: "Merchant")
+ created = merchant.blank?
+ merchant ||= @family.merchants.build
+
+ merchant.assign_attributes(
name: data["name"],
color: data["color"],
logo_url: data["logo_url"]
)
merchant.save!
-
- @id_mappings[:merchants][old_id] = merchant.id
+ map_source!(:merchants, old_id, merchant)
+ increment_summary("Merchant", created ? :created : :updated)
end
end
@@ -250,10 +480,20 @@ class Family::DataImporter
data = record["data"]
old_id = data["id"]
- new_account_id = remap_optional_id(:accounts, data["account_id"])
+ require_source_id!("RecurringTransaction", old_id)
+
+ recurring_transaction = mapped_record(
+ :recurring_transactions,
+ old_id,
+ @family.recurring_transactions,
+ record_type: "RecurringTransaction"
+ )
+ created = recurring_transaction.blank?
+
+ new_account_id = remap_optional_id(:accounts, data["account_id"], record_type: "RecurringTransaction")
next if data["account_id"].present? && new_account_id.blank?
- new_merchant_id = remap_optional_id(:merchants, data["merchant_id"])
+ new_merchant_id = remap_optional_id(:merchants, data["merchant_id"], record_type: "RecurringTransaction")
next if data["merchant_id"].present? && new_merchant_id.blank?
expected_day_of_month = recurring_expected_day_for(data["expected_day_of_month"])
@@ -262,7 +502,8 @@ class Family::DataImporter
next_expected_date = parse_import_date(data["next_expected_date"])
next unless last_occurrence_date && next_expected_date
- recurring_transaction = @family.recurring_transactions.build(
+ recurring_transaction ||= @family.recurring_transactions.build
+ recurring_transaction.assign_attributes(
account_id: new_account_id,
merchant_id: new_merchant_id,
amount: data["amount"].to_d,
@@ -280,14 +521,15 @@ class Family::DataImporter
)
recurring_transaction.save!
- @id_mappings[:recurring_transactions][old_id] = recurring_transaction.id
+ map_source!(:recurring_transactions, old_id, recurring_transaction)
+ increment_summary("RecurringTransaction", created ? :created : :updated)
end
end
- def remap_optional_id(mapping_key, old_id)
+ def remap_optional_id(mapping_key, old_id, record_type:)
return if old_id.blank?
- @id_mappings[mapping_key][old_id]
+ mapped_id(mapping_key, old_id, record_type: record_type)
end
def recurring_transaction_status_for(status)
@@ -312,8 +554,10 @@ class Family::DataImporter
data = record["data"]
old_id = data["id"]
+ require_source_id!("Transaction", old_id)
+
# Map account ID
- new_account_id = @id_mappings[:accounts][data["account_id"]]
+ new_account_id = mapped_id(:accounts, data["account_id"], record_type: "Transaction")
next unless new_account_id
account = @family.accounts.find(new_account_id)
@@ -321,55 +565,69 @@ class Family::DataImporter
# Map category ID (optional)
new_category_id = nil
if data["category_id"].present?
- new_category_id = @id_mappings[:categories][data["category_id"]]
+ new_category_id = mapped_id(:categories, data["category_id"], record_type: "Transaction")
end
# Map merchant ID (optional)
new_merchant_id = nil
if data["merchant_id"].present?
- new_merchant_id = @id_mappings[:merchants][data["merchant_id"]]
+ new_merchant_id = mapped_id(:merchants, data["merchant_id"], record_type: "Transaction")
end
# Map tag IDs (optional)
- new_tag_ids = mapped_tag_ids(data["tag_ids"])
+ new_tag_ids = mapped_tag_ids(data["tag_ids"], record_type: "Transaction")
- transaction = Transaction.new(
+ entry = session_imported_entry(account, "Transaction", old_id)
+ transaction = entry&.entryable if entry&.entryable.is_a?(Transaction)
+ created = transaction.blank?
+
+ transaction ||= Transaction.new
+ transaction.assign_attributes(
category_id: new_category_id,
merchant_id: new_merchant_id,
kind: data["kind"] || "standard"
)
- entry = Entry.new(
+ entry ||= Entry.new(entryable: transaction)
+ entry.assign_attributes(
account: account,
date: Date.parse(data["date"].to_s),
amount: data["amount"].to_d,
name: data["name"] || "Imported transaction",
currency: data["currency"] || account.currency,
notes: data["notes"],
- excluded: data["excluded"] || false,
- entryable: transaction
+ excluded: data["excluded"] || false
)
+ if @import_session
+ entry.external_id = session_entry_external_id("Transaction", old_id)
+ entry.source = session_entry_source
+ end
entry.save!
- @id_mappings[:transactions][old_id] = transaction.id
+ map_source!(:transactions, old_id, transaction)
split_rows = importable_split_rows(data)
if split_rows.any?
- @created_entries << entry
+ @created_entries << entry if created
import_split_lines!(entry, split_rows, fallback_tag_ids: new_tag_ids)
else
+ transaction.taggings.destroy_all unless created
new_tag_ids.each do |tag_id|
transaction.taggings.create!(tag_id: tag_id)
end
- @created_entries << entry
+ @created_entries << entry if created
end
+
+ increment_summary("Transaction", created ? :created : :updated)
end
end
- def mapped_tag_ids(old_tag_ids)
- Array(old_tag_ids).map { |old_tag_id| @id_mappings[:tags][old_tag_id] }.compact
+ def mapped_tag_ids(old_tag_ids, record_type:)
+ Array(old_tag_ids).map do |old_tag_id|
+ mapped_id(:tags, old_tag_id, record_type: record_type)
+ end.compact
end
def importable_split_rows(data)
@@ -380,8 +638,8 @@ class Family::DataImporter
amount = row["amount"] || row["amount_money"] || row["amount_decimal"]
next if amount.blank?
- category_id = remap_optional_id(:categories, row["category_id"])
- merchant_id = remap_optional_id(:merchants, row["merchant_id"])
+ category_id = remap_optional_id(:categories, row["category_id"], record_type: "Transaction")
+ merchant_id = remap_optional_id(:merchants, row["merchant_id"], record_type: "Transaction")
{
old_id: row["id"],
@@ -392,7 +650,7 @@ class Family::DataImporter
merchant_id_provided: row.key?("merchant_id"),
notes: row["notes"],
excluded: boolean_import_value(row, "excluded", default: false),
- tag_ids: mapped_tag_ids(row["tag_ids"]),
+ tag_ids: mapped_tag_ids(row["tag_ids"], record_type: "Transaction"),
tag_ids_provided: row.key?("tag_ids"),
kind: row["kind"]
}
@@ -424,7 +682,7 @@ class Family::DataImporter
transaction.taggings.create!(tag_id: tag_id)
end
- @id_mappings[:transactions][row[:old_id]] = transaction.id if row[:old_id].present?
+ map_source!(:transactions, row[:old_id], transaction) if row[:old_id].present?
@created_entries << child_entry
end
end
@@ -432,31 +690,60 @@ class Family::DataImporter
def import_transfers(records)
records.each do |record|
data = record["data"]
- inflow_transaction_id = @id_mappings[:transactions][data["inflow_transaction_id"]]
- outflow_transaction_id = @id_mappings[:transactions][data["outflow_transaction_id"]]
+ inflow_transaction_id = mapped_id(:transactions, data["inflow_transaction_id"], record_type: "Transfer")
+ outflow_transaction_id = mapped_id(:transactions, data["outflow_transaction_id"], record_type: "Transfer")
next unless inflow_transaction_id && outflow_transaction_id
- Transfer.find_or_create_by!(
+ transfer = Transfer.find_or_create_by!(
inflow_transaction_id: inflow_transaction_id,
outflow_transaction_id: outflow_transaction_id
) do |transfer|
transfer.status = transfer_status_for(data["status"])
transfer.notes = data["notes"]
end
+ apply_transfer_transaction_kinds!(transfer)
+ increment_summary("Transfer", transfer.previously_new_record? ? :created : :updated)
end
end
+ def apply_transfer_transaction_kinds!(transfer)
+ destination_account = transfer.inflow_transaction.entry.account
+ outflow_kind = imported_transfer_outflow_kind(transfer)
+ outflow_attrs = { kind: outflow_kind }
+ if outflow_kind == "investment_contribution" && transfer.outflow_transaction.category_id.blank?
+ outflow_attrs[:category] = destination_account.family.investment_contributions_category
+ end
+
+ transfer.outflow_transaction.update!(outflow_attrs)
+ transfer.inflow_transaction.update!(kind: "funds_movement")
+ end
+
+ def imported_transfer_outflow_kind(transfer)
+ source_account = transfer.outflow_transaction.entry.account
+ destination_account = transfer.inflow_transaction.entry.account
+ return "loan_payment" if destination_account.loan?
+ return "cc_payment" if destination_account.liability?
+ return "investment_contribution" if investment_account?(destination_account) && !investment_account?(source_account)
+
+ "funds_movement"
+ end
+
+ def investment_account?(account)
+ account.investment? || account.crypto?
+ end
+
def import_rejected_transfers(records)
records.each do |record|
data = record["data"]
- inflow_transaction_id = @id_mappings[:transactions][data["inflow_transaction_id"]]
- outflow_transaction_id = @id_mappings[:transactions][data["outflow_transaction_id"]]
+ inflow_transaction_id = mapped_id(:transactions, data["inflow_transaction_id"], record_type: "RejectedTransfer")
+ outflow_transaction_id = mapped_id(:transactions, data["outflow_transaction_id"], record_type: "RejectedTransfer")
next unless inflow_transaction_id && outflow_transaction_id
- RejectedTransfer.find_or_create_by!(
+ rejected_transfer = RejectedTransfer.find_or_create_by!(
inflow_transaction_id: inflow_transaction_id,
outflow_transaction_id: outflow_transaction_id
)
+ increment_summary("RejectedTransfer", rejected_transfer.previously_new_record? ? :created : :updated)
end
end
@@ -471,9 +758,12 @@ class Family::DataImporter
def import_trades(records)
records.each do |record|
data = record["data"]
+ old_id = data["id"]
+
+ require_source_id!("Trade", old_id)
# Map account ID
- new_account_id = @id_mappings[:accounts][data["account_id"]]
+ new_account_id = mapped_id(:accounts, data["account_id"], record_type: "Trade")
next unless new_account_id
account = @family.accounts.find(new_account_id)
@@ -490,34 +780,47 @@ class Family::DataImporter
exchange_operating_mic: data["exchange_operating_mic"]
)
- trade = Trade.new(
+ entry = session_imported_entry(account, "Trade", old_id)
+ trade = entry&.entryable if entry&.entryable.is_a?(Trade)
+ created = trade.blank?
+
+ trade ||= Trade.new
+ trade.assign_attributes(
security: security,
qty: data["qty"].to_d,
price: data["price"].to_d,
currency: data["currency"] || account.currency
)
- entry = Entry.new(
+ entry ||= Entry.new(entryable: trade)
+ entry.assign_attributes(
account: account,
date: Date.parse(data["date"].to_s),
amount: data["amount"].to_d,
name: "#{data["qty"].to_d >= 0 ? 'Buy' : 'Sell'} #{ticker}",
- currency: data["currency"] || account.currency,
- entryable: trade
+ currency: data["currency"] || account.currency
)
+ if @import_session
+ entry.external_id = session_entry_external_id("Trade", old_id)
+ entry.source = session_entry_source
+ end
entry.save!
- @created_entries << entry
+ @created_entries << entry if created
+ increment_summary("Trade", created ? :created : :updated)
end
end
def import_holdings(records)
- accounts_by_id = @family.accounts.where(id: records.filter_map { |record| @id_mappings[:accounts][record.dig("data", "account_id")] }).index_by(&:id)
+ account_ids = records.filter_map do |record|
+ mapped_id(:accounts, record.dig("data", "account_id"), record_type: "Holding", required: false)
+ end
+ accounts_by_id = @family.accounts.where(id: account_ids).index_by(&:id)
records.each do |record|
data = record["data"]
- new_account_id = @id_mappings[:accounts][data["account_id"]]
+ new_account_id = mapped_id(:accounts, data["account_id"], record_type: "Holding")
next unless new_account_id
account = accounts_by_id[new_account_id]
@@ -552,33 +855,46 @@ class Family::DataImporter
security_locked: truthy?(data["security_locked"]) || false
}
- upsert_imported_holding!(account, security, holding_date, holding_currency, holding_attributes)
+ created = upsert_imported_holding!(account, security, holding_date, holding_currency, holding_attributes)
+ increment_summary("Holding", created ? :created : :updated)
end
end
def import_valuations(records)
records.each do |record|
data = record["data"]
+ old_id = data["id"]
+
+ require_source_id!("Valuation", old_id)
# Map account ID
- new_account_id = @id_mappings[:accounts][data["account_id"]]
+ new_account_id = mapped_id(:accounts, data["account_id"], record_type: "Valuation")
next unless new_account_id
account = @family.accounts.find(new_account_id)
- valuation = Valuation.new(kind: valuation_kind_for(data["kind"]))
+ entry = session_imported_entry(account, "Valuation", old_id)
+ valuation = entry&.entryable if entry&.entryable.is_a?(Valuation)
+ created = valuation.blank?
+ valuation ||= Valuation.new
+ valuation.kind = valuation_kind_for(data["kind"])
- entry = Entry.new(
+ entry ||= Entry.new(entryable: valuation)
+ entry.assign_attributes(
account: account,
date: Date.parse(data["date"].to_s),
amount: data["amount"].to_d,
name: data["name"] || "Valuation",
- currency: data["currency"] || account.currency,
- entryable: valuation
+ currency: data["currency"] || account.currency
)
+ if @import_session
+ entry.external_id = session_entry_external_id("Valuation", old_id)
+ entry.source = session_entry_source
+ end
entry.save!
- @created_entries << entry
+ @created_entries << entry if created
+ increment_summary("Valuation", created ? :created : :updated)
end
end
@@ -650,7 +966,13 @@ class Family::DataImporter
data = record["data"]
old_id = data["id"]
- budget = @family.budgets.build(
+ require_source_id!("Budget", old_id)
+
+ budget = mapped_record(:budgets, old_id, @family.budgets, record_type: "Budget")
+ created = budget.blank?
+ budget ||= @family.budgets.build
+
+ budget.assign_attributes(
start_date: Date.parse(data["start_date"].to_s),
end_date: Date.parse(data["end_date"].to_s),
budgeted_spending: data["budgeted_spending"]&.to_d,
@@ -659,7 +981,8 @@ class Family::DataImporter
)
budget.save!
- @id_mappings[:budgets][old_id] = budget.id
+ map_source!(:budgets, old_id, budget)
+ increment_summary("Budget", created ? :created : :updated)
end
end
@@ -668,36 +991,49 @@ class Family::DataImporter
data = record["data"]
# Map budget ID
- new_budget_id = @id_mappings[:budgets][data["budget_id"]]
+ new_budget_id = mapped_id(:budgets, data["budget_id"], record_type: "BudgetCategory")
next unless new_budget_id
# Map category ID
- new_category_id = @id_mappings[:categories][data["category_id"]]
+ new_category_id = mapped_id(:categories, data["category_id"], record_type: "BudgetCategory")
next unless new_category_id
budget = @family.budgets.find(new_budget_id)
- budget_category = budget.budget_categories.build(
+ budget_category = budget.budget_categories.find_or_initialize_by(category_id: new_category_id)
+ created = budget_category.new_record?
+ budget_category.assign_attributes(
category_id: new_category_id,
budgeted_spending: data["budgeted_spending"].to_d,
currency: data["currency"] || budget.currency
)
budget_category.save!
+ increment_summary("BudgetCategory", created ? :created : :updated)
end
end
def import_rules(records)
records.each do |record|
data = record["data"]
+ old_id = data["id"]
- rule = @family.rules.build(
+ require_source_id!("Rule", old_id)
+
+ rule = mapped_record(:rules, old_id, @family.rules, record_type: "Rule")
+ created = rule.blank?
+ rule ||= @family.rules.build
+
+ rule.assign_attributes(
name: data["name"],
resource_type: data["resource_type"] || "transaction",
active: data["active"] || false,
effective_date: data["effective_date"].present? ? Date.parse(data["effective_date"].to_s) : nil
)
+ rule.conditions.destroy_all unless created
+ rule.actions.destroy_all unless created
+
# Build conditions
(data["conditions"] || []).each do |condition_data|
build_rule_condition(rule, condition_data)
@@ -709,6 +1045,8 @@ class Family::DataImporter
end
rule.save!
+ map_source!(:rules, old_id, rule)
+ increment_summary("Rule", created ? :created : :updated)
end
end
@@ -845,8 +1183,9 @@ class Family::DataImporter
return security
end
- if old_security_id.present? && @id_mappings[:securities][old_security_id]
- security = Security.find(@id_mappings[:securities][old_security_id])
+ mapped_security_id = mapped_id(:securities, old_security_id, record_type: "Security", required: false)
+ if old_security_id.present? && mapped_security_id
+ security = Security.find(mapped_security_id)
apply_security_metadata(security, normalized_ticker, attributes)
@security_cache[cache_key] = security
return security
@@ -856,7 +1195,7 @@ class Family::DataImporter
apply_security_metadata(security, normalized_ticker, attributes)
@security_cache[cache_key] = security
- @id_mappings[:securities][old_security_id] = security.id if old_security_id.present?
+ map_source!(:securities, old_security_id, security) if old_security_id.present?
security
end
@@ -901,6 +1240,7 @@ class Family::DataImporter
def upsert_imported_holding!(account, security, date, currency, attributes)
holding = account.holdings.find_or_initialize_by(security: security, date: date, currency: currency)
+ created = holding.new_record?
holding.assign_attributes(attributes)
begin
@@ -908,7 +1248,10 @@ class Family::DataImporter
rescue ActiveRecord::RecordNotUnique
existing = account.holdings.find_by!(security: security, date: date, currency: currency)
existing.update!(attributes)
+ created = false
end
+
+ created
end
def security_kind_for(value)
diff --git a/app/models/family/financial_data_reset.rb b/app/models/family/financial_data_reset.rb
index 91759e239..021f1eab9 100644
--- a/app/models/family/financial_data_reset.rb
+++ b/app/models/family/financial_data_reset.rb
@@ -7,6 +7,8 @@ class Family::FinancialDataReset
account_statements
family_exports
imports
+ import_sessions
+ import_source_mappings
import_rows
import_mappings
accounts
@@ -127,6 +129,7 @@ class Family::FinancialDataReset
delete_active_storage_attachments!
scope(:transfers).destroy_all
scope(:rejected_transfers).destroy_all
+ scope(:import_source_mappings).destroy_all
scope(:import_mappings).destroy_all
scope(:import_rows).destroy_all
scope(:rule_runs).destroy_all
@@ -138,6 +141,7 @@ class Family::FinancialDataReset
scope(:account_statements).destroy_all
scope(:family_exports).destroy_all
scope(:imports).destroy_all
+ scope(:import_sessions).destroy_all
scope(:entries).destroy_all
scope(:holdings).destroy_all
scope(:balances).destroy_all
@@ -239,6 +243,7 @@ class Family::FinancialDataReset
account_scope = Account.where(family_id: family.id)
account_ids = account_scope.select(:id)
import_scope = Import.where(family_id: family.id)
+ import_session_scope = ImportSession.where(family_id: family.id)
import_ids = import_scope.select(:id)
rule_scope = Rule.where(family_id: family.id)
rule_ids = rule_scope.select(:id)
@@ -252,6 +257,8 @@ class Family::FinancialDataReset
account_statements: AccountStatement.where(family_id: family.id),
family_exports: FamilyExport.where(family_id: family.id),
imports: import_scope,
+ import_sessions: import_session_scope,
+ import_source_mappings: ImportSourceMapping.where(family_id: family.id),
import_rows: Import::Row.where(import_id: import_ids),
import_mappings: Import::Mapping.where(import_id: import_ids),
accounts: account_scope,
diff --git a/app/models/import.rb b/app/models/import.rb
index d70a19eb2..4ea45f684 100644
--- a/app/models/import.rb
+++ b/app/models/import.rb
@@ -41,11 +41,14 @@ class Import < ApplicationRecord
belongs_to :family
belongs_to :account, optional: true
belongs_to :account_statement, optional: true
+ belongs_to :import_session, optional: true
before_validation :set_default_number_format
before_validation :ensure_utf8_encoding
+ normalizes :client_chunk_id, with: ->(value) { value.strip.presence }
scope :ordered, -> { order(created_at: :desc) }
+ scope :ordered_by_sequence, -> { order(:sequence, :created_at) }
enum :status, {
pending: "pending",
@@ -61,9 +64,15 @@ class Import < ApplicationRecord
validates :col_sep, inclusion: { in: SEPARATORS.map(&:last) }
validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }, allow_nil: true
validates :number_format, presence: true, inclusion: { in: NUMBER_FORMATS.keys }
+ validates :sequence, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true
+ validates :client_chunk_id, length: { maximum: 255 }, allow_blank: true
+ validates :checksum, length: { is: 64 }, allow_blank: true
validate :custom_column_import_requires_identifier
validates :rows_to_skip, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validate :account_belongs_to_family
+ validate :import_session_belongs_to_family
+ validate :session_chunk_metadata
+ validate :session_payloads_are_json_objects
validate :rows_to_skip_within_file_bounds
has_many :rows, dependent: :destroy
@@ -564,6 +573,25 @@ class Import < ApplicationRecord
errors.add(:account, "must belong to your family")
end
+ def import_session_belongs_to_family
+ return if import_session.nil?
+ return if import_session.family_id == family_id
+
+ errors.add(:import_session, "must belong to your family")
+ end
+
+ def session_chunk_metadata
+ return if import_session.nil?
+
+ errors.add(:sequence, "must be present for import session chunks") if sequence.blank?
+ errors.add(:checksum, "must be present for import session chunks") if checksum.blank?
+ end
+
+ def session_payloads_are_json_objects
+ errors.add(:summary, "must be an object") unless summary.is_a?(Hash)
+ errors.add(:error_details, "must be an object") unless error_details.is_a?(Hash)
+ end
+
def rows_to_skip_within_file_bounds
return if raw_file_str.blank?
return if rows_to_skip.to_i == 0
diff --git a/app/models/import_session.rb b/app/models/import_session.rb
new file mode 100644
index 000000000..8ce8c1fd9
--- /dev/null
+++ b/app/models/import_session.rb
@@ -0,0 +1,425 @@
+require "digest"
+
+class ImportSession < ApplicationRecord
+ ConflictError = Class.new(StandardError)
+ EnqueueError = Class.new(StandardError)
+
+ IMPORT_TYPES = %w[SureImport].freeze
+ STATUSES = %w[pending importing complete failed].freeze
+
+ belongs_to :family
+ has_many :imports, -> { order(:sequence, :created_at) }, dependent: :destroy
+ has_many :source_mappings,
+ class_name: "ImportSourceMapping",
+ dependent: :destroy
+
+ enum :status, {
+ pending: "pending",
+ importing: "importing",
+ complete: "complete",
+ failed: "failed"
+ }, validate: true, default: "pending"
+
+ validates :import_type, inclusion: { in: IMPORT_TYPES }
+ validates :client_session_id, uniqueness: { scope: :family_id }, allow_blank: true
+ validates :client_session_id, length: { maximum: 255 }, allow_blank: true
+ normalizes :client_session_id, with: ->(value) { value.strip.presence }
+ validates :expected_chunks,
+ numericality: { only_integer: true, greater_than: 0 },
+ allow_nil: true
+ validate :payloads_are_json_objects
+
+ def self.create_or_find_for!(family:, import_type:, client_session_id:, expected_chunks:)
+ import_type = import_type.presence || "SureImport"
+ expected_chunks = normalize_positive_integer(expected_chunks)
+ unless IMPORT_TYPES.include?(import_type)
+ session = new(import_type: import_type)
+ session.errors.add(:import_type, "must be SureImport")
+ raise ActiveRecord::RecordInvalid.new(session)
+ end
+
+ if client_session_id.present?
+ session = family.import_sessions.find_or_initialize_by(client_session_id: client_session_id)
+ if session.persisted? &&
+ expected_chunks.present? &&
+ session.expected_chunks.present? &&
+ session.expected_chunks != expected_chunks
+ raise ConflictError, "client_session_id already exists with a different expected_chunks value"
+ end
+ else
+ session = family.import_sessions.build
+ end
+
+ session.import_type = import_type
+ session.expected_chunks ||= expected_chunks
+ session.save!
+ session
+ rescue ActiveRecord::RecordNotUnique
+ raise unless client_session_id.present?
+
+ existing = family.import_sessions.find_by(client_session_id: client_session_id)
+ raise unless existing
+
+ if expected_chunks.present? &&
+ existing.expected_chunks.present? &&
+ existing.expected_chunks != expected_chunks
+ raise ConflictError, "client_session_id already exists with a different expected_chunks value"
+ end
+ if expected_chunks.present? && existing.expected_chunks.nil?
+ existing.update!(expected_chunks: expected_chunks)
+ end
+
+ existing
+ end
+
+ def self.normalize_positive_integer(value)
+ return if value.blank?
+
+ Integer(value, exception: false) || 0
+ end
+ private_class_method :normalize_positive_integer
+
+ def attach_chunk!(sequence:, content:, filename:, content_type:, client_chunk_id: nil)
+ sequence = self.class.send(:normalize_positive_integer, sequence)
+ raise ConflictError, "sequence must be a positive integer" unless sequence.positive?
+ raise ConflictError, "sequence exceeds expected_chunks" if expected_chunks.present? && sequence > expected_chunks
+
+ checksum = Digest::SHA256.hexdigest(content)
+ normalized_client_chunk_id = client_chunk_id.presence
+ chunk_needs_finalization = false
+
+ chunk = with_lock do
+ raise ConflictError, "cannot add chunks after publishing starts" unless pending? || failed?
+
+ existing = existing_chunk_for!(
+ sequence: sequence,
+ client_chunk_id: normalized_client_chunk_id,
+ checksum: checksum
+ )
+
+ if existing
+ chunk_needs_finalization = prepare_existing_chunk_for_retry!(
+ existing,
+ checksum: checksum,
+ content: content,
+ filename: filename,
+ content_type: content_type
+ )
+ existing
+ else
+ chunk_needs_finalization = true
+ chunk = create_chunk!(
+ sequence: sequence,
+ client_chunk_id: normalized_client_chunk_id,
+ checksum: checksum,
+ content: content,
+ filename: filename,
+ content_type: content_type
+ )
+ end
+ end
+
+ finalize_chunk_for_retry!(chunk, checksum) if chunk_needs_finalization
+ chunk
+ rescue ActiveRecord::RecordNotUnique
+ imports.reset
+ existing = existing_chunk_for!(
+ sequence: sequence,
+ client_chunk_id: normalized_client_chunk_id,
+ checksum: checksum
+ )
+ return prepare_and_finalize_existing_chunk!(
+ existing,
+ checksum: checksum,
+ content: content,
+ filename: filename,
+ content_type: content_type
+ ) if existing
+
+ raise ConflictError, "chunk already exists with different content"
+ end
+
+ def create_chunk!(sequence:, client_chunk_id:, checksum:, content:, filename:, content_type:)
+ imports.create!(
+ family: family,
+ type: "SureImport",
+ sequence: sequence,
+ client_chunk_id: client_chunk_id,
+ checksum: checksum
+ ).tap do |import|
+ import.ndjson_file.attach(
+ io: StringIO.new(content),
+ filename: filename,
+ content_type: content_type
+ )
+ end
+ end
+ private :create_chunk!
+
+ def publish_later
+ previous_status = nil
+ should_enqueue = false
+
+ sync_chunk_row_counts!
+
+ with_lock do
+ return if complete? || importing?
+
+ validate_publishable_chunks!
+
+ previous_status = status
+ update!(status: :importing, error_details: {})
+ should_enqueue = true
+ end
+
+ return unless should_enqueue
+
+ begin
+ ImportSessionJob.perform_later(self)
+ rescue => error
+ with_lock do
+ reload
+ if importing?
+ update!(status: previous_status, error_details: enqueue_error_details)
+ end
+ end
+ Rails.logger.error("ImportSession enqueue failed import_session_id=#{id} exception=#{error.class}")
+ raise EnqueueError, "Import session could not be queued."
+ end
+ end
+
+ def publish
+ return unless prepare_for_publish!
+
+ Rails.logger.info("ImportSession publish started import_session_id=#{id}")
+
+ imports.ordered_by_sequence.each do |import|
+ process_chunk!(import)
+ end
+
+ update!(status: :complete, summary: aggregate_chunk_summaries, error_details: {})
+ enqueue_family_sync
+ Rails.logger.info("ImportSession publish completed import_session_id=#{id}")
+ rescue => error
+ update!(
+ status: :failed,
+ error_details: error_details_for(error),
+ summary: aggregate_chunk_summaries
+ )
+ Rails.logger.error("ImportSession publish failed import_session_id=#{id} exception=#{error.class}")
+ end
+
+ def aggregate_chunk_summaries
+ imports.reload.each_with_object({}) do |import, totals|
+ merge_summary!(totals, import.summary || {})
+ end
+ end
+
+ private
+ def prepare_for_publish!
+ sync_chunk_row_counts!
+
+ with_lock do
+ return false if complete?
+
+ validate_publishable_chunks!
+
+ update!(status: :importing, error_details: {}) unless importing?
+ true
+ end
+ end
+
+ def enqueue_family_sync
+ family.sync_later
+ rescue => error
+ update!(error_details: sync_enqueue_error_details)
+ Rails.logger.error(
+ "ImportSession family sync enqueue failed import_session_id=#{id} exception=#{error.class}"
+ )
+ end
+
+ def existing_chunk_for!(sequence:, client_chunk_id:, checksum:)
+ sequence_match = imports.find_by(sequence: sequence)
+ client_chunk_match = imports.find_by(client_chunk_id: client_chunk_id) if client_chunk_id.present?
+
+ if sequence_match && client_chunk_match && sequence_match.id != client_chunk_match.id
+ raise ConflictError, "sequence and client_chunk_id refer to different chunks"
+ end
+
+ existing = sequence_match || client_chunk_match
+ return unless existing
+
+ if existing.sequence != sequence
+ raise ConflictError, "client_chunk_id already exists with a different sequence"
+ end
+
+ if client_chunk_id.present? && existing.client_chunk_id.present? && existing.client_chunk_id != client_chunk_id
+ raise ConflictError, "sequence already exists with a different client_chunk_id"
+ end
+
+ raise ConflictError, "chunk already exists with different content" unless existing.checksum == checksum
+
+ existing
+ end
+
+ def prepare_and_finalize_existing_chunk!(chunk, checksum:, content:, filename:, content_type:)
+ needs_finalization = with_lock do
+ prepare_existing_chunk_for_retry!(
+ chunk.reload,
+ checksum: checksum,
+ content: content,
+ filename: filename,
+ content_type: content_type
+ )
+ end
+
+ finalize_chunk_for_retry!(chunk, checksum) if needs_finalization
+ chunk
+ end
+
+ def prepare_existing_chunk_for_retry!(chunk, checksum:, content:, filename:, content_type:)
+ return false if chunk_ready_for_retry?(chunk, checksum)
+ return true if chunk.ndjson_file.attached? && chunk_content_checksum(chunk) == checksum
+
+ chunk.ndjson_file.attach(
+ io: StringIO.new(content),
+ filename: filename,
+ content_type: content_type
+ )
+ true
+ end
+
+ def finalize_chunk_for_retry!(chunk, checksum)
+ chunk.sync_ndjson_rows_count!
+ chunk.reload
+ return chunk if chunk_ready_for_retry?(chunk, checksum)
+
+ raise ConflictError, "chunk already exists but is incomplete"
+ rescue ActiveStorage::FileNotFoundError
+ raise ConflictError, "chunk already exists but is incomplete"
+ end
+
+ def chunk_ready_for_retry?(chunk, checksum)
+ chunk.ndjson_file.attached? &&
+ chunk.rows_count.to_i.positive? &&
+ chunk_content_checksum(chunk) == checksum
+ end
+
+ def chunk_content_checksum(chunk)
+ Digest::SHA256.hexdigest(chunk.ndjson_file.download)
+ rescue ActiveStorage::FileNotFoundError
+ nil
+ end
+
+ def process_chunk!(import)
+ return if import.complete?
+
+ import.update!(status: :importing, error: nil, error_details: {})
+ result = import.import!(import_session: self)
+ import.update!(status: :complete, summary: result.fetch(:summary, {}), error_details: {})
+ rescue => error
+ import.update!(
+ status: :failed,
+ error: public_error_message_for(error),
+ error_details: error_details_for(error),
+ summary: failed_summary_for(error)
+ )
+ raise
+ end
+
+ def row_count_exceeded?
+ imports.sum(:rows_count) > SureImport.max_row_count
+ end
+
+ def validate_publishable_chunks!
+ raise ConflictError, "import session has no chunks" unless imports.exists?
+ raise Import::MaxRowCountExceededError if row_count_exceeded?
+ validate_expected_chunk_sequences!
+ end
+
+ def sync_chunk_row_counts!
+ raise ConflictError, "import session has no chunks" unless imports.exists?
+ imports.reload.each(&:sync_ndjson_rows_count!)
+ rescue ActiveStorage::FileNotFoundError
+ raise ConflictError, "import session chunks are incomplete"
+ end
+
+ def validate_expected_chunk_sequences!
+ return if expected_chunks.blank?
+
+ expected_sequences = (1..expected_chunks).to_a
+ actual_sequences = imports.pluck(:sequence).sort
+ return if actual_sequences == expected_sequences
+
+ missing_sequences = expected_sequences - actual_sequences
+ unexpected_sequences = actual_sequences - expected_sequences
+ details = []
+ details << "missing sequences: #{missing_sequences.join(', ')}" if missing_sequences.any?
+ details << "unexpected sequences: #{unexpected_sequences.join(', ')}" if unexpected_sequences.any?
+
+ raise ConflictError, "import session chunks do not match expected sequences (#{details.join('; ')})"
+ end
+
+ def error_details_for(error)
+ details = {
+ "code" => error.respond_to?(:code) ? error.code : "import_failed",
+ "message" => public_error_message_for(error)
+ }
+
+ if error.respond_to?(:details)
+ details.merge!(error.details.stringify_keys)
+ end
+
+ details
+ end
+
+ def public_error_message_for(error)
+ return error.message if error.respond_to?(:code)
+
+ "Import session failed."
+ end
+
+ def enqueue_error_details
+ {
+ "code" => "import_enqueue_failed",
+ "message" => "Import session could not be queued."
+ }
+ end
+
+ def sync_enqueue_error_details
+ {
+ "code" => "family_sync_enqueue_failed",
+ "message" => "Family sync could not be queued after import completion."
+ }
+ end
+
+ def merge_summary!(totals, summary)
+ summary.each do |entity_type, counts|
+ next unless counts.respond_to?(:each)
+
+ totals[entity_type] ||= {}
+ counts.each do |status, count|
+ totals[entity_type][status] = totals[entity_type].fetch(status, 0) + count.to_i
+ end
+ end
+ end
+
+ def failed_summary_for(error)
+ record_type = error_details_for(error)["record_type"]
+ return {} if record_type.blank?
+
+ {
+ record_type.to_s.underscore.pluralize => {
+ "created" => 0,
+ "updated" => 0,
+ "skipped" => 0,
+ "failed" => 1
+ }
+ }
+ end
+
+ def payloads_are_json_objects
+ errors.add(:summary, "must be an object") unless summary.is_a?(Hash)
+ errors.add(:error_details, "must be an object") unless error_details.is_a?(Hash)
+ end
+end
diff --git a/app/models/import_source_mapping.rb b/app/models/import_source_mapping.rb
new file mode 100644
index 000000000..c59bd8a81
--- /dev/null
+++ b/app/models/import_source_mapping.rb
@@ -0,0 +1,41 @@
+class ImportSourceMapping < ApplicationRecord
+ SOURCE_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Budget Security Rule].freeze
+
+ belongs_to :family
+ belongs_to :import_session
+ belongs_to :target, polymorphic: true, optional: true
+
+ validates :source_type, :source_id, :target_type, :target_id, presence: true
+ validates :source_type, inclusion: { in: SOURCE_TYPES }
+ validates :target_type, inclusion: { in: SOURCE_TYPES }, allow_blank: true
+ validates :source_type, length: { maximum: 64 }
+ validates :source_id, length: { maximum: 255 }
+ validates :source_id, uniqueness: { scope: [ :import_session_id, :source_type ] }
+ normalizes :source_type, :source_id, with: ->(value) { value.strip.presence }
+ validate :family_matches_import_session
+ validate :target_exists
+ validate :target_matches_family
+
+ private
+ def family_matches_import_session
+ return if import_session.blank? || family_id == import_session.family_id
+
+ errors.add(:family, "must match import session")
+ end
+
+ def target_exists
+ return if target_type.blank? || target_id.blank? || !SOURCE_TYPES.include?(target_type)
+ return if target.present?
+
+ errors.add(:target, "must exist")
+ end
+
+ def target_matches_family
+ return if target_type.blank? || !SOURCE_TYPES.include?(target_type)
+ return if target.blank?
+ return unless target.respond_to?(:family_id)
+ return if target.family_id == family_id
+
+ errors.add(:target, "must belong to your family")
+ end
+end
diff --git a/app/models/pdf_import.rb b/app/models/pdf_import.rb
index 1e1f494ef..69c74cb9e 100644
--- a/app/models/pdf_import.rb
+++ b/app/models/pdf_import.rb
@@ -89,6 +89,8 @@ class PdfImport < Import
end
def process_with_ai
+ # TODO(#2113): hardcoded to OpenAI. Provider::Anthropic implements
+ # process_pdf (PR #1985); this should honor Setting.llm_provider.
provider = Provider::Registry.get_provider(:openai)
raise "AI provider not configured" unless provider
raise "AI provider does not support PDF processing" unless provider.supports_pdf_processing?
@@ -115,6 +117,8 @@ class PdfImport < Import
def extract_transactions
return unless statement_with_transactions?
+ # TODO(#2113): hardcoded to OpenAI. Provider::Anthropic implements
+ # extract_bank_statement (PR #1985); this should honor Setting.llm_provider.
provider = Provider::Registry.get_provider(:openai)
raise "AI provider not configured" unless provider
diff --git a/app/models/period.rb b/app/models/period.rb
index b4857daee..b5a3c80b6 100644
--- a/app/models/period.rb
+++ b/app/models/period.rb
@@ -124,7 +124,10 @@ class Period
def current_month_for(family)
return from_key("current_month") unless family&.uses_custom_month_start?
- family.current_custom_month_period
+ # Keep the semantic key so callers (e.g. the period picker) can identify
+ # this as "current month" even though the date range is custom.
+ custom_period = family.current_custom_month_period
+ new(key: "current_month", start_date: custom_period.start_date, end_date: custom_period.end_date)
end
def last_month_for(family)
@@ -134,7 +137,7 @@ class Period
last_month_date = current_start - 1.day
start_date = family.custom_month_start_for(last_month_date)
end_date = family.custom_month_end_for(last_month_date)
- custom(start_date: start_date, end_date: end_date)
+ new(key: "last_month", start_date: start_date, end_date: end_date)
end
end
diff --git a/app/models/provider/anthropic.rb b/app/models/provider/anthropic.rb
index 8e2e1fa50..ab05a0e0b 100644
--- a/app/models/provider/anthropic.rb
+++ b/app/models/provider/anthropic.rb
@@ -155,11 +155,50 @@ class Provider::Anthropic < Provider
end
def process_pdf(pdf_content:, model: "", family: nil)
- raise Error, "process_pdf not yet implemented for Provider::Anthropic"
+ with_provider_response do
+ effective_model = model.presence || @default_model
+ raise Error, "Model does not support PDF processing: #{effective_model}" unless supports_pdf_processing?(model: effective_model)
+
+ trace = create_langfuse_trace(
+ name: "anthropic.process_pdf",
+ input: { pdf_size: pdf_content&.bytesize }
+ )
+
+ result = PdfProcessor.new(
+ client,
+ model: effective_model,
+ pdf_content: pdf_content,
+ langfuse_trace: trace,
+ family: family
+ ).process
+
+ upsert_langfuse_trace(trace: trace, output: result.to_h)
+
+ result
+ end
end
def extract_bank_statement(pdf_content:, model: "", family: nil)
- raise Error, "extract_bank_statement not yet implemented for Provider::Anthropic"
+ with_provider_response do
+ effective_model = model.presence || @default_model
+
+ trace = create_langfuse_trace(
+ name: "anthropic.extract_bank_statement",
+ input: { pdf_size: pdf_content&.bytesize }
+ )
+
+ result = BankStatementExtractor.new(
+ client: client,
+ pdf_content: pdf_content,
+ model: effective_model,
+ langfuse_trace: trace,
+ family: family
+ ).extract
+
+ upsert_langfuse_trace(trace: trace, output: { transaction_count: result[:transactions].size })
+
+ result
+ end
end
def chat_response(
diff --git a/app/models/provider/anthropic/bank_statement_extractor.rb b/app/models/provider/anthropic/bank_statement_extractor.rb
new file mode 100644
index 000000000..e91d44b47
--- /dev/null
+++ b/app/models/provider/anthropic/bank_statement_extractor.rb
@@ -0,0 +1,229 @@
+class Provider::Anthropic::BankStatementExtractor
+ include Provider::Anthropic::Concerns::UsageRecorder
+
+ TOOL_NAME = "report_bank_statement".freeze
+
+ # Mirrors Provider::Anthropic::PdfProcessor::MAX_PDF_BYTES.
+ MAX_PDF_BYTES = 32 * 1024 * 1024
+
+ attr_reader :client, :model, :pdf_content, :langfuse_trace, :family
+
+ def initialize(client:, model:, pdf_content:, langfuse_trace: nil, family: nil)
+ @client = client
+ @model = model
+ @pdf_content = pdf_content
+ @langfuse_trace = langfuse_trace
+ @family = family
+ end
+
+ def extract
+ raise Provider::Anthropic::Error, "PDF content is required" if pdf_content.blank?
+ if pdf_content.bytesize > MAX_PDF_BYTES
+ raise Provider::Anthropic::Error,
+ "PDF exceeds Anthropic's 32 MB limit (#{pdf_content.bytesize} bytes)"
+ end
+
+ span = langfuse_trace&.span(name: "extract_bank_statement_api_call", input: {
+ model: model,
+ pdf_size: pdf_content.bytesize
+ })
+
+ response = client.messages.create(
+ model: model,
+ max_tokens: max_tokens,
+ system_: instructions,
+ messages: [ { role: "user", content: user_content } ],
+ tools: [ output_tool ],
+ tool_choice: { type: "tool", name: TOOL_NAME, disable_parallel_tool_use: true }
+ )
+
+ parsed = extract_tool_input(response)
+ result = build_result(parsed)
+
+ truncated = stop_reason(response) == :max_tokens
+ if truncated
+ Rails.logger.warn(
+ "[BankStatementExtractor] response truncated by max_tokens — extracted #{result[:transactions].size} " \
+ "transactions but more may be present in the statement. Raise ANTHROPIC_MAX_TOKENS or chunk the PDF."
+ )
+ result[:truncated] = true
+ end
+
+ record_usage(model, response.usage, operation: "extract_bank_statement", metadata: {
+ pdf_size: pdf_content.bytesize,
+ transaction_count: result[:transactions].size,
+ truncated: truncated
+ })
+
+ span&.end(output: { transaction_count: result[:transactions].size }, usage: usage_hash(response.usage))
+ result
+ rescue => e
+ span&.end(output: { error: e.message }, level: "ERROR")
+ record_usage_error(model, operation: "extract_bank_statement", error: e, metadata: { pdf_size: pdf_content&.bytesize })
+ raise
+ end
+
+ private
+ def max_tokens
+ ENV.fetch("ANTHROPIC_MAX_TOKENS", 4096).to_i
+ end
+
+ def user_content
+ [
+ {
+ type: "document",
+ source: {
+ type: "base64",
+ media_type: "application/pdf",
+ data: Base64.strict_encode64(pdf_content)
+ }
+ },
+ {
+ type: "text",
+ text: "Extract every transaction from this bank statement and return them via the report_bank_statement tool."
+ }
+ ]
+ end
+
+ def output_tool
+ {
+ name: TOOL_NAME,
+ description: "Return the full set of transactions and statement metadata extracted from the PDF.",
+ input_schema: {
+ type: "object",
+ properties: {
+ bank_name: { type: [ "string", "null" ] },
+ account_holder: { type: [ "string", "null" ] },
+ account_number: { type: [ "string", "null" ], description: "Typically last 4 digits only." },
+ statement_period: {
+ type: "object",
+ properties: {
+ start_date: { type: [ "string", "null" ], description: "YYYY-MM-DD" },
+ end_date: { type: [ "string", "null" ], description: "YYYY-MM-DD" }
+ },
+ required: [],
+ additionalProperties: false
+ },
+ opening_balance: { type: [ "number", "null" ] },
+ closing_balance: { type: [ "number", "null" ] },
+ transactions: {
+ type: "array",
+ description: "Every transaction in the statement, in document order.",
+ items: {
+ type: "object",
+ properties: {
+ date: { type: "string", description: "YYYY-MM-DD" },
+ description: { type: "string" },
+ amount: { type: "number", description: "Negative for debits / expenses, positive for credits / deposits." },
+ reference: { type: [ "string", "null" ] },
+ category: { type: [ "string", "null" ] }
+ },
+ required: [ "date", "description", "amount" ],
+ additionalProperties: false
+ }
+ }
+ },
+ required: [ "transactions" ],
+ additionalProperties: false
+ }
+ }
+ end
+
+ def instructions
+ <<~INSTRUCTIONS
+ Extract bank statement data from the attached PDF and return the result via the report_bank_statement tool.
+
+ Rules:
+ - Extract EVERY transaction in document order
+ - Negative amounts for debits / expenses, positive for credits / deposits
+ - Dates in YYYY-MM-DD
+ - Use null for any field you cannot read; do not invent values
+ INSTRUCTIONS
+ end
+
+ def stop_reason(response)
+ raw = response.respond_to?(:stop_reason) ? response.stop_reason : nil
+ raw.to_s.to_sym if raw
+ end
+
+ def extract_tool_input(response)
+ tool_use = Array(response.content).find { |block| block_type(block) == :tool_use }
+ raise Provider::Anthropic::Error, "Model did not invoke #{TOOL_NAME}" unless tool_use
+
+ input = block_input(tool_use)
+ input = JSON.parse(input) if input.is_a?(String)
+ input
+ end
+
+ def build_result(parsed)
+ # Intentionally NOT deduplicated, unlike Provider::Openai's extractor. That
+ # one chunks the PDF text with overlap and must drop transactions repeated
+ # across adjacent chunks. We send the whole PDF as a single native document
+ # block — no chunk artifacts — so deduping here would wrongly merge
+ # legitimate same-day, same-amount rows (e.g. two identical purchases).
+ # Preserve every transaction the model returns.
+ transactions = Array(parsed["transactions"] || parsed[:transactions]).map { |t| normalize_transaction(t) }.compact
+
+ {
+ transactions: transactions,
+ period: {
+ start_date: dig_period(parsed, :start_date),
+ end_date: dig_period(parsed, :end_date)
+ },
+ account_holder: parsed["account_holder"] || parsed[:account_holder],
+ account_number: parsed["account_number"] || parsed[:account_number],
+ bank_name: parsed["bank_name"] || parsed[:bank_name],
+ opening_balance: parsed["opening_balance"] || parsed[:opening_balance],
+ closing_balance: parsed["closing_balance"] || parsed[:closing_balance]
+ }
+ end
+
+ def dig_period(parsed, key)
+ period = parsed["statement_period"] || parsed[:statement_period]
+ return nil unless period.is_a?(Hash)
+ period[key.to_s] || period[key]
+ end
+
+ def normalize_transaction(txn)
+ return nil unless txn.is_a?(Hash)
+
+ {
+ date: parse_date(txn["date"] || txn[:date]),
+ amount: parse_amount(txn["amount"] || txn[:amount]),
+ name: txn["description"] || txn[:description] || txn["name"] || txn[:name],
+ category: txn["category"] || txn[:category],
+ notes: txn["reference"] || txn[:reference]
+ }
+ end
+
+ def parse_date(date_str)
+ return nil if date_str.blank?
+ Date.parse(date_str.to_s).strftime("%Y-%m-%d")
+ rescue ArgumentError
+ nil
+ end
+
+ def parse_amount(amount)
+ return nil if amount.nil?
+ return amount.to_f if amount.is_a?(Numeric)
+ amount.to_s.gsub(/[^0-9.\-]/, "").to_f
+ end
+
+ def block_type(block)
+ raw = block.respond_to?(:type) ? block.type : block[:type] || block["type"]
+ raw.to_s.to_sym
+ end
+
+ def block_input(block)
+ block.respond_to?(:input) ? block.input : (block[:input] || block["input"])
+ end
+
+ def usage_hash(raw_usage)
+ return {} unless raw_usage
+ {
+ "input_tokens" => raw_usage.input_tokens.to_i,
+ "output_tokens" => raw_usage.output_tokens.to_i,
+ "total_tokens" => raw_usage.input_tokens.to_i + raw_usage.output_tokens.to_i
+ }
+ end
+end
diff --git a/app/models/provider/anthropic/pdf_processor.rb b/app/models/provider/anthropic/pdf_processor.rb
new file mode 100644
index 000000000..dc6dc2c96
--- /dev/null
+++ b/app/models/provider/anthropic/pdf_processor.rb
@@ -0,0 +1,185 @@
+class Provider::Anthropic::PdfProcessor
+ include Provider::Anthropic::Concerns::UsageRecorder
+
+ TOOL_NAME = "report_document_analysis".freeze
+
+ # Anthropic enforces a 32 MB limit on the whole Messages *request body*, and
+ # the PDF travels base64-encoded (~4/3 larger) inside that body alongside the
+ # JSON envelope (instructions, tool schema). So a 32 MB raw PDF would encode
+ # to ~42 MB and be rejected. Cap the raw bytes at 3/4 of the request budget,
+ # minus a generous envelope reserve, so the encoded request stays under the
+ # limit. Guarding upstream also avoids base64-encoding an over-size blob in
+ # vain (peak heap before the API would reject it).
+ MAX_REQUEST_BYTES = 32 * 1024 * 1024
+ REQUEST_ENVELOPE_BYTES = 1 * 1024 * 1024
+ MAX_PDF_BYTES = (MAX_REQUEST_BYTES - REQUEST_ENVELOPE_BYTES) * 3 / 4
+
+ attr_reader :client, :model, :pdf_content, :langfuse_trace, :family
+
+ def initialize(client, model:, pdf_content:, langfuse_trace: nil, family: nil)
+ @client = client
+ @model = model
+ @pdf_content = pdf_content
+ @langfuse_trace = langfuse_trace
+ @family = family
+ end
+
+ def process
+ raise Provider::Anthropic::Error, "PDF content is required" if pdf_content.blank?
+ if pdf_content.bytesize > MAX_PDF_BYTES
+ raise Provider::Anthropic::Error,
+ "PDF is too large (#{pdf_content.bytesize} bytes); base64-encoded it would exceed Anthropic's 32 MB request limit"
+ end
+
+ span = langfuse_trace&.span(name: "process_pdf_api_call", input: {
+ model: model,
+ pdf_size: pdf_content&.bytesize
+ })
+
+ response = client.messages.create(
+ model: model,
+ max_tokens: max_tokens,
+ system_: instructions,
+ messages: [ { role: "user", content: user_content } ],
+ tools: [ output_tool ],
+ tool_choice: { type: "tool", name: TOOL_NAME, disable_parallel_tool_use: true }
+ )
+
+ parsed = extract_tool_input(response)
+ result = build_result(parsed)
+
+ record_usage(model, response.usage, operation: "process_pdf", metadata: { pdf_size: pdf_content.bytesize })
+
+ span&.end(output: result.to_h, usage: usage_hash(response.usage))
+ result
+ rescue => e
+ span&.end(output: { error: e.message }, level: "ERROR")
+ record_usage_error(model, operation: "process_pdf", error: e, metadata: { pdf_size: pdf_content&.bytesize })
+ raise
+ end
+
+ private
+ PdfProcessingResult = Provider::LlmConcept::PdfProcessingResult
+
+ def max_tokens
+ ENV.fetch("ANTHROPIC_MAX_TOKENS", 4096).to_i
+ end
+
+ def user_content
+ [
+ {
+ type: "document",
+ source: {
+ type: "base64",
+ media_type: "application/pdf",
+ data: Base64.strict_encode64(pdf_content)
+ }
+ },
+ {
+ type: "text",
+ text: "Analyze the attached document and return the result via the report_document_analysis tool."
+ }
+ ]
+ end
+
+ def output_tool
+ {
+ name: TOOL_NAME,
+ description: "Return the structured analysis of the attached document.",
+ input_schema: {
+ type: "object",
+ properties: {
+ document_type: {
+ type: "string",
+ enum: Import::DOCUMENT_TYPES,
+ description: "Classification of the document."
+ },
+ summary: {
+ type: "string",
+ description: "Concise human-readable summary of the document."
+ },
+ extracted_data: {
+ type: "object",
+ properties: {
+ institution_name: { type: [ "string", "null" ] },
+ statement_period_start: { type: [ "string", "null" ], pattern: "^\\d{4}-\\d{2}-\\d{2}$", description: "YYYY-MM-DD or null" },
+ statement_period_end: { type: [ "string", "null" ], pattern: "^\\d{4}-\\d{2}-\\d{2}$", description: "YYYY-MM-DD or null" },
+ transaction_count: { type: [ "integer", "null" ] },
+ opening_balance: { type: [ "number", "null" ] },
+ closing_balance: { type: [ "number", "null" ] },
+ currency: { type: [ "string", "null" ] },
+ account_holder: { type: [ "string", "null" ] }
+ },
+ required: [],
+ additionalProperties: false
+ }
+ },
+ required: [ "document_type", "summary", "extracted_data" ],
+ additionalProperties: false
+ }
+ }
+ end
+
+ def instructions
+ <<~INSTRUCTIONS
+ You analyze financial documents. For the attached PDF, classify the document type,
+ summarize it, and extract key metadata. Return the result via the report_document_analysis tool.
+
+ Classification options:
+ - bank_statement: bank account statements (incl. mobile money / digital wallets)
+ - credit_card_statement: credit card statements
+ - investment_statement: brokerage / investment statements
+ - financial_document: tax forms, receipts, invoices, financial reports
+ - contract: legal agreements, loans, terms of service
+ - other: anything else
+
+ Rules:
+ - Be factual; only report what is clearly visible
+ - If a field is unclear/redacted, return null for it
+ - Do not invent figures or names you cannot read
+ - For statements with many transactions, return the count rather than enumerating them
+ INSTRUCTIONS
+ end
+
+ def extract_tool_input(response)
+ tool_use = Array(response.content).find { |block| block_type(block) == :tool_use }
+ raise Provider::Anthropic::Error, "Model did not invoke #{TOOL_NAME}" unless tool_use
+
+ input = block_input(tool_use)
+ input = JSON.parse(input) if input.is_a?(String)
+ input
+ end
+
+ def build_result(parsed)
+ PdfProcessingResult.new(
+ summary: parsed["summary"] || parsed[:summary],
+ document_type: normalize_document_type(parsed["document_type"] || parsed[:document_type]),
+ extracted_data: parsed["extracted_data"] || parsed[:extracted_data] || {}
+ )
+ end
+
+ def normalize_document_type(doc_type)
+ return "other" if doc_type.blank?
+
+ normalized = doc_type.to_s.strip.downcase.gsub(/\s+/, "_")
+ Import::DOCUMENT_TYPES.include?(normalized) ? normalized : "other"
+ end
+
+ def block_type(block)
+ raw = block.respond_to?(:type) ? block.type : block[:type] || block["type"]
+ raw.to_s.to_sym
+ end
+
+ def block_input(block)
+ block.respond_to?(:input) ? block.input : (block[:input] || block["input"])
+ end
+
+ def usage_hash(raw_usage)
+ return {} unless raw_usage
+ {
+ "input_tokens" => raw_usage.input_tokens.to_i,
+ "output_tokens" => raw_usage.output_tokens.to_i,
+ "total_tokens" => raw_usage.input_tokens.to_i + raw_usage.output_tokens.to_i
+ }
+ end
+end
diff --git a/app/models/provider/enable_banking.rb b/app/models/provider/enable_banking.rb
index 88df7ec62..da6a2ea06 100644
--- a/app/models/provider/enable_banking.rb
+++ b/app/models/provider/enable_banking.rb
@@ -39,9 +39,12 @@ class Provider::EnableBanking
# @param psu_type [String] "personal" or "business"
# @param maximum_consent_validity [Integer, nil] Max consent duration in seconds from ASPSP (nil = use 90 days)
# @param language [String, nil] Two-letter language code (e.g. "fr", "en")
+ # @param auth_method [String, nil] Name of a specific authentication method to use (from the ASPSP's
+ # auth_methods list). Required to drive DECOUPLED/EMBEDDED banks that expose several methods; when nil
+ # Enable Banking falls back to the ASPSP's default method.
# @return [Hash] Contains :url and :authorization_id
def start_authorization(aspsp_name:, aspsp_country:, redirect_url:, state: nil,
- psu_type: "personal", maximum_consent_validity: nil, language: nil)
+ psu_type: "personal", maximum_consent_validity: nil, language: nil, auth_method: nil)
max_seconds = maximum_consent_validity ? [ maximum_consent_validity, 1 ].max : 90.days.to_i
valid_until = [ Time.current + max_seconds.seconds, Time.current + 90.days ].min
@@ -60,6 +63,7 @@ class Provider::EnableBanking
psu_type: psu_type
}
body[:language] = language if language.present?
+ body[:auth_method] = auth_method if auth_method.present?
body = body.compact
response = self.class.post(
diff --git a/app/models/sure_import.rb b/app/models/sure_import.rb
index c738e6156..096b3771d 100644
--- a/app/models/sure_import.rb
+++ b/app/models/sure_import.rb
@@ -129,16 +129,28 @@ class SureImport < Import
self.class.dry_run_totals_from_ndjson(ndjson_blob_string)
end
- def import!
+ def import!(import_session: nil)
sync_ndjson_counts!
before_counts = readback_count_snapshot
- importer = Family::DataImporter.new(family, ndjson_blob_string)
+ importer = Family::DataImporter.new(family, ndjson_blob_string, import_session: import_session, import: self)
result = importer.import!
- result[:accounts].each { |account| accounts << account }
- result[:entries].each { |entry| entries << entry }
+ Import.transaction do
+ result[:accounts].each { |account| account.save! if account.new_record? }
+ result[:entries].each { |entry| entry.save! if entry.new_record? }
+
+ account_ids = result[:accounts].filter_map(&:id)
+ entry_ids = result[:entries].filter_map(&:id)
+ existing_account_ids = accounts.where(id: account_ids).pluck(:id)
+ existing_entry_ids = entries.where(id: entry_ids).pluck(:id)
+
+ accounts.concat(result[:accounts].reject { |account| existing_account_ids.include?(account.id) })
+ entries.concat(result[:entries].reject { |entry| existing_entry_ids.include?(entry.id) })
+ update!(summary: result[:summary]) if has_attribute?(:summary)
+ end
record_readback_verification!(before_counts:)
+ result
rescue => error
record_failed_readback_verification!(before_counts:, error:)
raise
diff --git a/app/views/budget_categories/_budget_category.html.erb b/app/views/budget_categories/_budget_category.html.erb
index 6b55b971e..647f2e91b 100644
--- a/app/views/budget_categories/_budget_category.html.erb
+++ b/app/views/budget_categories/_budget_category.html.erb
@@ -38,7 +38,7 @@
<%# Progress Bar %>
- <% bar_color = budget_category.over_budget? ? "bg-red-500" : (budget_category.near_limit? ? "bg-yellow-500" : "bg-green-500") %>
+ <% bar_color = budget_category.over_budget? ? "bg-destructive" : (budget_category.near_limit? ? "bg-warning" : "bg-success") %>
@@ -67,8 +67,7 @@
<%= t("reports.budget_performance.suggested_daily",
amount: daily_info[:amount].format,
- days: daily_info[:days_remaining])
- %>
+ days: daily_info[:days_remaining]) %>
<% end %>
<% end %>
diff --git a/app/views/category/dropdowns/_row.html.erb b/app/views/category/dropdowns/_row.html.erb
index 067286e24..9099fd215 100644
--- a/app/views/category/dropdowns/_row.html.erb
+++ b/app/views/category/dropdowns/_row.html.erb
@@ -20,7 +20,9 @@
data: { turbo_frame: "category_dropdown" },
class: "flex w-full items-center gap-1.5 cursor-pointer focus:outline-none" do %>
- <%= icon("check") if is_selected %>
+
+ <% if is_selected %><%= icon("check") %><% end %>
+
<% if category.subcategory? %>
<%= icon("corner-down-right", size: "sm") %>
diff --git a/app/views/layouts/auth.html.erb b/app/views/layouts/auth.html.erb
index 271abef83..54d0adf51 100644
--- a/app/views/layouts/auth.html.erb
+++ b/app/views/layouts/auth.html.erb
@@ -7,29 +7,25 @@
<%= image_tag "logomark.svg", class: "w-16 mb-6" %>
- <% if (controller_name == "sessions" && action_name == "new") || (controller_name == "registrations" && action_name == "new") %>
-
+ <% if controller_name.in?(%w[sessions registrations]) && action_name.in?(%w[new create]) %>
+ <%# Determine the active tab from the controller, not current_page?:
+ a failed POST re-renders :new from #create, where the request
+ path is the form target (not /sessions/new), so current_page?
+ would mis-highlight the switch. %>
+ <% on_sign_in = controller_name == "sessions" %>
+
<%= link_to new_session_path,
- class: "w-1/2 px-2 py-1 rounded-md text-sm text-center font-medium #{current_page?(new_session_path) ? 'bg-surface shadow-sm text-primary' : 'text-secondary'}" do %>
+ class: "w-1/2 px-2 py-1 rounded-md text-sm text-center font-medium #{on_sign_in ? 'bg-surface shadow-sm text-primary' : 'text-secondary'}" do %>
<%= t("layouts.auth.sign_in") %>
<% end %>
<%= link_to new_registration_path,
- class: "w-1/2 px-2 py-1 rounded-md text-sm text-center font-medium #{!current_page?(new_session_path) ? 'bg-surface shadow-sm text-primary' : 'text-secondary'}" do %>
+ class: "w-1/2 px-2 py-1 rounded-md text-sm text-center font-medium #{!on_sign_in ? 'bg-surface shadow-sm text-primary' : 'text-secondary'}" do %>
<%= t("layouts.auth.sign_up") %>
<% end %>
<% end %>
- <% if controller_name == "sessions" %>
-
- <%= tag.span t("layouts.auth.no_account", product_name: product_name), class: "text-secondary" %> <%= link_to t("layouts.auth.sign_up"), new_registration_path, class: "font-medium text-primary hover:underline transition" %>
-
- <% elsif controller_name == "registrations" %>
-
- <%= t("layouts.auth.existing_account") %> <%= link_to t("layouts.auth.sign_in"), new_session_path, class: "font-medium text-primary hover:underline transition" %>
-
- <% end %>
gap-6 pb-6 lg:pb-12" data-controller="dashboard-sortable" data-action="dragover->dashboard-sortable#dragOver drop->dashboard-sortable#drop" role="list" aria-label="Dashboard sections">
+<%= turbo_frame_tag "dashboard_sections", target: "_top" do %>
+ <% if accessible_accounts.any? %>
+
+ <%= render UI::PeriodPicker.new(selected: @period, url: root_path, frame: "dashboard_sections") %>
+
+ <% end %>
+
+
gap-6 pb-6 lg:pb-12" data-controller="dashboard-sortable" data-action="dragover->dashboard-sortable#dragOver drop->dashboard-sortable#drop" role="list" aria-label="Dashboard sections">
<% if accessible_accounts.any? %>
<% @dashboard_sections.each do |section| %>
<% next unless section[:visible] %>
@@ -96,4 +103,5 @@
<%= render "pages/dashboard/no_accounts_graph_placeholder" %>
<% end %>
-
+
-
- <%= form_with url: root_path, method: :get, data: { controller: "auto-submit-form", turbo_frame: "_top" } do |form| %>
- <%= form.select :period,
- Period.as_options,
- { selected: period.key },
- data: { "auto-submit-form-target": "auto" },
- class: "bg-container border border-secondary font-medium rounded-lg px-3 py-2 text-sm pr-7 cursor-pointer text-primary focus:outline-hidden focus:ring-0" %>
- <% end %>
-
<% if sankey_data[:links].present? %>
<%= render "pages/dashboard/cashflow_sankey_chart", sankey_data: sankey_data %>
diff --git a/app/views/pages/dashboard/_investment_summary.html.erb b/app/views/pages/dashboard/_investment_summary.html.erb
index 8509bfe08..b38c2f901 100644
--- a/app/views/pages/dashboard/_investment_summary.html.erb
+++ b/app/views/pages/dashboard/_investment_summary.html.erb
@@ -97,7 +97,7 @@
<%= t(".contributions") %>
-
<%= format_money(totals.contributions) %>
+
<%= format_money(totals.contributions) %>
@@ -106,7 +106,7 @@
<%= t(".withdrawals") %>
-
<%= format_money(totals.withdrawals) %>
+
<%= format_money(totals.withdrawals) %>
diff --git a/app/views/pages/dashboard/_net_worth_chart.html.erb b/app/views/pages/dashboard/_net_worth_chart.html.erb
index 1a5f81d68..c9696f1ab 100644
--- a/app/views/pages/dashboard/_net_worth_chart.html.erb
+++ b/app/views/pages/dashboard/_net_worth_chart.html.erb
@@ -13,14 +13,6 @@
<%= t(".data_not_available") %>
<% end %>
-
- <%= form_with url: root_path, method: :get, data: { controller: "auto-submit-form", turbo_frame: "_top" } do |form| %>
- <%= form.select :period,
- Period.as_options,
- { selected: period.key },
- data: { "auto-submit-form-target": "auto" },
- class: "bg-container border border-secondary font-medium rounded-lg px-3 py-2 text-sm pr-7 cursor-pointer text-primary focus:outline-hidden focus:ring-0" %>
- <% end %>
-
-
diff --git a/app/views/reports/_breakdown_table.html.erb b/app/views/reports/_breakdown_table.html.erb
index ff3141557..95fbfab1b 100644
--- a/app/views/reports/_breakdown_table.html.erb
+++ b/app/views/reports/_breakdown_table.html.erb
@@ -2,7 +2,7 @@
<%# Local variables: groups, total, type (:income or :expense), amount_sort_params, current_sort_by, current_sort_direction %>
<%
- color_class = type == :income ? "text-success" : "text-destructive"
+ color_class = type == :income ? "text-success" : "text-primary"
icon_name = type == :income ? "trending-up" : "trending-down"
title_key = type == :income ? "reports.transactions_breakdown.table.income" : "reports.transactions_breakdown.table.expense"
%>
diff --git a/app/views/reports/_net_worth.html.erb b/app/views/reports/_net_worth.html.erb
index 492d636b6..93f979983 100644
--- a/app/views/reports/_net_worth.html.erb
+++ b/app/views/reports/_net_worth.html.erb
@@ -29,9 +29,9 @@
<%= t("reports.net_worth.assets_vs_liabilities") %>
- <%= net_worth_metrics[:total_assets].format %>
+ <%= net_worth_metrics[:total_assets].format %>
-
- <%= net_worth_metrics[:total_liabilities].format %>
+ <%= net_worth_metrics[:total_liabilities].format %>
@@ -52,7 +52,7 @@
<% net_worth_metrics[:asset_groups].each_with_index do |group, idx| %>
">
| <%= group[:name] %> |
- <%= group[:total].format %> |
+ <%= group[:total].format %> |
<% end %>
<% if net_worth_metrics[:asset_groups].empty? %>
@@ -79,7 +79,7 @@
<% net_worth_metrics[:liability_groups].each_with_index do |group, idx| %>
">
| <%= group[:name] %> |
- <%= group[:total].format %> |
+ <%= group[:total].format %> |
<% end %>
<% if net_worth_metrics[:liability_groups].empty? %>
diff --git a/app/views/reports/_trends_insights.html.erb b/app/views/reports/_trends_insights.html.erb
index f3106ea60..c74ab704f 100644
--- a/app/views/reports/_trends_insights.html.erb
+++ b/app/views/reports/_trends_insights.html.erb
@@ -66,7 +66,7 @@
<%= t("reports.trends.avg_monthly_expenses") %>
-
+
<%= Money.new(avg_expenses, Current.family.currency).format %>
diff --git a/app/views/settings/llm_usages/show.html.erb b/app/views/settings/llm_usages/show.html.erb
index 39929a55b..d0b8e5967 100644
--- a/app/views/settings/llm_usages/show.html.erb
+++ b/app/views/settings/llm_usages/show.html.erb
@@ -103,7 +103,7 @@
<%= t(".recent_usage") %>
-