From 14993d871c8578d817b113091e729512ce0e940a Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Sat, 3 Jan 2026 17:56:42 -0500 Subject: [PATCH 01/54] feat: comprehensive SSO/OIDC upgrade with enterprise features Multi-provider SSO support: - Database-backed SSO provider management with admin UI - Support for OpenID Connect, Google OAuth2, GitHub, and SAML 2.0 - Flipper feature flag (db_sso_providers) for dynamic provider loading - ProviderLoader service for YAML or database configuration Admin functionality: - Admin::SsoProvidersController for CRUD operations - Admin::UsersController for super_admin role management - Pundit policies for authorization - Test connection endpoint for validating provider config User provisioning improvements: - JIT (just-in-time) account creation with configurable default role - Changed default JIT role from admin to member (security) - User attribute sync on each SSO login - Group/role mapping from IdP claims SSO identity management: - Settings::SsoIdentitiesController for users to manage connected accounts - Issuer validation for OIDC identities - Unlink protection when no password set Audit logging: - SsoAuditLog model tracking login, logout, link, unlink, JIT creation - Captures IP address, user agent, and metadata Advanced OIDC features: - Custom scopes per provider - Configurable prompt parameter (login, consent, select_account, none) - RP-initiated logout (federated logout to IdP) - id_token storage for logout SAML 2.0 support: - omniauth-saml gem integration - IdP metadata URL or manual configuration - Certificate and fingerprint validation - NameID format configuration --- Gemfile | 8 +- Gemfile.lock | 17 ++ app/controllers/admin/base_controller.rb | 16 + .../admin/sso_providers_controller.rb | 157 ++++++++++ app/controllers/admin/users_controller.rb | 38 +++ app/controllers/application_controller.rb | 6 + app/controllers/oidc_accounts_controller.rb | 20 +- app/controllers/sessions_controller.rb | 108 ++++++- .../settings/sso_identities_controller.rb | 35 +++ app/helpers/settings_helper.rb | 5 + .../controllers/admin_sso_form_controller.js | 226 ++++++++++++++ app/middleware/omniauth_error_handler.rb | 29 ++ app/models/oidc_identity.rb | 83 ++++++ app/models/sso_audit_log.rb | 108 +++++++ app/models/sso_provider.rb | 144 +++++++++ app/models/sso_provider_tester.rb | 201 +++++++++++++ app/policies/application_policy.rb | 53 ++++ app/policies/sso_provider_policy.rb | 46 +++ app/policies/user_policy.rb | 24 ++ app/services/provider_loader.rb | 87 ++++++ app/views/admin/sso_providers/_form.html.erb | 277 ++++++++++++++++++ app/views/admin/sso_providers/edit.html.erb | 9 + app/views/admin/sso_providers/index.html.erb | 88 ++++++ app/views/admin/sso_providers/new.html.erb | 9 + app/views/admin/users/index.html.erb | 73 +++++ app/views/settings/_settings_nav.html.erb | 4 +- .../settings/sso_identities/show.html.erb | 59 ++++ config/application.rb | 4 + config/auth.yml | 14 + config/brakeman.ignore | 23 ++ config/initializers/flipper.rb | 45 +++ config/initializers/omniauth.rb | 162 ++++++++-- config/initializers/rack_attack.rb | 6 + .../locales/views/admin/sso_providers/en.yml | 109 +++++++ config/locales/views/admin/users/en.yml | 22 ++ config/locales/views/sessions/en.yml | 3 + .../views/settings/sso_identities/en.yml | 22 ++ config/routes.rb | 16 +- .../20251228181150_create_flipper_tables.rb | 22 ++ .../20251228181429_create_sso_providers.rb | 21 ++ ...228182113_add_issuer_to_oidc_identities.rb | 6 + .../20260103170412_create_sso_audit_logs.rb | 18 ++ db/schema.rb | 53 +++- docs/hosting/oidc.md | 235 +++++++++++++++ lib/tasks/sso_providers.rake | 154 ++++++++++ .../oidc_accounts_controller_test.rb | 2 +- test/models/sso_provider_test.rb | 263 +++++++++++++++++ test/policies/sso_provider_policy_test.rb | 111 +++++++ test/policies/user_policy_test.rb | 59 ++++ test/test_helper.rb | 1 + 50 files changed, 3267 insertions(+), 34 deletions(-) create mode 100644 app/controllers/admin/base_controller.rb create mode 100644 app/controllers/admin/sso_providers_controller.rb create mode 100644 app/controllers/admin/users_controller.rb create mode 100644 app/controllers/settings/sso_identities_controller.rb create mode 100644 app/javascript/controllers/admin_sso_form_controller.js create mode 100644 app/middleware/omniauth_error_handler.rb create mode 100644 app/models/sso_audit_log.rb create mode 100644 app/models/sso_provider.rb create mode 100644 app/models/sso_provider_tester.rb create mode 100644 app/policies/application_policy.rb create mode 100644 app/policies/sso_provider_policy.rb create mode 100644 app/policies/user_policy.rb create mode 100644 app/services/provider_loader.rb create mode 100644 app/views/admin/sso_providers/_form.html.erb create mode 100644 app/views/admin/sso_providers/edit.html.erb create mode 100644 app/views/admin/sso_providers/index.html.erb create mode 100644 app/views/admin/sso_providers/new.html.erb create mode 100644 app/views/admin/users/index.html.erb create mode 100644 app/views/settings/sso_identities/show.html.erb create mode 100644 config/initializers/flipper.rb create mode 100644 config/locales/views/admin/sso_providers/en.yml create mode 100644 config/locales/views/admin/users/en.yml create mode 100644 config/locales/views/settings/sso_identities/en.yml create mode 100644 db/migrate/20251228181150_create_flipper_tables.rb create mode 100644 db/migrate/20251228181429_create_sso_providers.rb create mode 100644 db/migrate/20251228182113_add_issuer_to_oidc_identities.rb create mode 100644 db/migrate/20260103170412_create_sso_audit_logs.rb create mode 100644 lib/tasks/sso_providers.rake create mode 100644 test/models/sso_provider_test.rb create mode 100644 test/policies/sso_provider_policy_test.rb create mode 100644 test/policies/user_policy_test.rb diff --git a/Gemfile b/Gemfile index 91cdfcd8b..0658791a6 100644 --- a/Gemfile +++ b/Gemfile @@ -59,6 +59,7 @@ gem "countries" # OAuth & API Security gem "doorkeeper" gem "rack-attack", "~> 6.6" +gem "pundit" gem "faraday" gem "faraday-retry" gem "faraday-multipart" @@ -77,17 +78,22 @@ gem "rqrcode", "~> 3.0" gem "activerecord-import" gem "rubyzip", "~> 2.3" -# OpenID Connect & OAuth authentication +# OpenID Connect, OAuth & SAML authentication gem "omniauth", "~> 2.1" gem "omniauth-rails_csrf_protection" gem "omniauth_openid_connect" gem "omniauth-google-oauth2" gem "omniauth-github" +gem "omniauth-saml", "~> 2.1" # State machines gem "aasm" gem "after_commit_everywhere", "~> 1.0" +# Feature flags +gem "flipper" +gem "flipper-active_record" + # AI gem "ruby-openai" gem "langfuse-ruby", "~> 0.1.4", require: "langfuse" diff --git a/Gemfile.lock b/Gemfile.lock index 7a4f3959e..d251bef9a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -223,6 +223,11 @@ GEM ffi (1.17.2-x86_64-darwin) ffi (1.17.2-x86_64-linux-gnu) ffi (1.17.2-x86_64-linux-musl) + flipper (1.3.6) + concurrent-ruby (< 2) + flipper-active_record (1.3.6) + activerecord (>= 4.2, < 9) + flipper (~> 1.3.6) foreman (0.88.1) fugit (1.11.1) et-orbi (~> 1, >= 1.2.11) @@ -420,6 +425,9 @@ GEM omniauth-rails_csrf_protection (1.0.2) actionpack (>= 4.2) omniauth (~> 2.0) + omniauth-saml (2.2.4) + omniauth (~> 2.1) + ruby-saml (~> 1.18) omniauth_openid_connect (0.8.0) omniauth (>= 1.9, < 3) openid_connect (~> 2.2) @@ -466,6 +474,8 @@ GEM public_suffix (6.0.2) puma (6.6.0) nio4r (~> 2.0) + pundit (2.5.2) + activesupport (>= 3.0.0) raabro (1.4.0) racc (1.8.1) rack (3.1.18) @@ -619,6 +629,9 @@ GEM faraday (>= 1) faraday-multipart (>= 1) ruby-progressbar (1.13.0) + ruby-saml (1.18.1) + nokogiri (>= 1.13.10) + rexml ruby-statistics (4.1.0) ruby-vips (2.2.4) ffi (~> 1.12) @@ -774,6 +787,8 @@ DEPENDENCIES faraday faraday-multipart faraday-retry + flipper + flipper-active_record foreman hotwire-livereload hotwire_combobox @@ -795,6 +810,7 @@ DEPENDENCIES omniauth-github omniauth-google-oauth2 omniauth-rails_csrf_protection + omniauth-saml (~> 2.1) omniauth_openid_connect ostruct pagy @@ -803,6 +819,7 @@ DEPENDENCIES posthog-ruby propshaft puma (>= 5.0) + pundit rack-attack (~> 6.6) rack-mini-profiler rails (~> 7.2.2) diff --git a/app/controllers/admin/base_controller.rb b/app/controllers/admin/base_controller.rb new file mode 100644 index 000000000..7e0252491 --- /dev/null +++ b/app/controllers/admin/base_controller.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Admin + class BaseController < ApplicationController + before_action :require_super_admin! + + layout "settings" + + private + def require_super_admin! + unless Current.user&.super_admin? + redirect_to root_path, alert: t("admin.unauthorized") + end + end + end +end diff --git a/app/controllers/admin/sso_providers_controller.rb b/app/controllers/admin/sso_providers_controller.rb new file mode 100644 index 000000000..d47864f43 --- /dev/null +++ b/app/controllers/admin/sso_providers_controller.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module Admin + class SsoProvidersController < Admin::BaseController + before_action :set_sso_provider, only: %i[show edit update destroy toggle test_connection] + + def index + authorize SsoProvider + @sso_providers = policy_scope(SsoProvider).order(:name) + end + + def show + authorize @sso_provider + end + + def new + @sso_provider = SsoProvider.new + authorize @sso_provider + end + + def create + @sso_provider = SsoProvider.new(processed_params) + authorize @sso_provider + + # Auto-generate redirect_uri if not provided + if @sso_provider.redirect_uri.blank? && @sso_provider.name.present? + @sso_provider.redirect_uri = "#{request.base_url}/auth/#{@sso_provider.name}/callback" + end + + if @sso_provider.save + log_provider_change(:create, @sso_provider) + clear_provider_cache + redirect_to admin_sso_providers_path, notice: t(".success") + else + render :new, status: :unprocessable_entity + end + end + + def edit + authorize @sso_provider + end + + def update + authorize @sso_provider + + # Auto-update redirect_uri if name changed + params_hash = processed_params.to_h + if params_hash[:name].present? && params_hash[:name] != @sso_provider.name + params_hash[:redirect_uri] = "#{request.base_url}/auth/#{params_hash[:name]}/callback" + end + + if @sso_provider.update(params_hash) + log_provider_change(:update, @sso_provider) + clear_provider_cache + redirect_to admin_sso_providers_path, notice: t(".success") + else + render :edit, status: :unprocessable_entity + end + end + + def destroy + authorize @sso_provider + + @sso_provider.destroy! + log_provider_change(:destroy, @sso_provider) + clear_provider_cache + + redirect_to admin_sso_providers_path, notice: t(".success") + end + + def toggle + authorize @sso_provider + + @sso_provider.update!(enabled: !@sso_provider.enabled) + log_provider_change(:toggle, @sso_provider) + clear_provider_cache + + notice = @sso_provider.enabled? ? t(".success_enabled") : t(".success_disabled") + redirect_to admin_sso_providers_path, notice: notice + end + + def test_connection + authorize @sso_provider + + tester = SsoProviderTester.new(@sso_provider) + result = tester.test! + + render json: { + success: result.success?, + message: result.message, + details: result.details + } + end + + private + def set_sso_provider + @sso_provider = SsoProvider.find(params[:id]) + end + + def sso_provider_params + params.require(:sso_provider).permit( + :strategy, + :name, + :label, + :icon, + :enabled, + :issuer, + :client_id, + :client_secret, + :redirect_uri, + :scopes, + :prompt, + settings: [ + :default_role, :scopes, :prompt, + # SAML settings + :idp_metadata_url, :idp_sso_url, :idp_slo_url, + :idp_certificate, :idp_cert_fingerprint, :name_id_format, + role_mapping: {} + ] + ) + end + + # Process params to convert role_mapping comma-separated strings to arrays + def processed_params + result = sso_provider_params.to_h + + if result[:settings].present? && result[:settings][:role_mapping].present? + result[:settings][:role_mapping] = result[:settings][:role_mapping].transform_values do |v| + # Convert comma-separated string to array, removing empty values + v.to_s.split(",").map(&:strip).reject(&:blank?) + end + + # Remove empty role mappings + result[:settings][:role_mapping] = result[:settings][:role_mapping].reject { |_, v| v.empty? } + result[:settings].delete(:role_mapping) if result[:settings][:role_mapping].empty? + end + + result + end + + def log_provider_change(action, provider) + Rails.logger.info( + "[Admin::SsoProviders] #{action.to_s.upcase} - " \ + "user_id=#{Current.user.id} " \ + "provider_id=#{provider.id} " \ + "provider_name=#{provider.name} " \ + "strategy=#{provider.strategy} " \ + "enabled=#{provider.enabled}" + ) + end + + def clear_provider_cache + ProviderLoader.clear_cache + Rails.logger.info("[Admin::SsoProviders] Provider cache cleared by user_id=#{Current.user.id}") + end + end +end diff --git a/app/controllers/admin/users_controller.rb b/app/controllers/admin/users_controller.rb new file mode 100644 index 000000000..fdbc6e281 --- /dev/null +++ b/app/controllers/admin/users_controller.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Admin + class UsersController < Admin::BaseController + before_action :set_user, only: %i[update] + + def index + authorize User + @users = policy_scope(User).order(:email) + end + + def update + authorize @user + + if @user.update(user_params) + Rails.logger.info( + "[Admin::Users] Role changed - " \ + "by_user_id=#{Current.user.id} " \ + "target_user_id=#{@user.id} " \ + "new_role=#{@user.role}" + ) + redirect_to admin_users_path, notice: t(".success") + else + redirect_to admin_users_path, alert: t(".failure") + end + end + + private + + def set_user + @user = User.find(params[:id]) + end + + def user_params + params.require(:user).permit(:role) + end + end +end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index d63adfa8a..b171a080d 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -2,9 +2,15 @@ class ApplicationController < ActionController::Base include RestoreLayoutPreferences, Onboardable, Localize, AutoSync, Authentication, Invitable, SelfHostable, StoreLocation, Impersonatable, Breadcrumbable, FeatureGuardable, Notifiable + include Pundit::Authorization include Pagy::Backend + # Pundit uses current_user by default, but this app uses Current.user + def pundit_user + Current.user + end + before_action :detect_os before_action :set_default_chat before_action :set_active_storage_url_options diff --git a/app/controllers/oidc_accounts_controller.rb b/app/controllers/oidc_accounts_controller.rb index a89dc9395..bfc6597ab 100644 --- a/app/controllers/oidc_accounts_controller.rb +++ b/app/controllers/oidc_accounts_controller.rb @@ -37,6 +37,13 @@ class OidcAccountsController < ApplicationController user ) + # Log account linking + SsoAuditLog.log_link!( + user: user, + provider: @pending_auth["provider"], + request: request + ) + # Clear pending auth from session session.delete(:pending_oidc_auth) @@ -103,7 +110,11 @@ class OidcAccountsController < ApplicationController # Create new family for this user @user.family = Family.new - @user.role = :admin + + # Use provider-configured default role, or fall back to member (not admin) + provider_config = Rails.configuration.x.auth.sso_providers&.find { |p| p[:name] == @pending_auth["provider"] } + default_role = provider_config&.dig(:settings, :default_role) || "member" + @user.role = default_role if @user.save # Create the OIDC (or other SSO) identity @@ -112,6 +123,13 @@ class OidcAccountsController < ApplicationController @user ) + # Log JIT account creation + SsoAuditLog.log_jit_account_created!( + user: @user, + provider: @pending_auth["provider"], + request: request + ) + # Clear pending auth from session session.delete(:pending_oidc_auth) diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index f42fac451..cb375d752 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -1,9 +1,14 @@ class SessionsController < ApplicationController before_action :set_session, only: :destroy - skip_authentication only: %i[new create openid_connect failure] + skip_authentication only: %i[index new create openid_connect failure post_logout] layout "auth" + # Handle GET /sessions (usually from browser back button) + def index + redirect_to new_session_path + end + def new begin demo = Rails.application.config_for(:demo) @@ -62,7 +67,32 @@ class SessionsController < ApplicationController end def destroy + user = Current.user + id_token = session[:id_token_hint] + oidc_identity = user.oidc_identities.first + + # Destroy local session @session.destroy + session.delete(:id_token_hint) + + # Check if we should redirect to IdP for federated logout + if oidc_identity && id_token.present? + idp_logout_url = build_idp_logout_url(oidc_identity, id_token) + + if idp_logout_url + SsoAuditLog.log_logout_idp!(user: user, provider: oidc_identity.provider, request: request) + redirect_to idp_logout_url, allow_other_host: true + return + end + end + + # Standard local logout + SsoAuditLog.log_logout!(user: user, request: request) + redirect_to new_session_path, notice: t(".logout_successful") + end + + # Handle redirect back from IdP after federated logout + def post_logout redirect_to new_session_path, notice: t(".logout_successful") end @@ -82,6 +112,13 @@ class SessionsController < ApplicationController # Existing OIDC identity found - authenticate the user user = oidc_identity.user oidc_identity.record_authentication! + oidc_identity.sync_user_attributes!(auth) + + # Store id_token for RP-initiated logout + session[:id_token_hint] = auth.credentials&.id_token if auth.credentials&.id_token + + # Log successful SSO login + SsoAuditLog.log_login!(user: user, provider: auth.provider, request: request) # MFA check: If user has MFA enabled, require verification if user.otp_required? @@ -107,7 +144,25 @@ class SessionsController < ApplicationController end def failure - redirect_to new_session_path, alert: t("sessions.failure.failed") + # Log failed SSO attempt + SsoAuditLog.log_login_failed!( + provider: params[:strategy], + request: request, + reason: params[:message] + ) + + message = case params[:message] + when "sso_provider_unavailable" + t("sessions.failure.sso_provider_unavailable") + when "sso_invalid_response" + t("sessions.failure.sso_invalid_response") + when "sso_failed" + t("sessions.failure.sso_failed") + else + t("sessions.failure.failed") + end + + redirect_to new_session_path, alert: message end private @@ -130,4 +185,53 @@ class SessionsController < ApplicationController demo["hosts"].include?(request.host) end + + def build_idp_logout_url(oidc_identity, id_token) + # Find the provider configuration + provider_config = Rails.configuration.x.auth.sso_providers&.find do |p| + p[:name] == oidc_identity.provider + end + + return nil unless provider_config + + # For OIDC providers, fetch end_session_endpoint from discovery + if provider_config[:strategy] == "openid_connect" && provider_config[:issuer].present? + begin + discovery_url = discovery_url_for(provider_config[:issuer]) + response = Faraday.get(discovery_url) do |req| + req.options.timeout = 5 + req.options.open_timeout = 3 + end + + return nil unless response.success? + + discovery = JSON.parse(response.body) + end_session_endpoint = discovery["end_session_endpoint"] + + return nil unless end_session_endpoint.present? + + # Build the logout URL with post_logout_redirect_uri + post_logout_redirect = "#{request.base_url}/auth/logout/callback" + params = { + id_token_hint: id_token, + post_logout_redirect_uri: post_logout_redirect + } + + "#{end_session_endpoint}?#{params.to_query}" + rescue Faraday::Error, JSON::ParserError, StandardError => e + Rails.logger.warn("[SSO] Failed to fetch OIDC discovery for logout: #{e.message}") + nil + end + else + nil + end + end + + def discovery_url_for(issuer) + if issuer.end_with?("/") + "#{issuer}.well-known/openid-configuration" + else + "#{issuer}/.well-known/openid-configuration" + end + end end diff --git a/app/controllers/settings/sso_identities_controller.rb b/app/controllers/settings/sso_identities_controller.rb new file mode 100644 index 000000000..f42175c62 --- /dev/null +++ b/app/controllers/settings/sso_identities_controller.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Settings::SsoIdentitiesController < ApplicationController + layout "settings" + + def show + @oidc_identities = Current.user.oidc_identities.order(:provider) + @breadcrumbs = [ + [ t("settings.nav.home"), root_path ], + [ t(".page_title"), nil ] + ] + end + + def destroy + @identity = Current.user.oidc_identities.find(params[:id]) + + # Prevent unlinking last identity if user has no password + if Current.user.oidc_identities.count == 1 && Current.user.password_digest.blank? + redirect_to settings_sso_identities_path, alert: t(".cannot_unlink_last") + return + end + + provider_name = @identity.provider + @identity.destroy! + + # Log account unlinking + SsoAuditLog.log_unlink!( + user: Current.user, + provider: provider_name, + request: request + ) + + redirect_to settings_sso_identities_path, notice: t(".success", provider: provider_name) + end +end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index a907cd7c8..1428aeda0 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -6,6 +6,7 @@ module SettingsHelper { name: "Preferences", path: :settings_preferences_path }, { name: "Profile Info", path: :settings_profile_path }, { name: "Security", path: :settings_security_path }, + { name: "Connected Accounts", path: :settings_sso_identities_path, condition: :has_sso_connections? }, { name: "Billing", path: :settings_billing_path, condition: :not_self_hosted? }, # Transactions section { name: "Categories", path: :categories_path }, @@ -81,4 +82,8 @@ module SettingsHelper def self_hosted_and_admin? self_hosted? && admin_user? end + + def has_sso_connections? + Current.user&.oidc_identities&.exists? || AuthConfig.sso_providers.any? + end end diff --git a/app/javascript/controllers/admin_sso_form_controller.js b/app/javascript/controllers/admin_sso_form_controller.js new file mode 100644 index 000000000..2344b8b63 --- /dev/null +++ b/app/javascript/controllers/admin_sso_form_controller.js @@ -0,0 +1,226 @@ +import { Controller } from "@hotwired/stimulus" + +// Connects to data-controller="admin-sso-form" +export default class extends Controller { + static targets = ["callbackUrl", "testResult", "samlCallbackUrl"] + + connect() { + // Initialize field visibility on page load + this.toggleFields() + // Initialize callback URL + this.updateCallbackUrl() + } + + updateCallbackUrl() { + const nameInput = this.element.querySelector('input[name*="[name]"]') + const callbackDisplay = this.callbackUrlTarget + + if (!nameInput || !callbackDisplay) return + + const providerName = nameInput.value.trim() || 'PROVIDER_NAME' + const baseUrl = window.location.origin + callbackDisplay.textContent = `${baseUrl}/auth/${providerName}/callback` + } + + toggleFields() { + const strategySelect = this.element.querySelector('select[name*="[strategy]"]') + if (!strategySelect) return + + const strategy = strategySelect.value + const isOidc = strategy === "openid_connect" + const isSaml = strategy === "saml" + + // Toggle OIDC fields + const oidcFields = this.element.querySelectorAll('[data-oidc-field]') + oidcFields.forEach(field => { + if (isOidc) { + field.classList.remove('hidden') + } else { + field.classList.add('hidden') + } + }) + + // Toggle SAML fields + const samlFields = this.element.querySelectorAll('[data-saml-field]') + samlFields.forEach(field => { + if (isSaml) { + field.classList.remove('hidden') + } else { + field.classList.add('hidden') + } + }) + + // Update SAML callback URL if present + if (this.hasSamlCallbackUrlTarget) { + this.updateSamlCallbackUrl() + } + } + + updateSamlCallbackUrl() { + const nameInput = this.element.querySelector('input[name*="[name]"]') + if (!nameInput || !this.hasSamlCallbackUrlTarget) return + + const providerName = nameInput.value.trim() || 'PROVIDER_NAME' + const baseUrl = window.location.origin + this.samlCallbackUrlTarget.textContent = `${baseUrl}/auth/${providerName}/callback` + } + + copySamlCallback(event) { + event.preventDefault() + + if (!this.hasSamlCallbackUrlTarget) return + + const callbackUrl = this.samlCallbackUrlTarget.textContent + + navigator.clipboard.writeText(callbackUrl).then(() => { + const button = event.currentTarget + const originalText = button.innerHTML + button.innerHTML = ' Copied!' + button.classList.add('text-green-600') + + setTimeout(() => { + button.innerHTML = originalText + button.classList.remove('text-green-600') + }, 2000) + }).catch(err => { + console.error('Failed to copy:', err) + alert('Failed to copy to clipboard') + }) + } + + async validateIssuer(event) { + const issuerInput = event.target + const issuer = issuerInput.value.trim() + + if (!issuer) return + + try { + // Construct discovery URL + const discoveryUrl = issuer.endsWith('/') + ? `${issuer}.well-known/openid-configuration` + : `${issuer}/.well-known/openid-configuration` + + // Show loading state + issuerInput.classList.add('border-yellow-300') + + const response = await fetch(discoveryUrl, { + method: 'GET', + headers: { 'Accept': 'application/json' } + }) + + if (response.ok) { + const data = await response.json() + if (data.issuer) { + // Valid OIDC discovery endpoint + issuerInput.classList.remove('border-yellow-300', 'border-red-300') + issuerInput.classList.add('border-green-300') + this.showValidationMessage(issuerInput, 'Valid OIDC issuer', 'success') + } else { + throw new Error('Invalid discovery response') + } + } else { + throw new Error(`Discovery endpoint returned ${response.status}`) + } + } catch (error) { + // CORS errors are expected when validating from browser - show as warning not error + issuerInput.classList.remove('border-yellow-300', 'border-green-300') + issuerInput.classList.add('border-amber-300') + this.showValidationMessage(issuerInput, "Could not validate from browser (CORS). Provider can still be saved.", 'warning') + } + } + + copyCallback(event) { + event.preventDefault() + + const callbackDisplay = this.callbackUrlTarget + if (!callbackDisplay) return + + const callbackUrl = callbackDisplay.textContent + + // Copy to clipboard + navigator.clipboard.writeText(callbackUrl).then(() => { + // Show success feedback + const button = event.currentTarget + const originalText = button.innerHTML + button.innerHTML = ' Copied!' + button.classList.add('text-green-600') + + setTimeout(() => { + button.innerHTML = originalText + button.classList.remove('text-green-600') + }, 2000) + }).catch(err => { + console.error('Failed to copy:', err) + alert('Failed to copy to clipboard') + }) + } + + showValidationMessage(input, message, type) { + // Remove any existing validation message + const existingMessage = input.parentElement.querySelector('.validation-message') + if (existingMessage) { + existingMessage.remove() + } + + // Create new validation message + const messageEl = document.createElement('p') + const colorClass = type === 'success' ? 'text-green-600' : type === 'warning' ? 'text-amber-600' : 'text-red-600' + messageEl.className = `validation-message mt-1 text-sm ${colorClass}` + messageEl.textContent = message + + input.parentElement.appendChild(messageEl) + + // Auto-remove after 5 seconds (except warnings which stay) + if (type !== 'warning') { + setTimeout(() => { + messageEl.remove() + input.classList.remove('border-green-300', 'border-red-300', 'border-amber-300') + }, 5000) + } + } + + async testConnection(event) { + const button = event.currentTarget + const testUrl = button.dataset.adminSsoFormTestUrlValue + const resultEl = this.testResultTarget + + if (!testUrl) return + + // Show loading state + button.disabled = true + button.textContent = 'Testing...' + resultEl.textContent = '' + resultEl.className = 'ml-2 text-sm' + + try { + const response = await fetch(testUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-CSRF-Token': document.querySelector('meta[name="csrf-token"]')?.content + } + }) + + const data = await response.json() + + if (data.success) { + resultEl.textContent = `✓ ${data.message}` + resultEl.classList.add('text-green-600') + } else { + resultEl.textContent = `✗ ${data.message}` + resultEl.classList.add('text-red-600') + } + + // Show details in console for debugging + if (data.details && Object.keys(data.details).length > 0) { + console.log('SSO Test Connection Details:', data.details) + } + } catch (error) { + resultEl.textContent = `✗ Request failed: ${error.message}` + resultEl.classList.add('text-red-600') + } finally { + button.disabled = false + button.textContent = 'Test Connection' + } + } +} diff --git a/app/middleware/omniauth_error_handler.rb b/app/middleware/omniauth_error_handler.rb new file mode 100644 index 000000000..8373cb760 --- /dev/null +++ b/app/middleware/omniauth_error_handler.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# Middleware to catch OmniAuth/OIDC errors and redirect gracefully +# instead of showing ugly error pages +class OmniauthErrorHandler + def initialize(app) + @app = app + end + + def call(env) + @app.call(env) + rescue OpenIDConnect::Discovery::DiscoveryFailed => e + Rails.logger.error("[OmniAuth] OIDC Discovery failed: #{e.message}") + redirect_to_failure(env, "sso_provider_unavailable") + rescue OmniAuth::Error => e + Rails.logger.error("[OmniAuth] Authentication error: #{e.message}") + redirect_to_failure(env, "sso_failed") + end + + private + + def redirect_to_failure(env, message) + [ + 302, + { "Location" => "/auth/failure?message=#{message}", "Content-Type" => "text/html" }, + [ "Redirecting..." ] + ] + end +end diff --git a/app/models/oidc_identity.rb b/app/models/oidc_identity.rb index 78276546d..e8993142f 100644 --- a/app/models/oidc_identity.rb +++ b/app/models/oidc_identity.rb @@ -10,12 +10,79 @@ class OidcIdentity < ApplicationRecord update!(last_authenticated_at: Time.current) end + # Sync user attributes from IdP on each login + # Updates stored identity info and syncs name to user (not email - that's identity) + def sync_user_attributes!(auth) + # Extract groups from claims (various common claim names) + groups = extract_groups(auth) + + # Update stored identity info with latest from IdP + update!(info: { + email: auth.info&.email, + name: auth.info&.name, + first_name: auth.info&.first_name, + last_name: auth.info&.last_name, + groups: groups + }) + + # Sync name to user if provided (keep existing if IdP doesn't provide) + user.update!( + first_name: auth.info&.first_name.presence || user.first_name, + last_name: auth.info&.last_name.presence || user.last_name + ) + + # Apply role mapping based on group membership + apply_role_mapping!(groups) + end + + # Extract groups from various common IdP claim formats + def extract_groups(auth) + # Try various common group claim locations + groups = auth.extra&.raw_info&.groups || + auth.extra&.raw_info&.[]("groups") || + auth.extra&.raw_info&.[]("Group") || + auth.info&.groups || + auth.extra&.raw_info&.[]("http://schemas.microsoft.com/ws/2008/06/identity/claims/groups") || + auth.extra&.raw_info&.[]("cognito:groups") || + [] + + # Normalize to array of strings + Array(groups).map(&:to_s) + end + + # Apply role mapping based on IdP group membership + def apply_role_mapping!(groups) + config = provider_config + return unless config.present? + + role_mapping = config.dig(:settings, :role_mapping) || config.dig(:settings, "role_mapping") + return unless role_mapping.present? + + # Check roles in order of precedence (highest to lowest) + %w[super_admin admin member].each do |role| + mapped_groups = role_mapping[role] || role_mapping[role.to_sym] || [] + mapped_groups = Array(mapped_groups) + + # Check if user is in any of the mapped groups + if mapped_groups.include?("*") || (mapped_groups & groups).any? + # Only update if different to avoid unnecessary writes + user.update!(role: role) unless user.role == role + Rails.logger.info("[SSO] Applied role mapping: user_id=#{user.id} role=#{role} groups=#{groups}") + return + end + end + end + # Extract and store relevant info from OmniAuth auth hash def self.create_from_omniauth(auth, user) + # Extract issuer from OIDC auth response if available + issuer = auth.extra&.raw_info&.iss || auth.extra&.raw_info&.[]("iss") + create!( user: user, provider: auth.provider, uid: auth.uid, + issuer: issuer, info: { email: auth.info&.email, name: auth.info&.name, @@ -25,4 +92,20 @@ class OidcIdentity < ApplicationRecord last_authenticated_at: Time.current ) end + + # Find the configured provider for this identity + def provider_config + Rails.configuration.x.auth.sso_providers&.find { |p| p[:name] == provider || p[:id] == provider } + end + + # Validate that the stored issuer matches the configured provider's issuer + # Returns true if valid, false if mismatch (security concern) + def issuer_matches_config? + return true if issuer.blank? # Backward compatibility for old records + + config = provider_config + return true if config.blank? || config[:issuer].blank? # No config to validate against + + issuer == config[:issuer] + end end diff --git a/app/models/sso_audit_log.rb b/app/models/sso_audit_log.rb new file mode 100644 index 000000000..21aa4e05e --- /dev/null +++ b/app/models/sso_audit_log.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +class SsoAuditLog < ApplicationRecord + belongs_to :user, optional: true + + # Event types for SSO audit logging + EVENT_TYPES = %w[ + login + login_failed + logout + logout_idp + link + unlink + jit_account_created + ].freeze + + validates :event_type, presence: true, inclusion: { in: EVENT_TYPES } + + scope :recent, -> { order(created_at: :desc) } + scope :for_user, ->(user) { where(user: user) } + scope :by_event, ->(event) { where(event_type: event) } + + class << self + # Log a successful SSO login + def log_login!(user:, provider:, request:, metadata: {}) + create!( + user: user, + event_type: "login", + provider: provider, + ip_address: request.remote_ip, + user_agent: request.user_agent&.truncate(500), + metadata: metadata + ) + end + + # Log a failed SSO login attempt + def log_login_failed!(provider:, request:, reason:, metadata: {}) + create!( + user: nil, + event_type: "login_failed", + provider: provider, + ip_address: request.remote_ip, + user_agent: request.user_agent&.truncate(500), + metadata: metadata.merge(reason: reason) + ) + end + + # Log a logout (local only) + def log_logout!(user:, request:, metadata: {}) + create!( + user: user, + event_type: "logout", + provider: nil, + ip_address: request.remote_ip, + user_agent: request.user_agent&.truncate(500), + metadata: metadata + ) + end + + # Log a federated logout (to IdP) + def log_logout_idp!(user:, provider:, request:, metadata: {}) + create!( + user: user, + event_type: "logout_idp", + provider: provider, + ip_address: request.remote_ip, + user_agent: request.user_agent&.truncate(500), + metadata: metadata + ) + end + + # Log an account link (existing user links SSO identity) + def log_link!(user:, provider:, request:, metadata: {}) + create!( + user: user, + event_type: "link", + provider: provider, + ip_address: request.remote_ip, + user_agent: request.user_agent&.truncate(500), + metadata: metadata + ) + end + + # Log an account unlink (user disconnects SSO identity) + def log_unlink!(user:, provider:, request:, metadata: {}) + create!( + user: user, + event_type: "unlink", + provider: provider, + ip_address: request.remote_ip, + user_agent: request.user_agent&.truncate(500), + metadata: metadata + ) + end + + # Log JIT account creation via SSO + def log_jit_account_created!(user:, provider:, request:, metadata: {}) + create!( + user: user, + event_type: "jit_account_created", + provider: provider, + ip_address: request.remote_ip, + user_agent: request.user_agent&.truncate(500), + metadata: metadata + ) + end + end +end diff --git a/app/models/sso_provider.rb b/app/models/sso_provider.rb new file mode 100644 index 000000000..21e3dbf33 --- /dev/null +++ b/app/models/sso_provider.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +class SsoProvider < ApplicationRecord + # Encrypt sensitive credentials using Rails 7.2 built-in encryption + encrypts :client_secret, deterministic: false + + # Default enabled to true for new providers + attribute :enabled, :boolean, default: true + + # Validations + validates :strategy, presence: true, inclusion: { + in: %w[openid_connect google_oauth2 github saml], + message: "%{value} is not a supported strategy" + } + validates :name, presence: true, uniqueness: true, format: { + with: /\A[a-z0-9_]+\z/, + message: "must contain only lowercase letters, numbers, and underscores" + } + validates :label, presence: true + validates :enabled, inclusion: { in: [ true, false ] } + + # Strategy-specific validations + validate :validate_oidc_fields, if: -> { strategy == "openid_connect" } + validate :validate_oauth_fields, if: -> { strategy.in?(%w[google_oauth2 github]) } + validate :validate_saml_fields, if: -> { strategy == "saml" } + validate :validate_default_role_setting + # Note: OIDC discovery validation is done client-side via Stimulus + # Server-side validation can fail due to network issues, so we skip it + # validate :validate_oidc_discovery, if: -> { strategy == "openid_connect" && issuer.present? && will_save_change_to_issuer? } + + # Scopes + scope :enabled, -> { where(enabled: true) } + scope :by_strategy, ->(strategy) { where(strategy: strategy) } + + # Convert to hash format compatible with OmniAuth initializer + def to_omniauth_config + { + id: name, + strategy: strategy, + name: name, + label: label, + icon: icon, + issuer: issuer, + client_id: client_id, + client_secret: client_secret, + redirect_uri: redirect_uri, + settings: settings || {} + }.compact + end + + private + def validate_oidc_fields + if issuer.blank? + errors.add(:issuer, "is required for OpenID Connect providers") + elsif issuer.present? && !valid_url?(issuer) + errors.add(:issuer, "must be a valid URL") + end + + errors.add(:client_id, "is required for OpenID Connect providers") if client_id.blank? + errors.add(:client_secret, "is required for OpenID Connect providers") if client_secret.blank? + + if redirect_uri.present? && !valid_url?(redirect_uri) + errors.add(:redirect_uri, "must be a valid URL") + end + end + + def validate_oauth_fields + errors.add(:client_id, "is required for OAuth providers") if client_id.blank? + errors.add(:client_secret, "is required for OAuth providers") if client_secret.blank? + end + + def validate_saml_fields + # SAML requires either a metadata URL or manual configuration + idp_metadata_url = settings&.dig("idp_metadata_url") + idp_sso_url = settings&.dig("idp_sso_url") + + if idp_metadata_url.blank? && idp_sso_url.blank? + errors.add(:settings, "Either IdP Metadata URL or IdP SSO URL is required for SAML providers") + end + + # If using manual config, require certificate + if idp_metadata_url.blank? && idp_sso_url.present? + idp_cert = settings&.dig("idp_certificate") + idp_fingerprint = settings&.dig("idp_cert_fingerprint") + + if idp_cert.blank? && idp_fingerprint.blank? + errors.add(:settings, "Either IdP Certificate or Certificate Fingerprint is required when not using metadata URL") + end + end + + # Validate URL formats if provided + if idp_metadata_url.present? && !valid_url?(idp_metadata_url) + errors.add(:settings, "IdP Metadata URL must be a valid URL") + end + + if idp_sso_url.present? && !valid_url?(idp_sso_url) + errors.add(:settings, "IdP SSO URL must be a valid URL") + end + end + + def validate_default_role_setting + default_role = settings&.dig("default_role") + return if default_role.blank? + + unless User.roles.key?(default_role) + errors.add(:settings, "default_role must be member, admin, or super_admin") + end + end + + def validate_oidc_discovery + return unless issuer.present? + + begin + discovery_url = issuer.end_with?("/") ? "#{issuer}.well-known/openid-configuration" : "#{issuer}/.well-known/openid-configuration" + response = Faraday.get(discovery_url) do |req| + req.options.timeout = 5 + req.options.open_timeout = 3 + end + + unless response.success? + errors.add(:issuer, "discovery endpoint returned #{response.status}") + return + end + + discovery_data = JSON.parse(response.body) + unless discovery_data["issuer"].present? + errors.add(:issuer, "discovery endpoint did not return valid issuer") + end + rescue Faraday::Error => e + errors.add(:issuer, "could not connect to discovery endpoint: #{e.message}") + rescue JSON::ParserError + errors.add(:issuer, "discovery endpoint returned invalid JSON") + rescue StandardError => e + errors.add(:issuer, "discovery validation failed: #{e.message}") + end + end + + def valid_url?(url) + uri = URI.parse(url) + uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS) + rescue URI::InvalidURIError + false + end +end diff --git a/app/models/sso_provider_tester.rb b/app/models/sso_provider_tester.rb new file mode 100644 index 000000000..0464088c4 --- /dev/null +++ b/app/models/sso_provider_tester.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +# Tests SSO provider configuration by validating discovery endpoints +class SsoProviderTester + attr_reader :provider, :result + + Result = Struct.new(:success?, :message, :details, keyword_init: true) + + def initialize(provider) + @provider = provider + @result = nil + end + + def test! + @result = case provider.strategy + when "openid_connect" + test_oidc_discovery + when "google_oauth2" + test_google_oauth + when "github" + test_github_oauth + when "saml" + test_saml_metadata + else + Result.new(success?: false, message: "Unknown strategy: #{provider.strategy}", details: {}) + end + end + + private + + def test_oidc_discovery + return Result.new(success?: false, message: "Issuer URL is required", details: {}) if provider.issuer.blank? + + discovery_url = build_discovery_url(provider.issuer) + + begin + response = Faraday.get(discovery_url) do |req| + req.options.timeout = 10 + req.options.open_timeout = 5 + end + + unless response.success? + return Result.new( + success?: false, + message: "Discovery endpoint returned HTTP #{response.status}", + details: { url: discovery_url, status: response.status } + ) + end + + discovery = JSON.parse(response.body) + + # Validate required OIDC fields + required_fields = %w[issuer authorization_endpoint token_endpoint] + missing = required_fields.select { |f| discovery[f].blank? } + + if missing.any? + return Result.new( + success?: false, + message: "Discovery document missing required fields: #{missing.join(", ")}", + details: { url: discovery_url, missing_fields: missing } + ) + end + + # Check if issuer matches + if discovery["issuer"] != provider.issuer && discovery["issuer"] != provider.issuer.chomp("/") + return Result.new( + success?: false, + message: "Issuer mismatch: expected #{provider.issuer}, got #{discovery["issuer"]}", + details: { expected: provider.issuer, actual: discovery["issuer"] } + ) + end + + Result.new( + success?: true, + message: "OIDC discovery validated successfully", + details: { + issuer: discovery["issuer"], + authorization_endpoint: discovery["authorization_endpoint"], + token_endpoint: discovery["token_endpoint"], + end_session_endpoint: discovery["end_session_endpoint"], + scopes_supported: discovery["scopes_supported"] + } + ) + + rescue Faraday::TimeoutError + Result.new(success?: false, message: "Connection timed out", details: { url: discovery_url }) + rescue Faraday::ConnectionFailed => e + Result.new(success?: false, message: "Connection failed: #{e.message}", details: { url: discovery_url }) + rescue JSON::ParserError + Result.new(success?: false, message: "Invalid JSON response from discovery endpoint", details: { url: discovery_url }) + rescue StandardError => e + Result.new(success?: false, message: "Error: #{e.message}", details: { url: discovery_url }) + end + end + + def test_google_oauth + # Google OAuth doesn't require discovery validation - just check credentials present + if provider.client_id.blank? + return Result.new(success?: false, message: "Client ID is required", details: {}) + end + + if provider.client_secret.blank? + return Result.new(success?: false, message: "Client Secret is required", details: {}) + end + + Result.new( + success?: true, + message: "Google OAuth2 configuration looks valid", + details: { + note: "Full validation occurs during actual authentication" + } + ) + end + + def test_github_oauth + # GitHub OAuth doesn't require discovery validation - just check credentials present + if provider.client_id.blank? + return Result.new(success?: false, message: "Client ID is required", details: {}) + end + + if provider.client_secret.blank? + return Result.new(success?: false, message: "Client Secret is required", details: {}) + end + + Result.new( + success?: true, + message: "GitHub OAuth configuration looks valid", + details: { + note: "Full validation occurs during actual authentication" + } + ) + end + + def test_saml_metadata + # SAML testing - check for IdP metadata or SSO URL + if provider.settings&.dig("idp_metadata_url").blank? && + provider.settings&.dig("idp_sso_url").blank? + return Result.new( + success?: false, + message: "Either IdP Metadata URL or IdP SSO URL is required", + details: {} + ) + end + + # If metadata URL is provided, try to fetch it + metadata_url = provider.settings&.dig("idp_metadata_url") + if metadata_url.present? + begin + response = Faraday.get(metadata_url) do |req| + req.options.timeout = 10 + req.options.open_timeout = 5 + end + + unless response.success? + return Result.new( + success?: false, + message: "Metadata endpoint returned HTTP #{response.status}", + details: { url: metadata_url, status: response.status } + ) + end + + # Basic XML validation + unless response.body.include?("<") && response.body.include?("EntityDescriptor") + return Result.new( + success?: false, + message: "Response does not appear to be valid SAML metadata", + details: { url: metadata_url } + ) + end + + return Result.new( + success?: true, + message: "SAML metadata fetched successfully", + details: { url: metadata_url } + ) + rescue Faraday::TimeoutError + return Result.new(success?: false, message: "Connection timed out", details: { url: metadata_url }) + rescue Faraday::ConnectionFailed => e + return Result.new(success?: false, message: "Connection failed: #{e.message}", details: { url: metadata_url }) + rescue StandardError => e + return Result.new(success?: false, message: "Error: #{e.message}", details: { url: metadata_url }) + end + end + + Result.new( + success?: true, + message: "SAML configuration looks valid", + details: { + note: "Full validation occurs during actual authentication" + } + ) + end + + def build_discovery_url(issuer) + if issuer.end_with?("/") + "#{issuer}.well-known/openid-configuration" + else + "#{issuer}/.well-known/openid-configuration" + end + end +end diff --git a/app/policies/application_policy.rb b/app/policies/application_policy.rb new file mode 100644 index 000000000..e232556f6 --- /dev/null +++ b/app/policies/application_policy.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class ApplicationPolicy + attr_reader :user, :record + + def initialize(user, record) + @user = user + @record = record + end + + def index? + false + end + + def show? + false + end + + def create? + false + end + + def new? + create? + end + + def update? + false + end + + def edit? + update? + end + + def destroy? + false + end + + class Scope + def initialize(user, scope) + @user = user + @scope = scope + end + + def resolve + raise NoMethodError, "You must define #resolve in #{self.class}" + end + + private + + attr_reader :user, :scope + end +end diff --git a/app/policies/sso_provider_policy.rb b/app/policies/sso_provider_policy.rb new file mode 100644 index 000000000..5c975dc66 --- /dev/null +++ b/app/policies/sso_provider_policy.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class SsoProviderPolicy < ApplicationPolicy + # Only super admins can manage SSO providers (instance-wide auth config) + def index? + user&.super_admin? + end + + def show? + user&.super_admin? + end + + def create? + user&.super_admin? + end + + def new? + create? + end + + def update? + user&.super_admin? + end + + def edit? + update? + end + + def destroy? + user&.super_admin? + end + + def toggle? + update? + end + + class Scope < ApplicationPolicy::Scope + def resolve + if user&.super_admin? + scope.all + else + scope.none + end + end + end +end diff --git a/app/policies/user_policy.rb b/app/policies/user_policy.rb new file mode 100644 index 000000000..c40bf6007 --- /dev/null +++ b/app/policies/user_policy.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class UserPolicy < ApplicationPolicy + # Only super_admins can manage user roles + def index? + user&.super_admin? + end + + def update? + return false unless user&.super_admin? + # Prevent users from changing their own role (must be done by another super_admin) + user.id != record.id + end + + class Scope < ApplicationPolicy::Scope + def resolve + if user&.super_admin? + scope.all + else + scope.none + end + end + end +end diff --git a/app/services/provider_loader.rb b/app/services/provider_loader.rb new file mode 100644 index 000000000..e2bf35365 --- /dev/null +++ b/app/services/provider_loader.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# Service class to load SSO provider configurations from either YAML or database +# based on the :db_sso_providers feature flag. +# +# Usage: +# providers = ProviderLoader.load_providers +# +class ProviderLoader + CACHE_KEY = "sso_providers_config" + CACHE_EXPIRES_IN = 5.minutes + + class << self + # Load providers from either DB or YAML based on feature flag + # Returns an array of provider configuration hashes + def load_providers + # Check cache first for performance + cached = Rails.cache.read(CACHE_KEY) + return cached if cached.present? + + providers = if use_database_providers? + load_from_database + else + load_from_yaml + end + + # Cache the result + Rails.cache.write(CACHE_KEY, providers, expires_in: CACHE_EXPIRES_IN) + providers + end + + # Clear the provider cache (call after updating providers in admin) + def clear_cache + Rails.cache.delete(CACHE_KEY) + end + + private + def use_database_providers? + return false if Rails.env.test? + + begin + # Check if feature exists, create if not (defaults to disabled) + unless Flipper.exist?(:db_sso_providers) + Flipper.add(:db_sso_providers) + end + Flipper.enabled?(:db_sso_providers) + rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid, StandardError => e + # Database not ready or other error, fall back to YAML + Rails.logger.warn("[ProviderLoader] Could not check feature flag (#{e.class}), falling back to YAML providers") + false + end + end + + def load_from_database + begin + providers = SsoProvider.enabled.order(:name).map(&:to_omniauth_config) + + if providers.empty? + Rails.logger.info("[ProviderLoader] No enabled providers in database, falling back to YAML") + return load_from_yaml + end + + Rails.logger.info("[ProviderLoader] Loaded #{providers.count} provider(s) from database") + providers + rescue ActiveRecord::StatementInvalid, ActiveRecord::NoDatabaseError => e + Rails.logger.error("[ProviderLoader] Database error loading providers: #{e.message}, falling back to YAML") + load_from_yaml + rescue StandardError => e + Rails.logger.error("[ProviderLoader] Unexpected error loading providers from database: #{e.message}, falling back to YAML") + load_from_yaml + end + end + + def load_from_yaml + begin + auth_config = Rails.application.config_for(:auth) + providers = auth_config.dig("providers") || [] + + Rails.logger.info("[ProviderLoader] Loaded #{providers.count} provider(s) from YAML") + providers + rescue RuntimeError, Errno::ENOENT => e + Rails.logger.error("[ProviderLoader] Error loading auth.yml: #{e.message}") + [] + end + end + end +end diff --git a/app/views/admin/sso_providers/_form.html.erb b/app/views/admin/sso_providers/_form.html.erb new file mode 100644 index 000000000..88de8c6c6 --- /dev/null +++ b/app/views/admin/sso_providers/_form.html.erb @@ -0,0 +1,277 @@ +<%# locals: (sso_provider:) %> + +<% if sso_provider.errors.any? %> +
+
+ <%= icon "alert-circle", class: "w-5 h-5 text-destructive mr-2 shrink-0" %> +
+

+ <%= pluralize(sso_provider.errors.count, "error") %> prohibited this provider from being saved: +

+
    + <% sso_provider.errors.full_messages.each do |message| %> +
  • <%= message %>
    • + <% end %> +
+
+
+
+<% end %> + +<%= styled_form_with model: [:admin, sso_provider], class: "space-y-6", data: { controller: "admin-sso-form" } do |form| %> +
+

Basic Information

+ +
+ <%= form.select :strategy, + options_for_select([ + ["OpenID Connect", "openid_connect"], + ["SAML 2.0", "saml"], + ["Google OAuth2", "google_oauth2"], + ["GitHub", "github"] + ], sso_provider.strategy), + { label: "Strategy" }, + { data: { action: "change->admin-sso-form#toggleFields" } } %> + + <%= form.text_field :name, + label: "Name", + placeholder: "e.g., keycloak, authentik", + required: true, + data: { action: "input->admin-sso-form#updateCallbackUrl" } %> +
+

Unique identifier (lowercase, numbers, underscores only)

+ +
+ <%= form.text_field :label, + label: "Button Label", + placeholder: "e.g., Sign in with Keycloak", + required: true %> + +
+ <%= form.text_field :icon, + label: "Icon (optional)", + placeholder: "e.g., key, shield" %> +

Lucide icon name for the login button

+
+
+ + <%= form.check_box :enabled, + label: "Enable this provider", + checked: sso_provider.enabled? %> +
+ +
+

OAuth/OIDC Configuration

+ +
"> + <%= form.text_field :issuer, + label: "Issuer URL", + placeholder: "https://your-idp.example.com/realms/your-realm", + data: { action: "blur->admin-sso-form#validateIssuer" } %> +

OIDC issuer URL (validates .well-known/openid-configuration)

+
+ + <%= form.text_field :client_id, + label: "Client ID", + placeholder: "your-client-id", + required: true %> + + <%= form.password_field :client_secret, + label: "Client Secret", + placeholder: sso_provider.persisted? ? "••••••••" : "your-client-secret", + required: !sso_provider.persisted? %> + <% if sso_provider.persisted? %> +

Leave blank to keep existing secret

+ <% end %> + +
"> + +
+ <%= "#{request.base_url}/auth/#{sso_provider.name.presence || 'PROVIDER_NAME'}/callback" %> + +
+

Configure this URL in your identity provider

+
+
+ +
"> +

<%= t("admin.sso_providers.form.saml_configuration") %>

+ +
+ + " + class="w-full px-3 py-2 border border-primary rounded-lg text-sm" + placeholder="https://idp.example.com/metadata" + autocomplete="off"> +

<%= t("admin.sso_providers.form.idp_metadata_url_help") %>

+
+ +
+ <%= t("admin.sso_providers.form.manual_saml_config") %> +
+

<%= t("admin.sso_providers.form.manual_saml_help") %>

+ +
+ + " + class="w-full px-3 py-2 border border-primary rounded-lg text-sm" + placeholder="https://idp.example.com/sso" + autocomplete="off"> +
+ +
+ + " + class="w-full px-3 py-2 border border-primary rounded-lg text-sm" + placeholder="https://idp.example.com/slo (optional)" + autocomplete="off"> +
+ +
+ + +

<%= t("admin.sso_providers.form.idp_certificate_help") %>

+
+ +
+ + " + class="w-full px-3 py-2 border border-primary rounded-lg text-sm font-mono" + placeholder="AB:CD:EF:..." + autocomplete="off"> +
+ +
+ + +
+
+
+ +
+ +
+ <%= "#{request.base_url}/auth/#{sso_provider.name.presence || 'PROVIDER_NAME'}/callback" %> + +
+

Configure this URL as the Assertion Consumer Service URL in your IdP

+
+
+ +
+

<%= t("admin.sso_providers.form.provisioning_title") %>

+ + <%= form.select "settings[default_role]", + options_for_select([ + [t("admin.sso_providers.form.role_member"), "member"], + [t("admin.sso_providers.form.role_admin"), "admin"], + [t("admin.sso_providers.form.role_super_admin"), "super_admin"] + ], sso_provider.settings&.dig("default_role") || "member"), + { label: t("admin.sso_providers.form.default_role_label"), include_blank: false } %> +

<%= t("admin.sso_providers.form.default_role_help") %>

+ +
+ <%= t("admin.sso_providers.form.role_mapping_title") %> +
+

<%= t("admin.sso_providers.form.role_mapping_help") %>

+ +
+ + " + class="w-full px-3 py-2 border border-primary rounded-lg text-sm" + placeholder="Platform-Admins, IdP-Superusers" + autocomplete="off"> +

<%= t("admin.sso_providers.form.groups_help") %>

+
+ +
+ + " + class="w-full px-3 py-2 border border-primary rounded-lg text-sm" + placeholder="Team-Leads, Managers" + autocomplete="off"> +
+ +
+ + " + class="w-full px-3 py-2 border border-primary rounded-lg text-sm" + placeholder="* (all groups)" + autocomplete="off"> +
+
+
+
+ +
"> +

<%= t("admin.sso_providers.form.advanced_title") %>

+ +
+ <%= form.text_field "settings[scopes]", + label: t("admin.sso_providers.form.scopes_label"), + value: sso_provider.settings&.dig("scopes"), + placeholder: "openid email profile groups" %> +

<%= t("admin.sso_providers.form.scopes_help") %>

+
+ + <%= form.select "settings[prompt]", + options_for_select([ + [t("admin.sso_providers.form.prompt_default"), ""], + [t("admin.sso_providers.form.prompt_login"), "login"], + [t("admin.sso_providers.form.prompt_consent"), "consent"], + [t("admin.sso_providers.form.prompt_select_account"), "select_account"], + [t("admin.sso_providers.form.prompt_none"), "none"] + ], sso_provider.settings&.dig("prompt")), + { label: t("admin.sso_providers.form.prompt_label"), include_blank: false } %> +

<%= t("admin.sso_providers.form.prompt_help") %>

+
+ +
+
+ <% if sso_provider.persisted? %> + + + <% end %> +
+ +
+ <%= link_to "Cancel", admin_sso_providers_path, class: "px-4 py-2 text-sm font-medium text-secondary hover:text-primary" %> + <%= form.submit sso_provider.persisted? ? "Update Provider" : "Create Provider", + class: "px-4 py-2 bg-primary text-inverse rounded-lg text-sm font-medium hover:bg-primary/90" %> +
+
+<% end %> diff --git a/app/views/admin/sso_providers/edit.html.erb b/app/views/admin/sso_providers/edit.html.erb new file mode 100644 index 000000000..7fc531668 --- /dev/null +++ b/app/views/admin/sso_providers/edit.html.erb @@ -0,0 +1,9 @@ +<%= content_for :page_title, "Edit #{@sso_provider.label}" %> + +
+

Update configuration for <%= @sso_provider.label %>.

+ + <%= settings_section title: "Provider Configuration" do %> + <%= render "form", sso_provider: @sso_provider %> + <% end %> +
diff --git a/app/views/admin/sso_providers/index.html.erb b/app/views/admin/sso_providers/index.html.erb new file mode 100644 index 000000000..b99272db6 --- /dev/null +++ b/app/views/admin/sso_providers/index.html.erb @@ -0,0 +1,88 @@ +<%= content_for :page_title, "SSO Providers" %> + +
+

+ Manage single sign-on authentication providers for your instance. + <% unless Flipper.enabled?(:db_sso_providers) %> + Changes require a server restart to take effect. + <% end %> +

+ + <%= settings_section title: "Configured Providers" do %> + <% if @sso_providers.any? %> +
+ <% @sso_providers.each do |provider| %> +
+
+ <% if provider.icon.present? %> + <%= icon provider.icon, class: "w-5 h-5 text-secondary" %> + <% else %> + <%= icon "key", class: "w-5 h-5 text-secondary" %> + <% end %> +
+

<%= provider.label %>

+

<%= provider.strategy.titleize %> · <%= provider.name %>

+
+
+
+ <% if provider.enabled? %> + + Enabled + + <% else %> + + Disabled + + <% end %> + <%= link_to edit_admin_sso_provider_path(provider), class: "p-1 text-secondary hover:text-primary", title: "Edit" do %> + <%= icon "pencil", class: "w-4 h-4" %> + <% end %> + <%= button_to toggle_admin_sso_provider_path(provider), method: :patch, class: "p-1 text-secondary hover:text-primary", title: provider.enabled? ? "Disable" : "Enable", form: { data: { turbo_confirm: "Are you sure you want to #{provider.enabled? ? 'disable' : 'enable'} this provider?" } } do %> + <%= icon provider.enabled? ? "toggle-right" : "toggle-left", class: "w-4 h-4" %> + <% end %> + <%= button_to admin_sso_provider_path(provider), method: :delete, class: "p-1 text-destructive hover:text-destructive", title: "Delete", form: { data: { turbo_confirm: "Are you sure you want to delete this provider? This action cannot be undone." } } do %> + <%= icon "trash-2", class: "w-4 h-4" %> + <% end %> +
+
+ <% end %> +
+ <% else %> +
+ <%= icon "key", class: "w-12 h-12 mx-auto text-secondary mb-3" %> +

No SSO providers configured yet.

+
+ <% end %> + +
+ <%= link_to new_admin_sso_provider_path, class: "inline-flex items-center gap-2 text-sm font-medium text-primary hover:text-secondary" do %> + <%= icon "plus", class: "w-4 h-4" %> + Add Provider + <% end %> +
+ <% end %> + + <%= settings_section title: "Configuration Mode", collapsible: true, open: false do %> +
+
+
+

Database-backed providers

+

Load providers from database instead of YAML config

+
+ <% if Flipper.enabled?(:db_sso_providers) %> + + Enabled + + <% else %> + + Disabled + + <% end %> +
+

+ Set AUTH_PROVIDERS_SOURCE=db to enable database-backed providers. + This allows changes without server restarts. +

+
+ <% end %> +
diff --git a/app/views/admin/sso_providers/new.html.erb b/app/views/admin/sso_providers/new.html.erb new file mode 100644 index 000000000..20be829c8 --- /dev/null +++ b/app/views/admin/sso_providers/new.html.erb @@ -0,0 +1,9 @@ +<%= content_for :page_title, "Add SSO Provider" %> + +
+

Configure a new single sign-on authentication provider.

+ + <%= settings_section title: "Provider Configuration" do %> + <%= render "form", sso_provider: @sso_provider %> + <% end %> +
diff --git a/app/views/admin/users/index.html.erb b/app/views/admin/users/index.html.erb new file mode 100644 index 000000000..551cd4d10 --- /dev/null +++ b/app/views/admin/users/index.html.erb @@ -0,0 +1,73 @@ +<%= content_for :page_title, t(".title") %> + +
+

<%= t(".description") %>

+ + <%= settings_section title: t(".section_title") do %> +
+ <% @users.each do |user| %> +
+
+
+ <%= user.initials %> +
+
+

<%= user.display_name %>

+

<%= user.email %>

+
+
+
+ <% if user.id == Current.user.id %> + <%= t(".you") %> + + <%= t(".roles.#{user.role}") %> + + <% else %> + <%= form_with model: [:admin, user], method: :patch, class: "flex items-center gap-2" do |form| %> + <%= form.select :role, + options_for_select([ + [t(".roles.member"), "member"], + [t(".roles.admin"), "admin"], + [t(".roles.super_admin"), "super_admin"] + ], user.role), + {}, + class: "text-sm rounded-lg border-primary bg-container text-primary px-2 py-1", + onchange: "this.form.requestSubmit()" %> + <% end %> + <% end %> +
+
+ <% end %> +
+ + <% if @users.empty? %> +
+ <%= icon "users", class: "w-12 h-12 mx-auto text-secondary mb-3" %> +

<%= t(".no_users") %>

+
+ <% end %> + <% end %> + + <%= settings_section title: t(".role_descriptions_title"), collapsible: true, open: false do %> +
+
+ + <%= t(".roles.member") %> + +

<%= t(".role_descriptions.member") %>

+
+
+ + <%= t(".roles.admin") %> + +

<%= t(".role_descriptions.admin") %>

+
+
+ + <%= t(".roles.super_admin") %> + +

<%= t(".role_descriptions.super_admin") %>

+
+
+ <% end %> +
diff --git a/app/views/settings/_settings_nav.html.erb b/app/views/settings/_settings_nav.html.erb index e34f64f2d..89a1a73c2 100644 --- a/app/views/settings/_settings_nav.html.erb +++ b/app/views/settings/_settings_nav.html.erb @@ -30,7 +30,9 @@ nav_sections = [ { label: t(".api_keys_label"), path: settings_api_key_path, icon: "key" }, { label: t(".self_hosting_label"), path: settings_hosting_path, icon: "database", if: self_hosted? }, { label: "Providers", path: settings_providers_path, icon: "plug" }, - { label: t(".imports_label"), path: imports_path, icon: "download" } + { label: t(".imports_label"), path: imports_path, icon: "download" }, + { label: "SSO Providers", path: admin_sso_providers_path, icon: "key-round", if: Current.user&.super_admin? }, + { label: "Users", path: admin_users_path, icon: "users", if: Current.user&.super_admin? } ] } : nil ), diff --git a/app/views/settings/sso_identities/show.html.erb b/app/views/settings/sso_identities/show.html.erb new file mode 100644 index 000000000..b9046425e --- /dev/null +++ b/app/views/settings/sso_identities/show.html.erb @@ -0,0 +1,59 @@ +<%= content_for :page_title, t(".page_title") %> + +<%= settings_section title: t(".identities_title"), subtitle: t(".identities_subtitle") do %> + <% if @oidc_identities.any? %> +
+ <% @oidc_identities.each do |identity| %> +
+
+
+ <%= icon identity.provider_config&.dig(:icon) || "key", class: "w-5 h-5 text-secondary" %> +
+
+

<%= identity.provider_config&.dig(:label) || identity.provider.titleize %>

+

<%= identity.info&.dig("email") || t(".no_email") %>

+

+ <%= t(".last_used") %>: + <%= identity.last_authenticated_at&.to_fs(:short) || t(".never") %> +

+
+
+ <% if @oidc_identities.count > 1 || Current.user.password_digest.present? %> + <%= render DS::Button.new( + text: t(".disconnect"), + variant: "outline", + size: "sm", + href: settings_sso_identity_path(identity), + method: :delete, + confirm: CustomConfirm.new( + title: t(".confirm_title"), + body: t(".confirm_body", provider: identity.provider_config&.dig(:label) || identity.provider.titleize), + btn_text: t(".confirm_button"), + destructive: true + ) + ) %> + <% end %> +
+ <% end %> +
+ <% else %> +
+ <%= icon "link", class: "w-12 h-12 mx-auto text-secondary mb-3" %> +

<%= t(".no_identities") %>

+ <% if AuthConfig.sso_providers.any? %> +

<%= t(".connect_hint") %>

+ <% end %> +
+ <% end %> +<% end %> + +<% if @oidc_identities.count == 1 && Current.user.password_digest.blank? %> + <%= settings_section title: t(".warning_title") do %> +
+
+ <%= icon "alert-triangle", class: "w-5 h-5 text-amber-600 shrink-0 mt-0.5" %> +

<%= t(".warning_message") %>

+
+
+ <% end %> +<% end %> diff --git a/config/application.rb b/config/application.rb index 8a7c2f7be..d0ef1361f 100644 --- a/config/application.rb +++ b/config/application.rb @@ -41,5 +41,9 @@ module Sure # Enable Rack::Attack middleware for API rate limiting config.middleware.use Rack::Attack + + # Handle OmniAuth/OIDC errors gracefully (must be before OmniAuth middleware) + require_relative "../app/middleware/omniauth_error_handler" + config.middleware.use OmniauthErrorHandler end end diff --git a/config/auth.yml b/config/auth.yml index 1e237cca2..ebcbc6ea0 100644 --- a/config/auth.yml +++ b/config/auth.yml @@ -23,11 +23,25 @@ default: &default # Generic OpenID Connect provider (e.g., Keycloak, Authentik, other OIDC issuers). # This maps to the existing :openid_connect OmniAuth strategy and keeps # backwards-compatible behavior for self-hosted setups using OIDC_* env vars. + # + # For the default OIDC provider, use these ENV vars: + # OIDC_ISSUER, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_REDIRECT_URI + # + # To add additional OIDC providers, add more entries with unique names and use + # provider-specific ENV vars with the pattern: OIDC__* + # Example for a provider named "keycloak": + # OIDC_KEYCLOAK_ISSUER, OIDC_KEYCLOAK_CLIENT_ID, + # OIDC_KEYCLOAK_CLIENT_SECRET, OIDC_KEYCLOAK_REDIRECT_URI - id: "oidc" strategy: "openid_connect" name: "openid_connect" label: <%= ENV.fetch("OIDC_BUTTON_LABEL", "Sign in with OpenID Connect") %> icon: <%= ENV.fetch("OIDC_BUTTON_ICON", "key") %> + # Per-provider credentials (optional, falls back to global OIDC_* vars) + issuer: <%= ENV["OIDC_ISSUER"] %> + client_id: <%= ENV["OIDC_CLIENT_ID"] %> + client_secret: <%= ENV["OIDC_CLIENT_SECRET"] %> + redirect_uri: <%= ENV["OIDC_REDIRECT_URI"] %> # Optional Google OAuth provider. Requires the omniauth-google-oauth2 gem # and GOOGLE_OAUTH_CLIENT_ID / GOOGLE_OAUTH_CLIENT_SECRET env vars. diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 05172fd20..0cb225287 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -69,6 +69,29 @@ ], "note": "" }, + { + "warning_type": "Mass Assignment", + "warning_code": 105, + "fingerprint": "01a88a0a17848e70999c17f6438a636b00e01da39a2c0aa0c46f20f0685c7202", + "check_name": "PermitAttributes", + "message": "Potentially dangerous key allowed for mass assignment", + "file": "app/controllers/admin/users_controller.rb", + "line": 35, + "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", + "code": "params.require(:user).permit(:role)", + "render_path": null, + "location": { + "type": "method", + "class": "Admin::UsersController", + "method": "user_params" + }, + "user_input": ":role", + "confidence": "Medium", + "cwe_id": [ + 915 + ], + "note": "Protected by Pundit authorization - UserPolicy requires super_admin and prevents users from changing their own role" + }, { "warning_type": "Dangerous Eval", "warning_code": 13, diff --git a/config/initializers/flipper.rb b/config/initializers/flipper.rb new file mode 100644 index 000000000..6ed3abe4a --- /dev/null +++ b/config/initializers/flipper.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require "flipper" +require "flipper/adapters/active_record" +require "flipper/adapters/memory" + +# Configure Flipper with ActiveRecord adapter for database-backed feature flags +# Falls back to memory adapter if tables don't exist yet (during migrations) +Flipper.configure do |config| + config.adapter do + begin + Flipper::Adapters::ActiveRecord.new + rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid, NameError + # Tables don't exist yet, use memory adapter as fallback + Flipper::Adapters::Memory.new + end + end +end + +# Initialize feature flags IMMEDIATELY (not in after_initialize) +# This must happen before OmniAuth initializer runs +unless Rails.env.test? + begin + # Feature flag to control SSO provider source (YAML vs DB) + # ENV: AUTH_PROVIDERS_SOURCE=db|yaml + # Default: "db" for self-hosted, "yaml" for managed + auth_source = ENV.fetch("AUTH_PROVIDERS_SOURCE") do + Rails.configuration.app_mode.self_hosted? ? "db" : "yaml" + end.downcase + + # Ensure feature exists before enabling/disabling + Flipper.add(:db_sso_providers) unless Flipper.exist?(:db_sso_providers) + + if auth_source == "db" + Flipper.enable(:db_sso_providers) + else + Flipper.disable(:db_sso_providers) + end + rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid + # Database not ready yet (e.g., during initial setup or migrations) + # This is expected during db:create or initial setup + rescue StandardError => e + Rails.logger.warn("[Flipper] Error initializing feature flags: #{e.message}") + end +end diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index 9b836e436..1b4d301b3 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -5,42 +5,101 @@ require "omniauth/rails_csrf_protection" Rails.configuration.x.auth.oidc_enabled = false Rails.configuration.x.auth.sso_providers ||= [] +# Configure OmniAuth to handle failures gracefully +OmniAuth.config.on_failure = proc do |env| + error = env["omniauth.error"] + error_type = env["omniauth.error.type"] + strategy = env["omniauth.error.strategy"] + + # Log the error for debugging + Rails.logger.error("[OmniAuth] Authentication failed: #{error_type} - #{error&.message}") + + # Redirect to failure handler with error info + message = case error_type + when :discovery_failed, :invalid_credentials + "sso_provider_unavailable" + when :invalid_response + "sso_invalid_response" + else + "sso_failed" + end + + Rack::Response.new([ "302 Moved" ], 302, "Location" => "/auth/failure?message=#{message}&strategy=#{strategy&.name}").finish +end + Rails.application.config.middleware.use OmniAuth::Builder do - (Rails.configuration.x.auth.providers || []).each do |raw_cfg| + # Load providers from either YAML or DB via ProviderLoader + providers = ProviderLoader.load_providers + + providers.each do |raw_cfg| cfg = raw_cfg.deep_symbolize_keys strategy = cfg[:strategy].to_s name = (cfg[:name] || cfg[:id]).to_s case strategy when "openid_connect" - required_env = %w[OIDC_ISSUER OIDC_CLIENT_ID OIDC_CLIENT_SECRET OIDC_REDIRECT_URI] - enabled = Rails.env.test? || required_env.all? { |k| ENV[k].present? } - next unless enabled + # Support per-provider credentials from config or fall back to global ENV vars + issuer = cfg[:issuer].presence || ENV["OIDC_ISSUER"].presence + client_id = cfg[:client_id].presence || ENV["OIDC_CLIENT_ID"].presence + client_secret = cfg[:client_secret].presence || ENV["OIDC_CLIENT_SECRET"].presence + redirect_uri = cfg[:redirect_uri].presence || ENV["OIDC_REDIRECT_URI"].presence - issuer = (ENV["OIDC_ISSUER"].presence || "https://test.example.com").to_s.strip - client_id = ENV["OIDC_CLIENT_ID"].presence || "test_client_id" - client_secret = ENV["OIDC_CLIENT_SECRET"].presence || "test_client_secret" - redirect_uri = ENV["OIDC_REDIRECT_URI"].presence || "http://test.example.com/callback" + # In test environment, use test values if nothing is configured + if Rails.env.test? + issuer ||= "https://test.example.com" + client_id ||= "test_client_id" + client_secret ||= "test_client_secret" + redirect_uri ||= "http://test.example.com/callback" + end - provider :openid_connect, - name: name.to_sym, - scope: %i[openid email profile], - response_type: :code, - issuer: issuer, - discovery: true, - pkce: true, - client_options: { - identifier: client_id, - secret: client_secret, - redirect_uri: redirect_uri - } + # Skip if required fields are missing (except in test) + unless issuer.present? && client_id.present? && client_secret.present? && redirect_uri.present? + Rails.logger.warn("[OmniAuth] Skipping OIDC provider '#{name}' - missing required configuration") + next + end + + # Custom scopes: parse from settings if provided, otherwise use defaults + custom_scopes = cfg.dig(:settings, :scopes).presence + scopes = if custom_scopes.present? + custom_scopes.to_s.split(/\s+/).map(&:to_sym) + else + %i[openid email profile] + end + + # Build provider options + oidc_options = { + name: name.to_sym, + scope: scopes, + response_type: :code, + issuer: issuer.to_s.strip, + discovery: true, + pkce: true, + client_options: { + identifier: client_id, + secret: client_secret, + redirect_uri: redirect_uri + } + } + + # Add prompt parameter if configured + prompt = cfg.dig(:settings, :prompt).presence + oidc_options[:prompt] = prompt if prompt.present? + + provider :openid_connect, oidc_options Rails.configuration.x.auth.oidc_enabled = true - Rails.configuration.x.auth.sso_providers << cfg.merge(name: name) + Rails.configuration.x.auth.sso_providers << cfg.merge(name: name, issuer: issuer) when "google_oauth2" - client_id = ENV["GOOGLE_OAUTH_CLIENT_ID"].presence || (Rails.env.test? ? "test_client_id" : nil) - client_secret = ENV["GOOGLE_OAUTH_CLIENT_SECRET"].presence || (Rails.env.test? ? "test_client_secret" : nil) + client_id = cfg[:client_id].presence || ENV["GOOGLE_OAUTH_CLIENT_ID"].presence + client_secret = cfg[:client_secret].presence || ENV["GOOGLE_OAUTH_CLIENT_SECRET"].presence + + # Test environment fallback + if Rails.env.test? + client_id ||= "test_client_id" + client_secret ||= "test_client_secret" + end + next unless client_id.present? && client_secret.present? provider :google_oauth2, @@ -54,8 +113,15 @@ Rails.application.config.middleware.use OmniAuth::Builder do Rails.configuration.x.auth.sso_providers << cfg.merge(name: name) when "github" - client_id = ENV["GITHUB_CLIENT_ID"].presence || (Rails.env.test? ? "test_client_id" : nil) - client_secret = ENV["GITHUB_CLIENT_SECRET"].presence || (Rails.env.test? ? "test_client_secret" : nil) + client_id = cfg[:client_id].presence || ENV["GITHUB_CLIENT_ID"].presence + client_secret = cfg[:client_secret].presence || ENV["GITHUB_CLIENT_SECRET"].presence + + # Test environment fallback + if Rails.env.test? + client_id ||= "test_client_id" + client_secret ||= "test_client_secret" + end + next unless client_id.present? && client_secret.present? provider :github, @@ -67,10 +133,54 @@ Rails.application.config.middleware.use OmniAuth::Builder do } Rails.configuration.x.auth.sso_providers << cfg.merge(name: name) + + when "saml" + settings = cfg[:settings] || {} + + # Require either metadata URL or manual SSO URL + idp_metadata_url = settings[:idp_metadata_url].presence || settings["idp_metadata_url"].presence + idp_sso_url = settings[:idp_sso_url].presence || settings["idp_sso_url"].presence + + unless idp_metadata_url.present? || idp_sso_url.present? + Rails.logger.warn("[OmniAuth] Skipping SAML provider '#{name}' - missing IdP configuration") + next + end + + # Build SAML options + saml_options = { + name: name.to_sym, + assertion_consumer_service_url: cfg[:redirect_uri].presence || "#{ENV['APP_URL']}/auth/#{name}/callback", + issuer: cfg[:issuer].presence || ENV["APP_URL"], + name_identifier_format: settings[:name_id_format].presence || settings["name_id_format"].presence || + "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress", + attribute_statements: { + email: [ "email", "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress" ], + first_name: [ "first_name", "givenName", "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname" ], + last_name: [ "last_name", "surname", "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname" ], + groups: [ "groups", "http://schemas.microsoft.com/ws/2008/06/identity/claims/groups" ] + } + } + + # Use metadata URL or manual configuration + if idp_metadata_url.present? + saml_options[:idp_metadata_url] = idp_metadata_url + else + saml_options[:idp_sso_service_url] = idp_sso_url + saml_options[:idp_cert] = settings[:idp_certificate].presence || settings["idp_certificate"].presence + saml_options[:idp_cert_fingerprint] = settings[:idp_cert_fingerprint].presence || settings["idp_cert_fingerprint"].presence + end + + # Optional: IdP SLO (Single Logout) URL + idp_slo_url = settings[:idp_slo_url].presence || settings["idp_slo_url"].presence + saml_options[:idp_slo_service_url] = idp_slo_url if idp_slo_url.present? + + provider :saml, saml_options + + Rails.configuration.x.auth.sso_providers << cfg.merge(name: name, strategy: "saml") end end end if Rails.configuration.x.auth.sso_providers.empty? - Rails.logger.warn("No SSO providers enabled; check auth.yml / ENV configuration") + Rails.logger.warn("No SSO providers enabled; check auth.yml / ENV configuration or database providers") end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 3d225e58b..c7b3ac0c2 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -9,6 +9,12 @@ class Rack::Attack request.ip if request.path == "/oauth/token" end + # Throttle admin endpoints to prevent brute-force attacks + # More restrictive than general API limits since admin access is sensitive + throttle("admin/ip", limit: 10, period: 1.minute) do |request| + request.ip if request.path.start_with?("/admin/") + end + # Determine limits based on self-hosted mode self_hosted = Rails.application.config.app_mode.self_hosted? diff --git a/config/locales/views/admin/sso_providers/en.yml b/config/locales/views/admin/sso_providers/en.yml new file mode 100644 index 000000000..c59c26380 --- /dev/null +++ b/config/locales/views/admin/sso_providers/en.yml @@ -0,0 +1,109 @@ +--- +en: + admin: + unauthorized: "You are not authorized to access this area." + sso_providers: + index: + title: "SSO Providers" + description: "Manage single sign-on authentication providers for your instance" + add_provider: "Add Provider" + no_providers_title: "No SSO Providers" + no_providers_message: "Get started by adding your first SSO provider." + note: "Changes to SSO providers require a server restart to take effect. Alternatively, enable the AUTH_PROVIDERS_SOURCE=db feature flag to load providers from the database dynamically." + table: + name: "Name" + strategy: "Strategy" + status: "Status" + issuer: "Issuer" + actions: "Actions" + enabled: "Enabled" + disabled: "Disabled" + new: + title: "Add SSO Provider" + description: "Configure a new single sign-on authentication provider" + edit: + title: "Edit SSO Provider" + description: "Update configuration for %{label}" + create: + success: "SSO provider was successfully created." + update: + success: "SSO provider was successfully updated." + destroy: + success: "SSO provider was successfully deleted." + confirm: "Are you sure you want to delete this provider? This action cannot be undone." + toggle: + success_enabled: "SSO provider was successfully enabled." + success_disabled: "SSO provider was successfully disabled." + confirm_enable: "Are you sure you want to enable this provider?" + confirm_disable: "Are you sure you want to disable this provider?" + form: + basic_information: "Basic Information" + oauth_configuration: "OAuth/OIDC Configuration" + strategy_label: "Strategy" + strategy_help: "The authentication strategy to use" + name_label: "Name" + name_placeholder: "e.g., openid_connect, keycloak, authentik" + name_help: "Unique identifier (lowercase, numbers, underscores only)" + label_label: "Label" + label_placeholder: "e.g., Sign in with Keycloak" + label_help: "Button text shown to users" + icon_label: "Icon" + icon_placeholder: "e.g., key, google, github" + icon_help: "Lucide icon name (optional)" + enabled_label: "Enable this provider" + issuer_label: "Issuer" + issuer_placeholder: "https://accounts.google.com" + issuer_help: "OIDC issuer URL (will validate .well-known/openid-configuration endpoint)" + client_id_label: "Client ID" + client_id_placeholder: "your-client-id" + client_id_help: "OAuth client ID from your identity provider" + client_secret_label: "Client Secret" + client_secret_placeholder_new: "your-client-secret" + client_secret_placeholder_existing: "••••••••••••••••" + client_secret_help: "OAuth client secret (encrypted in database)" + client_secret_help_existing: " - leave blank to keep existing" + redirect_uri_label: "Redirect URI" + redirect_uri_placeholder: "https://yourdomain.com/auth/openid_connect/callback" + redirect_uri_help: "Callback URL to configure in your identity provider" + copy_button: "Copy" + cancel: "Cancel" + submit: "Save Provider" + errors_title: "%{count} error prohibited this provider from being saved:" + provisioning_title: "User Provisioning" + default_role_label: "Default Role for New Users" + default_role_help: "Role assigned to users created via just-in-time (JIT) SSO account provisioning. Defaults to Member." + role_member: "Member" + role_admin: "Admin" + role_super_admin: "Super Admin" + role_mapping_title: "Group to Role Mapping (Optional)" + role_mapping_help: "Map IdP groups/claims to application roles. Users are assigned the highest matching role. Leave blank to use the default role above." + super_admin_groups: "Super Admin Groups" + admin_groups: "Admin Groups" + member_groups: "Member Groups" + groups_help: "Comma-separated list of IdP group names. Use * to match all groups." + advanced_title: "Advanced OIDC Settings" + scopes_label: "Custom Scopes" + scopes_help: "Space-separated list of OIDC scopes. Leave blank for defaults (openid email profile). Add 'groups' to retrieve group claims." + prompt_label: "Authentication Prompt" + prompt_default: "Default (IdP decides)" + prompt_login: "Force Login (re-authenticate)" + prompt_consent: "Force Consent (re-authorize)" + prompt_select_account: "Account Selection (choose account)" + prompt_none: "No Prompt (silent auth)" + prompt_help: "Controls how the IdP prompts the user during authentication." + test_connection: "Test Connection" + saml_configuration: "SAML Configuration" + idp_metadata_url: "IdP Metadata URL" + idp_metadata_url_help: "URL to your IdP's SAML metadata. If provided, other SAML settings will be auto-configured." + manual_saml_config: "Manual Configuration (if not using metadata URL)" + manual_saml_help: "Only use these settings if your IdP doesn't provide a metadata URL." + idp_sso_url: "IdP SSO URL" + idp_slo_url: "IdP SLO URL (optional)" + idp_certificate: "IdP Certificate" + idp_certificate_help: "X.509 certificate in PEM format. Required if not using metadata URL." + idp_cert_fingerprint: "Certificate Fingerprint (alternative)" + name_id_format: "NameID Format" + name_id_email: "Email Address (default)" + name_id_persistent: "Persistent" + name_id_transient: "Transient" + name_id_unspecified: "Unspecified" diff --git a/config/locales/views/admin/users/en.yml b/config/locales/views/admin/users/en.yml new file mode 100644 index 000000000..6e77b7011 --- /dev/null +++ b/config/locales/views/admin/users/en.yml @@ -0,0 +1,22 @@ +--- +en: + admin: + users: + index: + title: "User Management" + description: "Manage user roles for your instance. Super admins can access SSO provider settings and user management." + section_title: "Users" + you: "(You)" + no_users: "No users found." + role_descriptions_title: "Role Descriptions" + roles: + member: "Member" + admin: "Admin" + super_admin: "Super Admin" + role_descriptions: + member: "Basic user access. Can manage their own accounts, transactions, and settings." + admin: "Family administrator. Can access advanced settings like API keys, imports, and AI prompts." + super_admin: "Instance administrator. Can manage SSO providers, user roles, and impersonate users for support." + update: + success: "User role updated successfully." + failure: "Failed to update user role." diff --git a/config/locales/views/sessions/en.yml b/config/locales/views/sessions/en.yml index 8891e194d..6dfd39de4 100644 --- a/config/locales/views/sessions/en.yml +++ b/config/locales/views/sessions/en.yml @@ -10,6 +10,9 @@ en: failed: Could not authenticate via OpenID Connect. failure: failed: Could not authenticate. + sso_provider_unavailable: "The SSO provider is currently unavailable. Please try again later or contact an administrator." + sso_invalid_response: "Received an invalid response from the SSO provider. Please try again." + sso_failed: "Single sign-on authentication failed. Please try again." new: email: Email address email_placeholder: you@example.com diff --git a/config/locales/views/settings/sso_identities/en.yml b/config/locales/views/settings/sso_identities/en.yml new file mode 100644 index 000000000..c989ee974 --- /dev/null +++ b/config/locales/views/settings/sso_identities/en.yml @@ -0,0 +1,22 @@ +--- +en: + settings: + sso_identities: + show: + page_title: "Connected Accounts" + identities_title: "SSO Connections" + identities_subtitle: "Manage your single sign-on account connections" + disconnect: "Disconnect" + last_used: "Last used" + never: "Never" + no_email: "No email" + no_identities: "No SSO accounts connected" + connect_hint: "Log out and sign in with an SSO provider to connect an account." + confirm_title: "Disconnect Account?" + confirm_body: "Are you sure you want to disconnect your %{provider} account? You can reconnect it later by signing in with that provider again." + confirm_button: "Disconnect" + warning_title: "Important" + warning_message: "This is your only login method. You should set a password in your security settings before disconnecting, otherwise you may be locked out of your account." + destroy: + success: "Successfully disconnected %{provider}" + cannot_unlink_last: "Cannot disconnect your only login method. Please set a password first." diff --git a/config/routes.rb b/config/routes.rb index d6dd270e1..9d0e8fa4a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -54,9 +54,10 @@ Rails.application.routes.draw do resource :current_session, only: %i[update] resource :registration, only: %i[new create] - resources :sessions, only: %i[new create destroy] + resources :sessions, only: %i[index new create destroy] match "/auth/:provider/callback", to: "sessions#openid_connect", via: %i[get post] match "/auth/failure", to: "sessions#failure", via: %i[get post] + get "/auth/logout/callback", to: "sessions#post_logout" resource :oidc_account, only: [] do get :link, on: :collection post :create_link, on: :collection @@ -90,6 +91,8 @@ Rails.application.routes.draw do end resource :billing, only: :show resource :security, only: :show + resource :sso_identities, only: :show + resources :sso_identities, only: :destroy resource :api_key, only: [ :show, :new, :create, :destroy ] resource :ai_prompts, only: :show resource :llm_usage, only: :show @@ -368,6 +371,17 @@ Rails.application.routes.draw do get "privacy", to: redirect("about:blank") get "terms", to: redirect("about:blank") + # Admin namespace for super admin functionality + namespace :admin do + resources :sso_providers do + member do + patch :toggle + post :test_connection + end + end + resources :users, only: [ :index, :update ] + end + # Defines the root path route ("/") root "pages#dashboard" end diff --git a/db/migrate/20251228181150_create_flipper_tables.rb b/db/migrate/20251228181150_create_flipper_tables.rb new file mode 100644 index 000000000..811f528cf --- /dev/null +++ b/db/migrate/20251228181150_create_flipper_tables.rb @@ -0,0 +1,22 @@ +class CreateFlipperTables < ActiveRecord::Migration[7.2] + def up + create_table :flipper_features do |t| + t.string :key, null: false + t.timestamps null: false + end + add_index :flipper_features, :key, unique: true + + create_table :flipper_gates do |t| + t.string :feature_key, null: false + t.string :key, null: false + t.text :value + t.timestamps null: false + end + add_index :flipper_gates, [ :feature_key, :key, :value ], unique: true, length: { value: 255 } + end + + def down + drop_table :flipper_gates + drop_table :flipper_features + end +end diff --git a/db/migrate/20251228181429_create_sso_providers.rb b/db/migrate/20251228181429_create_sso_providers.rb new file mode 100644 index 000000000..5278af4e4 --- /dev/null +++ b/db/migrate/20251228181429_create_sso_providers.rb @@ -0,0 +1,21 @@ +class CreateSsoProviders < ActiveRecord::Migration[7.2] + def change + create_table :sso_providers, id: :uuid do |t| + t.string :strategy, null: false + t.string :name, null: false + t.string :label, null: false + t.string :icon + t.boolean :enabled, null: false, default: true + t.string :issuer + t.string :client_id + t.string :client_secret + t.string :redirect_uri + t.jsonb :settings, null: false, default: {} + + t.timestamps + end + + add_index :sso_providers, :name, unique: true + add_index :sso_providers, :enabled + end +end diff --git a/db/migrate/20251228182113_add_issuer_to_oidc_identities.rb b/db/migrate/20251228182113_add_issuer_to_oidc_identities.rb new file mode 100644 index 000000000..5922cd3fc --- /dev/null +++ b/db/migrate/20251228182113_add_issuer_to_oidc_identities.rb @@ -0,0 +1,6 @@ +class AddIssuerToOidcIdentities < ActiveRecord::Migration[7.2] + def change + add_column :oidc_identities, :issuer, :string + add_index :oidc_identities, :issuer + end +end diff --git a/db/migrate/20260103170412_create_sso_audit_logs.rb b/db/migrate/20260103170412_create_sso_audit_logs.rb new file mode 100644 index 000000000..f28ce14e9 --- /dev/null +++ b/db/migrate/20260103170412_create_sso_audit_logs.rb @@ -0,0 +1,18 @@ +class CreateSsoAuditLogs < ActiveRecord::Migration[7.2] + def change + create_table :sso_audit_logs, id: :uuid do |t| + t.references :user, type: :uuid, foreign_key: true, null: true + t.string :event_type, null: false + t.string :provider + t.string :ip_address + t.string :user_agent + t.jsonb :metadata, null: false, default: {} + + t.timestamps + end + + add_index :sso_audit_logs, :event_type + add_index :sso_audit_logs, :created_at + add_index :sso_audit_logs, [ :user_id, :created_at ] + end +end diff --git a/db/schema.rb b/db/schema.rb index 27adfde1e..1ff598f3c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_12_15_100443) do +ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -422,6 +422,22 @@ ActiveRecord::Schema[7.2].define(version: 2025_12_15_100443) do t.index ["family_id"], name: "index_family_exports_on_family_id" end + create_table "flipper_features", force: :cascade do |t| + t.string "key", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["key"], name: "index_flipper_features_on_key", unique: true + end + + create_table "flipper_gates", force: :cascade do |t| + t.string "feature_key", null: false + t.string "key", null: false + t.text "value" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["feature_key", "key", "value"], name: "index_flipper_gates_on_feature_key_and_key_and_value", unique: true + end + create_table "holdings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "account_id", null: false t.uuid "security_id", null: false @@ -741,6 +757,8 @@ ActiveRecord::Schema[7.2].define(version: 2025_12_15_100443) do t.datetime "last_authenticated_at" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "issuer" + t.index ["issuer"], name: "index_oidc_identities_on_issuer" t.index ["provider", "uid"], name: "index_oidc_identities_on_provider_and_uid", unique: true t.index ["user_id"], name: "index_oidc_identities_on_user_id" end @@ -994,6 +1012,38 @@ ActiveRecord::Schema[7.2].define(version: 2025_12_15_100443) do t.index ["status"], name: "index_simplefin_items_on_status" end + create_table "sso_audit_logs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "user_id" + t.string "event_type", null: false + t.string "provider" + t.string "ip_address" + t.string "user_agent" + t.jsonb "metadata", default: {}, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["created_at"], name: "index_sso_audit_logs_on_created_at" + t.index ["event_type"], name: "index_sso_audit_logs_on_event_type" + t.index ["user_id", "created_at"], name: "index_sso_audit_logs_on_user_id_and_created_at" + t.index ["user_id"], name: "index_sso_audit_logs_on_user_id" + end + + create_table "sso_providers", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "strategy", null: false + t.string "name", null: false + t.string "label", null: false + t.string "icon" + t.boolean "enabled", default: true, null: false + t.string "issuer" + t.string "client_id" + t.string "client_secret" + t.string "redirect_uri" + t.jsonb "settings", default: {}, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["enabled"], name: "index_sso_providers_on_enabled" + t.index ["name"], name: "index_sso_providers_on_name", unique: true + end + create_table "subscriptions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "family_id", null: false t.string "status", null: false @@ -1215,6 +1265,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_12_15_100443) do add_foreign_key "sessions", "users" add_foreign_key "simplefin_accounts", "simplefin_items" add_foreign_key "simplefin_items", "families" + add_foreign_key "sso_audit_logs", "users" add_foreign_key "subscriptions", "families" add_foreign_key "syncs", "syncs", column: "parent_id" add_foreign_key "taggings", "tags" diff --git a/docs/hosting/oidc.md b/docs/hosting/oidc.md index 1db92e8ed..04be10bed 100644 --- a/docs/hosting/oidc.md +++ b/docs/hosting/oidc.md @@ -250,3 +250,238 @@ With these settings, you can run Sure in: - Domain‑restricted and link‑only enterprise SSO modes Use the combination that best fits your self‑hosted environment and security posture. + +--- + +## 5. Multiple OIDC Providers + +Sure supports configuring multiple OIDC providers simultaneously, allowing users to choose between different identity providers (e.g., Keycloak, Authentik, Okta) on the login page. + +### 5.1 YAML-based multi-provider configuration + +To add multiple OIDC providers in `config/auth.yml`, add additional provider entries with unique names: + +```yaml +providers: + # First OIDC provider (e.g., Keycloak) + - id: "keycloak" + strategy: "openid_connect" + name: "keycloak" + label: "Sign in with Keycloak" + icon: "key" + issuer: <%= ENV["OIDC_KEYCLOAK_ISSUER"] %> + client_id: <%= ENV["OIDC_KEYCLOAK_CLIENT_ID"] %> + client_secret: <%= ENV["OIDC_KEYCLOAK_CLIENT_SECRET"] %> + redirect_uri: <%= ENV["OIDC_KEYCLOAK_REDIRECT_URI"] %> + + # Second OIDC provider (e.g., Authentik) + - id: "authentik" + strategy: "openid_connect" + name: "authentik" + label: "Sign in with Authentik" + icon: "shield" + issuer: <%= ENV["OIDC_AUTHENTIK_ISSUER"] %> + client_id: <%= ENV["OIDC_AUTHENTIK_CLIENT_ID"] %> + client_secret: <%= ENV["OIDC_AUTHENTIK_CLIENT_SECRET"] %> + redirect_uri: <%= ENV["OIDC_AUTHENTIK_REDIRECT_URI"] %> +``` + +Set the corresponding environment variables: + +```bash +# Keycloak provider +OIDC_KEYCLOAK_ISSUER="https://keycloak.example.com/realms/myrealm" +OIDC_KEYCLOAK_CLIENT_ID="sure-client" +OIDC_KEYCLOAK_CLIENT_SECRET="your-keycloak-secret" +OIDC_KEYCLOAK_REDIRECT_URI="https://yourdomain.com/auth/keycloak/callback" + +# Authentik provider +OIDC_AUTHENTIK_ISSUER="https://authentik.example.com/application/o/sure/" +OIDC_AUTHENTIK_CLIENT_ID="sure-authentik-client" +OIDC_AUTHENTIK_CLIENT_SECRET="your-authentik-secret" +OIDC_AUTHENTIK_REDIRECT_URI="https://yourdomain.com/auth/authentik/callback" +``` + +**Important:** Each provider must have a unique `name` field, which determines the callback URL path (`/auth//callback`). + +--- + +## 6. Database-Backed Provider Management + +For more dynamic provider management, Sure supports storing SSO provider configurations in the database with a web-based admin interface. + +### 6.1 Enabling database providers + +Set the feature flag to load providers from the database instead of YAML: + +```bash +AUTH_PROVIDERS_SOURCE=db +``` + +When enabled: +- Providers are loaded from the `sso_providers` database table +- Changes take effect immediately (no server restart required) +- Providers can be managed through the admin UI at `/admin/sso_providers` + +When disabled (default): +- Providers are loaded from `config/auth.yml` +- Changes require a server restart + +### 6.2 Admin UI for SSO providers + +Super-admin users can manage SSO providers through the web interface: + +1. Navigate to `/admin/sso_providers` +2. View all configured providers (enabled/disabled status) +3. Add new providers with the "Add Provider" button +4. Edit existing providers (credentials, labels, icons) +5. Enable/disable providers with the toggle button +6. Delete providers (with confirmation) + +**Security notes:** +- Only users with `super_admin` role can access the admin interface +- All provider changes are logged with user ID and timestamp +- Client secrets are encrypted in the database using Rails 7.2 encryption +- Admin endpoints are rate-limited (10 requests/minute per IP) + +### 6.3 Seeding providers from YAML to database + +To migrate your existing YAML configuration to the database: + +```bash +# Dry run (preview changes without saving) +DRY_RUN=true rails sso_providers:seed + +# Apply changes +rails sso_providers:seed +``` + +The seeding task: +- Reads providers from `config/auth.yml` +- Creates or updates database records (idempotent) +- Preserves existing client secrets if not provided in YAML +- Provides detailed output (created/updated/skipped/errors) + +To list all providers in the database: + +```bash +rails sso_providers:list +``` + +### 6.4 Migration workflow + +Recommended steps to migrate from YAML to database-backed providers: + +1. **Backup your configuration:** + ```bash + cp config/auth.yml config/auth.yml.backup + ``` + +2. **Run migrations:** + ```bash + rails db:migrate + ``` + +3. **Seed providers from YAML (dry run first):** + ```bash + DRY_RUN=true rails sso_providers:seed + ``` + +4. **Review the output, then apply:** + ```bash + rails sso_providers:seed + ``` + +5. **Enable database provider source:** + ```bash + # Add to .env or environment + AUTH_PROVIDERS_SOURCE=db + ``` + +6. **Restart the application:** + ```bash + # Docker Compose + docker-compose restart app + + # Or your process manager + systemctl restart sure + ``` + +7. **Verify providers are loaded:** + - Check logs for `[ProviderLoader] Loaded N provider(s) from database` + - Visit `/admin/sso_providers` to manage providers + +### 6.5 Rollback to YAML + +To switch back to YAML-based configuration: + +1. Remove or set `AUTH_PROVIDERS_SOURCE=yaml` +2. Restart the application +3. Providers will be loaded from `config/auth.yml` + +--- + +## 7. Troubleshooting + +### Provider not appearing on login page + +- **YAML mode:** Check that required environment variables are set (e.g., `OIDC_ISSUER`, `OIDC_CLIENT_ID`, `OIDC_CLIENT_SECRET`) +- **DB mode:** Verify provider is enabled in `/admin/sso_providers` +- Check application logs for provider loading messages +- Verify `AUTH_PROVIDERS_SOURCE` is set correctly + +### Discovery endpoint validation fails + +When adding an OIDC provider, Sure validates the `.well-known/openid-configuration` endpoint: + +- Ensure the issuer URL is correct and accessible +- Check firewall rules allow outbound HTTPS to the issuer +- Verify the issuer returns valid JSON with an `issuer` field +- For self-signed certificates, you may need to configure SSL verification + +### Rate limiting errors (429) + +Admin endpoints are rate-limited to 10 requests per minute per IP: + +- Wait 60 seconds before retrying +- If legitimate traffic is being blocked, adjust limits in `config/initializers/rack_attack.rb` + +### Callback URL mismatch + +Each provider requires a callback URL configured in your identity provider: + +- **Format:** `https://yourdomain.com/auth//callback` +- **Example:** For a provider with `name: "keycloak"`, use `https://yourdomain.com/auth/keycloak/callback` +- The callback URL is shown in the admin UI when editing a provider (with copy button) + +--- + +## 8. Security Considerations + +### Encryption + +- Client secrets are encrypted at rest using Rails 7.2 ActiveRecord Encryption +- Encryption keys are derived from `SECRET_KEY_BASE` by default +- For additional security, set custom encryption keys (see `.env` for `ACTIVE_RECORD_ENCRYPTION_*` variables) + +### Issuer validation + +- OIDC identities store the issuer claim from the ID token +- On subsequent logins, Sure verifies the issuer matches the configured provider +- This prevents issuer impersonation attacks + +### Admin access + +- SSO provider management requires `super_admin` role +- Regular `admin` users (family admins) cannot access `/admin/sso_providers` +- All provider changes are logged with user ID + +### Rate limiting + +- Admin endpoints: 10 requests/minute per IP +- OAuth token endpoint: 10 requests/minute per IP +- Failed login attempts should be monitored separately + +--- + +For additional help, see the main [hosting documentation](../README.md) or open an issue on GitHub. diff --git a/lib/tasks/sso_providers.rake b/lib/tasks/sso_providers.rake new file mode 100644 index 000000000..9b5762aac --- /dev/null +++ b/lib/tasks/sso_providers.rake @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +namespace :sso_providers do + desc "Seed SSO providers from config/auth.yml into the database" + task seed: :environment do + dry_run = ENV["DRY_RUN"] == "true" + + puts "=" * 80 + puts "SSO Provider Seeding Task" + puts "=" * 80 + puts "Mode: #{dry_run ? 'DRY RUN (no changes will be saved)' : 'LIVE (changes will be saved)'}" + puts "Source: config/auth.yml" + puts "-" * 80 + + begin + # Load auth.yml safely + auth_config_path = Rails.root.join("config", "auth.yml") + unless File.exist?(auth_config_path) + puts "ERROR: config/auth.yml not found" + exit 1 + end + + # Use safe_load to prevent code injection + auth_config = YAML.safe_load( + ERB.new(File.read(auth_config_path)).result, + permitted_classes: [ Symbol ], + aliases: true + ) + + # Get providers for current environment + env_config = auth_config[Rails.env] || auth_config["default"] + providers = env_config&.dig("providers") || [] + + if providers.empty? + puts "WARNING: No providers found in config/auth.yml for #{Rails.env} environment" + exit 0 + end + + puts "Found #{providers.count} provider(s) in config/auth.yml" + puts "-" * 80 + + created_count = 0 + updated_count = 0 + skipped_count = 0 + errors = [] + + ActiveRecord::Base.transaction do + providers.each do |provider_config| + provider_config = provider_config.deep_symbolize_keys + + # Extract provider attributes + name = provider_config[:name] || provider_config[:id] + strategy = provider_config[:strategy] + + unless name.present? && strategy.present? + puts "SKIP: Provider missing name or strategy: #{provider_config.inspect}" + skipped_count += 1 + next + end + + # Find or initialize provider + provider = SsoProvider.find_or_initialize_by(name: name) + is_new = provider.new_record? + + # Build attributes hash + attributes = { + strategy: strategy, + label: provider_config[:label] || name.titleize, + icon: provider_config[:icon], + enabled: provider_config.key?(:enabled) ? provider_config[:enabled] : true, + issuer: provider_config[:issuer], + client_id: provider_config[:client_id], + redirect_uri: provider_config[:redirect_uri], + settings: provider_config[:settings] || {} + } + + # Only set client_secret if provided (don't overwrite existing) + if provider_config[:client_secret].present? + attributes[:client_secret] = provider_config[:client_secret] + end + + # Assign attributes + provider.assign_attributes(attributes.compact) + + # Check if changed + if provider.changed? + if dry_run + puts "#{is_new ? 'CREATE' : 'UPDATE'} (dry-run): #{name} (#{strategy})" + puts " Changes: #{provider.changes.keys.join(', ')}" + else + if provider.save + puts "#{is_new ? 'CREATE' : 'UPDATE'}: #{name} (#{strategy})" + is_new ? created_count += 1 : updated_count += 1 + else + error_msg = "Failed to save #{name}: #{provider.errors.full_messages.join(', ')}" + puts "ERROR: #{error_msg}" + errors << error_msg + end + end + else + puts "SKIP: #{name} (no changes)" + skipped_count += 1 + end + end + + # Rollback transaction if dry run + raise ActiveRecord::Rollback if dry_run + end + + puts "-" * 80 + puts "Summary:" + puts " Created: #{created_count}" + puts " Updated: #{updated_count}" + puts " Skipped: #{skipped_count}" + puts " Errors: #{errors.count}" + + if errors.any? + puts "\nErrors encountered:" + errors.each { |error| puts " - #{error}" } + end + + if dry_run + puts "\nDRY RUN: No changes were saved to the database" + puts "Run without DRY_RUN=true to apply changes" + else + puts "\nSeeding completed successfully!" + puts "Note: Clear provider cache or restart server for changes to take effect" + end + + puts "=" * 80 + + rescue => e + puts "ERROR: #{e.class}: #{e.message}" + puts e.backtrace.first(5).join("\n") + exit 1 + end + end + + desc "List all SSO providers in the database" + task list: :environment do + providers = SsoProvider.order(:name) + + if providers.empty? + puts "No SSO providers found in database" + else + puts "SSO Providers (#{providers.count}):" + puts "-" * 80 + providers.each do |provider| + status = provider.enabled? ? "✓ enabled" : "✗ disabled" + puts "#{provider.name.ljust(20)} | #{provider.strategy.ljust(20)} | #{status}" + end + end + end +end diff --git a/test/controllers/oidc_accounts_controller_test.rb b/test/controllers/oidc_accounts_controller_test.rb index dab141d1d..c81b235be 100644 --- a/test/controllers/oidc_accounts_controller_test.rb +++ b/test/controllers/oidc_accounts_controller_test.rb @@ -166,7 +166,7 @@ class OidcAccountsControllerTest < ActionController::TestCase assert_not_nil new_user assert_equal new_user_auth["first_name"], new_user.first_name assert_equal new_user_auth["last_name"], new_user.last_name - assert_equal "admin", new_user.role + assert_equal "member", new_user.role # Verify OIDC identity was created oidc_identity = new_user.oidc_identities.first diff --git a/test/models/sso_provider_test.rb b/test/models/sso_provider_test.rb new file mode 100644 index 000000000..81ca0b6df --- /dev/null +++ b/test/models/sso_provider_test.rb @@ -0,0 +1,263 @@ +require "test_helper" + +class SsoProviderTest < ActiveSupport::TestCase + test "valid provider with all required fields" do + provider = SsoProvider.new( + strategy: "openid_connect", + name: "test_oidc", + label: "Test OIDC", + enabled: true, + issuer: "https://test.example.com", + client_id: "test_client", + client_secret: "test_secret" + ) + assert provider.valid? + end + + test "requires strategy" do + provider = SsoProvider.new(name: "test", label: "Test") + assert_not provider.valid? + assert_includes provider.errors[:strategy], "can't be blank" + end + + test "requires name" do + provider = SsoProvider.new(strategy: "openid_connect", label: "Test") + assert_not provider.valid? + assert_includes provider.errors[:name], "can't be blank" + end + + test "requires label" do + provider = SsoProvider.new(strategy: "openid_connect", name: "test") + assert_not provider.valid? + assert_includes provider.errors[:label], "can't be blank" + end + + test "requires unique name" do + SsoProvider.create!( + strategy: "openid_connect", + name: "duplicate", + label: "First", + client_id: "id1", + client_secret: "secret1", + issuer: "https://first.example.com" + ) + + provider = SsoProvider.new( + strategy: "google_oauth2", + name: "duplicate", + label: "Second", + client_id: "id2", + client_secret: "secret2" + ) + + assert_not provider.valid? + assert_includes provider.errors[:name], "has already been taken" + end + + test "validates name format" do + provider = SsoProvider.new( + strategy: "openid_connect", + name: "Invalid-Name!", + label: "Test", + client_id: "test", + client_secret: "secret", + issuer: "https://test.example.com" + ) + + assert_not provider.valid? + assert_includes provider.errors[:name], "must contain only lowercase letters, numbers, and underscores" + end + + test "validates strategy inclusion" do + provider = SsoProvider.new( + strategy: "invalid_strategy", + name: "test", + label: "Test" + ) + + assert_not provider.valid? + assert_includes provider.errors[:strategy], "invalid_strategy is not a supported strategy" + end + + test "encrypts client_secret" do + provider = SsoProvider.create!( + strategy: "openid_connect", + name: "encrypted_test", + label: "Encrypted Test", + client_id: "test_client", + client_secret: "super_secret_value", + issuer: "https://test.example.com" + ) + + # Reload from database + provider.reload + + # Should be able to read decrypted value + assert_equal "super_secret_value", provider.client_secret + + # Raw database value should be encrypted (not plain text) + raw_value = ActiveRecord::Base.connection.execute( + "SELECT client_secret FROM sso_providers WHERE id = '#{provider.id}'" + ).first["client_secret"] + + assert_not_equal "super_secret_value", raw_value + end + + test "OIDC provider requires issuer" do + provider = SsoProvider.new( + strategy: "openid_connect", + name: "test_oidc", + label: "Test", + client_id: "test", + client_secret: "secret" + ) + + assert_not provider.valid? + assert_includes provider.errors[:issuer], "is required for OpenID Connect providers" + end + + test "OIDC provider requires client_id" do + provider = SsoProvider.new( + strategy: "openid_connect", + name: "test_oidc", + label: "Test", + issuer: "https://test.example.com", + client_secret: "secret" + ) + + assert_not provider.valid? + assert_includes provider.errors[:client_id], "is required for OpenID Connect providers" + end + + test "OIDC provider requires client_secret" do + provider = SsoProvider.new( + strategy: "openid_connect", + name: "test_oidc", + label: "Test", + issuer: "https://test.example.com", + client_id: "test" + ) + + assert_not provider.valid? + assert_includes provider.errors[:client_secret], "is required for OpenID Connect providers" + end + + test "OIDC provider validates issuer URL format" do + provider = SsoProvider.new( + strategy: "openid_connect", + name: "test_oidc", + label: "Test", + issuer: "not-a-valid-url", + client_id: "test", + client_secret: "secret" + ) + + assert_not provider.valid? + assert_includes provider.errors[:issuer], "must be a valid URL" + end + + test "OAuth provider requires client_id" do + provider = SsoProvider.new( + strategy: "google_oauth2", + name: "test_google", + label: "Test", + client_secret: "secret" + ) + + assert_not provider.valid? + assert_includes provider.errors[:client_id], "is required for OAuth providers" + end + + test "OAuth provider requires client_secret" do + provider = SsoProvider.new( + strategy: "google_oauth2", + name: "test_google", + label: "Test", + client_id: "test" + ) + + assert_not provider.valid? + assert_includes provider.errors[:client_secret], "is required for OAuth providers" + end + + test "enabled scope returns only enabled providers" do + enabled = SsoProvider.create!( + strategy: "openid_connect", + name: "enabled_provider", + label: "Enabled", + enabled: true, + client_id: "test", + client_secret: "secret", + issuer: "https://enabled.example.com" + ) + + SsoProvider.create!( + strategy: "openid_connect", + name: "disabled_provider", + label: "Disabled", + enabled: false, + client_id: "test", + client_secret: "secret", + issuer: "https://disabled.example.com" + ) + + assert_includes SsoProvider.enabled, enabled + assert_equal 1, SsoProvider.enabled.count + end + + test "by_strategy scope filters by strategy" do + oidc = SsoProvider.create!( + strategy: "openid_connect", + name: "oidc_provider", + label: "OIDC", + client_id: "test", + client_secret: "secret", + issuer: "https://oidc.example.com" + ) + + SsoProvider.create!( + strategy: "google_oauth2", + name: "google_provider", + label: "Google", + client_id: "test", + client_secret: "secret" + ) + + oidc_providers = SsoProvider.by_strategy("openid_connect") + assert_includes oidc_providers, oidc + assert_equal 1, oidc_providers.count + end + + test "to_omniauth_config returns correct hash" do + provider = SsoProvider.create!( + strategy: "openid_connect", + name: "test_oidc", + label: "Test OIDC", + icon: "key", + enabled: true, + issuer: "https://test.example.com", + client_id: "test_client", + client_secret: "test_secret", + redirect_uri: "https://app.example.com/callback", + settings: { scope: "openid email" } + ) + + config = provider.to_omniauth_config + + assert_equal "test_oidc", config[:id] + assert_equal "openid_connect", config[:strategy] + assert_equal "test_oidc", config[:name] + assert_equal "Test OIDC", config[:label] + assert_equal "key", config[:icon] + assert_equal "https://test.example.com", config[:issuer] + assert_equal "test_client", config[:client_id] + assert_equal "test_secret", config[:client_secret] + assert_equal "https://app.example.com/callback", config[:redirect_uri] + assert_equal({ "scope" => "openid email" }, config[:settings]) + end + + # Note: OIDC discovery validation tests are skipped in test environment + # Discovery validation is disabled in test mode to avoid VCR cassette requirements + # In production, the validate_oidc_discovery method will validate the issuer's + # .well-known/openid-configuration endpoint +end diff --git a/test/policies/sso_provider_policy_test.rb b/test/policies/sso_provider_policy_test.rb new file mode 100644 index 000000000..8632afdd0 --- /dev/null +++ b/test/policies/sso_provider_policy_test.rb @@ -0,0 +1,111 @@ +require "test_helper" + +class SsoProviderPolicyTest < ActiveSupport::TestCase + def setup + @super_admin = users(:family_admin) # Assuming this fixture has super_admin role + @super_admin.update!(role: :super_admin) + + @regular_user = users(:family_member) + @regular_user.update!(role: :member) + + @provider = SsoProvider.create!( + strategy: "openid_connect", + name: "test_provider", + label: "Test Provider", + client_id: "test", + client_secret: "secret", + issuer: "https://test.example.com" + ) + end + + test "super admin can view index" do + assert SsoProviderPolicy.new(@super_admin, SsoProvider).index? + end + + test "regular user cannot view index" do + assert_not SsoProviderPolicy.new(@regular_user, SsoProvider).index? + end + + test "nil user cannot view index" do + assert_not SsoProviderPolicy.new(nil, SsoProvider).index? + end + + test "super admin can show provider" do + assert SsoProviderPolicy.new(@super_admin, @provider).show? + end + + test "regular user cannot show provider" do + assert_not SsoProviderPolicy.new(@regular_user, @provider).show? + end + + test "super admin can create provider" do + assert SsoProviderPolicy.new(@super_admin, SsoProvider.new).create? + end + + test "regular user cannot create provider" do + assert_not SsoProviderPolicy.new(@regular_user, SsoProvider.new).create? + end + + test "super admin can access new" do + assert SsoProviderPolicy.new(@super_admin, SsoProvider.new).new? + end + + test "regular user cannot access new" do + assert_not SsoProviderPolicy.new(@regular_user, SsoProvider.new).new? + end + + test "super admin can update provider" do + assert SsoProviderPolicy.new(@super_admin, @provider).update? + end + + test "regular user cannot update provider" do + assert_not SsoProviderPolicy.new(@regular_user, @provider).update? + end + + test "super admin can access edit" do + assert SsoProviderPolicy.new(@super_admin, @provider).edit? + end + + test "regular user cannot access edit" do + assert_not SsoProviderPolicy.new(@regular_user, @provider).edit? + end + + test "super admin can destroy provider" do + assert SsoProviderPolicy.new(@super_admin, @provider).destroy? + end + + test "regular user cannot destroy provider" do + assert_not SsoProviderPolicy.new(@regular_user, @provider).destroy? + end + + test "super admin can toggle provider" do + assert SsoProviderPolicy.new(@super_admin, @provider).toggle? + end + + test "regular user cannot toggle provider" do + assert_not SsoProviderPolicy.new(@regular_user, @provider).toggle? + end + + test "scope returns all providers for super admin" do + SsoProvider.create!( + strategy: "google_oauth2", + name: "google", + label: "Google", + client_id: "test", + client_secret: "secret" + ) + + scope = SsoProviderPolicy::Scope.new(@super_admin, SsoProvider).resolve + assert_equal 2, scope.count + end + + test "scope returns no providers for regular user" do + scope = SsoProviderPolicy::Scope.new(@regular_user, SsoProvider).resolve + assert_equal 0, scope.count + end + + test "scope returns no providers for nil user" do + scope = SsoProviderPolicy::Scope.new(nil, SsoProvider).resolve + assert_equal 0, scope.count + end +end diff --git a/test/policies/user_policy_test.rb b/test/policies/user_policy_test.rb new file mode 100644 index 000000000..c4d471d94 --- /dev/null +++ b/test/policies/user_policy_test.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require "test_helper" + +class UserPolicyTest < ActiveSupport::TestCase + def setup + @super_admin = users(:family_admin) + @super_admin.update!(role: :super_admin) + + @regular_user = users(:family_member) + @regular_user.update!(role: :member) + + @other_user = users(:sure_support_staff) + @other_user.update!(role: :member) + end + + test "super admin can view index" do + assert UserPolicy.new(@super_admin, User).index? + end + + test "regular user cannot view index" do + assert_not UserPolicy.new(@regular_user, User).index? + end + + test "nil user cannot view index" do + assert_not UserPolicy.new(nil, User).index? + end + + test "super admin can update another user" do + assert UserPolicy.new(@super_admin, @regular_user).update? + end + + test "super admin cannot update themselves" do + assert_not UserPolicy.new(@super_admin, @super_admin).update? + end + + test "regular user cannot update anyone" do + assert_not UserPolicy.new(@regular_user, @other_user).update? + end + + test "nil user cannot update anyone" do + assert_not UserPolicy.new(nil, @regular_user).update? + end + + test "scope returns all users for super admin" do + scope = UserPolicy::Scope.new(@super_admin, User).resolve + assert_equal User.count, scope.count + end + + test "scope returns no users for regular user" do + scope = UserPolicy::Scope.new(@regular_user, User).resolve + assert_equal 0, scope.count + end + + test "scope returns no users for nil user" do + scope = UserPolicy::Scope.new(nil, User).resolve + assert_equal 0, scope.count + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 1479ce88e..16bab4495 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -22,6 +22,7 @@ require "minitest/mock" require "minitest/autorun" require "mocha/minitest" require "aasm/minitest" +require "webmock/minitest" VCR.configure do |config| config.cassette_library_dir = "test/vcr_cassettes" From d3055b2e0bb4abaa842810ae8d92c7581809152b Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Sat, 3 Jan 2026 20:49:31 -0500 Subject: [PATCH 02/54] refactor: remove SSO settings page; consolidate SSO identity management under Security settings - Removed the `Settings::SsoIdentitiesController` and views for a simplified user experience. - Moved SSO identity management to the Security settings page (`Settings::SecuritiesController`). - Updated locale keys and layout for the new structure. - Fixed unlink protection warnings and adjusted redirection path. - Cleaned up routes, helper methods, and redundant code. --- .../UI/account/activity_date.html.erb | 2 +- .../UI/account/activity_feed.html.erb | 2 +- .../settings/securities_controller.rb | 1 + .../settings/sso_identities_controller.rb | 12 +--- app/helpers/settings_helper.rb | 5 -- app/policies/sso_provider_policy.rb | 4 ++ app/views/accounts/show/_activity.html.erb | 2 +- .../_budget_category.html.erb | 2 +- app/views/categories/_badge_mobile.html.erb | 2 +- app/views/entries/_entry_group.html.erb | 2 +- .../configurations/_rule_import.html.erb | 1 - .../pages/dashboard/_outflows_donut.html.erb | 2 +- .../recurring_transactions/index.html.erb | 2 +- app/views/rules/index.html.erb | 4 +- app/views/settings/llm_usages/show.html.erb | 2 +- app/views/settings/providers/show.html.erb | 1 - app/views/settings/securities/show.html.erb | 55 +++++++++++++++++ .../settings/sso_identities/show.html.erb | 59 ------------------- app/views/shared/_demo_warning.html.erb | 1 - .../simplefin_items/_simplefin_item.html.erb | 3 - app/views/transactions/index.html.erb | 2 +- config/locales/views/settings/en.yml | 17 ++++++ .../views/settings/sso_identities/en.yml | 22 ------- config/routes.rb | 1 - 24 files changed, 91 insertions(+), 115 deletions(-) delete mode 100644 app/views/settings/sso_identities/show.html.erb delete mode 100644 config/locales/views/settings/sso_identities/en.yml diff --git a/app/components/UI/account/activity_date.html.erb b/app/components/UI/account/activity_date.html.erb index 672c6d77c..a1a330588 100644 --- a/app/components/UI/account/activity_date.html.erb +++ b/app/components/UI/account/activity_date.html.erb @@ -6,7 +6,7 @@ <%= check_box_tag "#{date}_entries_selection", class: ["checkbox checkbox--light hidden lg:block", "lg:hidden": entries.size == 0], id: "selection_entry_#{date}", - data: { + data: { action: "bulk-select#toggleGroupSelection", checkbox_toggle_target: "selectionEntry" } %> diff --git a/app/components/UI/account/activity_feed.html.erb b/app/components/UI/account/activity_feed.html.erb index 362e1af06..f46053401 100644 --- a/app/components/UI/account/activity_feed.html.erb +++ b/app/components/UI/account/activity_feed.html.erb @@ -77,7 +77,7 @@
<%= check_box_tag "selection_entry", class: "checkbox checkbox--light hidden lg:block", - data: { + data: { action: "bulk-select#togglePageSelection", checkbox_toggle_target: "selectionEntry" } %> diff --git a/app/controllers/settings/securities_controller.rb b/app/controllers/settings/securities_controller.rb index 756accf79..fd6791994 100644 --- a/app/controllers/settings/securities_controller.rb +++ b/app/controllers/settings/securities_controller.rb @@ -6,5 +6,6 @@ class Settings::SecuritiesController < ApplicationController [ "Home", root_path ], [ "Security", nil ] ] + @oidc_identities = Current.user.oidc_identities.order(:provider) end end diff --git a/app/controllers/settings/sso_identities_controller.rb b/app/controllers/settings/sso_identities_controller.rb index f42175c62..97f946a90 100644 --- a/app/controllers/settings/sso_identities_controller.rb +++ b/app/controllers/settings/sso_identities_controller.rb @@ -3,20 +3,12 @@ class Settings::SsoIdentitiesController < ApplicationController layout "settings" - def show - @oidc_identities = Current.user.oidc_identities.order(:provider) - @breadcrumbs = [ - [ t("settings.nav.home"), root_path ], - [ t(".page_title"), nil ] - ] - end - def destroy @identity = Current.user.oidc_identities.find(params[:id]) # Prevent unlinking last identity if user has no password if Current.user.oidc_identities.count == 1 && Current.user.password_digest.blank? - redirect_to settings_sso_identities_path, alert: t(".cannot_unlink_last") + redirect_to settings_security_path, alert: t(".cannot_unlink_last") return end @@ -30,6 +22,6 @@ class Settings::SsoIdentitiesController < ApplicationController request: request ) - redirect_to settings_sso_identities_path, notice: t(".success", provider: provider_name) + redirect_to settings_security_path, notice: t(".success", provider: provider_name) end end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index 1428aeda0..a907cd7c8 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -6,7 +6,6 @@ module SettingsHelper { name: "Preferences", path: :settings_preferences_path }, { name: "Profile Info", path: :settings_profile_path }, { name: "Security", path: :settings_security_path }, - { name: "Connected Accounts", path: :settings_sso_identities_path, condition: :has_sso_connections? }, { name: "Billing", path: :settings_billing_path, condition: :not_self_hosted? }, # Transactions section { name: "Categories", path: :categories_path }, @@ -82,8 +81,4 @@ module SettingsHelper def self_hosted_and_admin? self_hosted? && admin_user? end - - def has_sso_connections? - Current.user&.oidc_identities&.exists? || AuthConfig.sso_providers.any? - end end diff --git a/app/policies/sso_provider_policy.rb b/app/policies/sso_provider_policy.rb index 5c975dc66..d68010415 100644 --- a/app/policies/sso_provider_policy.rb +++ b/app/policies/sso_provider_policy.rb @@ -34,6 +34,10 @@ class SsoProviderPolicy < ApplicationPolicy update? end + def test_connection? + user&.super_admin? + end + class Scope < ApplicationPolicy::Scope def resolve if user&.super_admin? diff --git a/app/views/accounts/show/_activity.html.erb b/app/views/accounts/show/_activity.html.erb index 7bd35639b..dbed8ad52 100644 --- a/app/views/accounts/show/_activity.html.erb +++ b/app/views/accounts/show/_activity.html.erb @@ -67,7 +67,7 @@
<%= check_box_tag "selection_entry", class: "checkbox checkbox--light hidden lg:block", - data: { + data: { action: "bulk-select#togglePageSelection", checkbox_toggle_target: "selectionEntry" } %> diff --git a/app/views/budget_categories/_budget_category.html.erb b/app/views/budget_categories/_budget_category.html.erb index 3ce90e6f4..a241101d2 100644 --- a/app/views/budget_categories/_budget_category.html.erb +++ b/app/views/budget_categories/_budget_category.html.erb @@ -6,7 +6,7 @@ <% if budget_category.initialized? %> <%# Category Header with Status Badge %>
-
<% end %> -
\ No newline at end of file +
diff --git a/app/views/entries/_entry_group.html.erb b/app/views/entries/_entry_group.html.erb index 7a0d14470..5499c9f84 100644 --- a/app/views/entries/_entry_group.html.erb +++ b/app/views/entries/_entry_group.html.erb @@ -6,7 +6,7 @@ <%= check_box_tag "#{date}_entries_selection", class: ["checkbox checkbox--light hidden lg:block", "lg:hidden": entries.size == 0], id: "selection_entry_#{date}", - data: { + data: { action: "bulk-select#toggleGroupSelection", checkbox_toggle_target: "selectionEntry" } %> diff --git a/app/views/import/configurations/_rule_import.html.erb b/app/views/import/configurations/_rule_import.html.erb index 7089a40ad..eb0f8be8b 100644 --- a/app/views/import/configurations/_rule_import.html.erb +++ b/app/views/import/configurations/_rule_import.html.erb @@ -12,4 +12,3 @@ <%= form.submit t("import.configurations.rule_import.process_button"), disabled: import.complete? %> <% end %>
- diff --git a/app/views/pages/dashboard/_outflows_donut.html.erb b/app/views/pages/dashboard/_outflows_donut.html.erb index 2657662c7..4b11de35c 100644 --- a/app/views/pages/dashboard/_outflows_donut.html.erb +++ b/app/views/pages/dashboard/_outflows_donut.html.erb @@ -78,7 +78,7 @@ action: "mouseenter->donut-chart#highlightSegment mouseleave->donut-chart#unhighlightSegment" } do %>
-

<%= t("recurring_transactions.title") %>

- <% unless @family.recurring_transactions_disabled? %> + <% unless @family.recurring_transactions_disabled? %> <%= render DS::Menu.new do |menu| %> <% menu.with_item( variant: "button", diff --git a/app/views/rules/index.html.erb b/app/views/rules/index.html.erb index 23613b959..f40020abe 100644 --- a/app/views/rules/index.html.erb +++ b/app/views/rules/index.html.erb @@ -117,12 +117,12 @@ <% @recent_runs.each do |run| %> - + "> <%= run.executed_at.strftime("%b %d, %Y %I:%M %p") %> - + "> <%= t("rules.recent_runs.execution_types.#{run.execution_type}") %> diff --git a/app/views/settings/llm_usages/show.html.erb b/app/views/settings/llm_usages/show.html.erb index 2e32e2f4c..387ebaf56 100644 --- a/app/views/settings/llm_usages/show.html.erb +++ b/app/views/settings/llm_usages/show.html.erb @@ -117,7 +117,7 @@ <% @llm_usages.each do |usage| %> - + "> <%= usage.created_at.strftime("%b %d, %Y %I:%M %p") %> diff --git a/app/views/settings/providers/show.html.erb b/app/views/settings/providers/show.html.erb index 4f84027ad..559a54972 100644 --- a/app/views/settings/providers/show.html.erb +++ b/app/views/settings/providers/show.html.erb @@ -26,7 +26,6 @@ <% end %> - <%= settings_section title: "Enable Banking (beta)", collapsible: true, open: false do %> <%= render "settings/providers/enable_banking_panel" %> diff --git a/app/views/settings/securities/show.html.erb b/app/views/settings/securities/show.html.erb index 1d3a7350f..f0cfb56bb 100644 --- a/app/views/settings/securities/show.html.erb +++ b/app/views/settings/securities/show.html.erb @@ -44,3 +44,58 @@
<% end %> + +<% if @oidc_identities.any? || AuthConfig.sso_providers.any? %> + <%= settings_section title: t(".sso_title"), subtitle: t(".sso_subtitle") do %> + <% if @oidc_identities.any? %> +
+ <% @oidc_identities.each do |identity| %> +
+
+
+ <%= icon identity.provider_config&.dig(:icon) || "key", class: "w-5 h-5 text-secondary" %> +
+
+

<%= identity.provider_config&.dig(:label) || identity.provider.titleize %>

+

<%= identity.info&.dig("email") || t(".sso_no_email") %>

+

+ <%= t(".sso_last_used") %>: + <%= identity.last_authenticated_at&.to_fs(:short) || t(".sso_never") %> +

+
+
+ <% if @oidc_identities.count > 1 || Current.user.password_digest.present? %> + <%= render DS::Button.new( + text: t(".sso_disconnect"), + variant: "outline", + size: "sm", + href: settings_sso_identity_path(identity), + method: :delete, + confirm: CustomConfirm.new( + title: t(".sso_confirm_title"), + body: t(".sso_confirm_body", provider: identity.provider_config&.dig(:label) || identity.provider.titleize), + btn_text: t(".sso_confirm_button"), + destructive: true + ) + ) %> + <% end %> +
+ <% end %> +
+ <% if @oidc_identities.count == 1 && Current.user.password_digest.blank? %> +
+
+ <%= icon "alert-triangle", class: "w-5 h-5 text-amber-600 shrink-0 mt-0.5" %> +

<%= t(".sso_warning_message") %>

+
+
+ <% end %> + <% else %> +
+ <%= icon "link", class: "w-12 h-12 mx-auto text-secondary mb-3" %> +

<%= t(".sso_no_identities") %>

+

<%= t(".sso_connect_hint") %>

+
+ <% end %> + <% end %> +<% end %> diff --git a/app/views/settings/sso_identities/show.html.erb b/app/views/settings/sso_identities/show.html.erb deleted file mode 100644 index b9046425e..000000000 --- a/app/views/settings/sso_identities/show.html.erb +++ /dev/null @@ -1,59 +0,0 @@ -<%= content_for :page_title, t(".page_title") %> - -<%= settings_section title: t(".identities_title"), subtitle: t(".identities_subtitle") do %> - <% if @oidc_identities.any? %> -
- <% @oidc_identities.each do |identity| %> -
-
-
- <%= icon identity.provider_config&.dig(:icon) || "key", class: "w-5 h-5 text-secondary" %> -
-
-

<%= identity.provider_config&.dig(:label) || identity.provider.titleize %>

-

<%= identity.info&.dig("email") || t(".no_email") %>

-

- <%= t(".last_used") %>: - <%= identity.last_authenticated_at&.to_fs(:short) || t(".never") %> -

-
-
- <% if @oidc_identities.count > 1 || Current.user.password_digest.present? %> - <%= render DS::Button.new( - text: t(".disconnect"), - variant: "outline", - size: "sm", - href: settings_sso_identity_path(identity), - method: :delete, - confirm: CustomConfirm.new( - title: t(".confirm_title"), - body: t(".confirm_body", provider: identity.provider_config&.dig(:label) || identity.provider.titleize), - btn_text: t(".confirm_button"), - destructive: true - ) - ) %> - <% end %> -
- <% end %> -
- <% else %> -
- <%= icon "link", class: "w-12 h-12 mx-auto text-secondary mb-3" %> -

<%= t(".no_identities") %>

- <% if AuthConfig.sso_providers.any? %> -

<%= t(".connect_hint") %>

- <% end %> -
- <% end %> -<% end %> - -<% if @oidc_identities.count == 1 && Current.user.password_digest.blank? %> - <%= settings_section title: t(".warning_title") do %> -
-
- <%= icon "alert-triangle", class: "w-5 h-5 text-amber-600 shrink-0 mt-0.5" %> -

<%= t(".warning_message") %>

-
-
- <% end %> -<% end %> diff --git a/app/views/shared/_demo_warning.html.erb b/app/views/shared/_demo_warning.html.erb index e1d003dac..ba3c08f23 100644 --- a/app/views/shared/_demo_warning.html.erb +++ b/app/views/shared/_demo_warning.html.erb @@ -10,4 +10,3 @@
- diff --git a/app/views/simplefin_items/_simplefin_item.html.erb b/app/views/simplefin_items/_simplefin_item.html.erb index 680a89d04..93510054c 100644 --- a/app/views/simplefin_items/_simplefin_item.html.erb +++ b/app/views/simplefin_items/_simplefin_item.html.erb @@ -125,8 +125,6 @@ ) %> <% end %> - - <%= render DS::Menu.new do |menu| %> <% menu.with_item( variant: "button", @@ -146,7 +144,6 @@ <%= render "accounts/index/account_groups", accounts: simplefin_item.accounts %> <% end %> - <%# Sync summary (collapsible) Prefer controller-provided map; fallback to latest sync stats so Turbo broadcasts can render the summary without requiring a full page refresh. %> diff --git a/app/views/transactions/index.html.erb b/app/views/transactions/index.html.erb index 58055f342..7251c4c0a 100644 --- a/app/views/transactions/index.html.erb +++ b/app/views/transactions/index.html.erb @@ -63,7 +63,7 @@
<%= check_box_tag "selection_entry", class: "checkbox checkbox--light hidden lg:block", - data: { + data: { action: "bulk-select#togglePageSelection", checkbox_toggle_target: "selectionEntry" } %> diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index 567310cc8..e84e9e056 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -89,6 +89,23 @@ en: securities: show: page_title: Security + mfa_title: Two-Factor Authentication + mfa_description: Add an extra layer of security to your account by requiring a code from your authenticator app when signing in + enable_mfa: Enable 2FA + disable_mfa: Disable 2FA + disable_mfa_confirm: Are you sure you want to disable two-factor authentication? + sso_title: Connected Accounts + sso_subtitle: Manage your single sign-on account connections + sso_disconnect: Disconnect + sso_last_used: Last used + sso_never: Never + sso_no_email: No email + sso_no_identities: No SSO accounts connected + sso_connect_hint: Log out and sign in with an SSO provider to connect an account. + sso_confirm_title: Disconnect Account? + sso_confirm_body: Are you sure you want to disconnect your %{provider} account? You can reconnect it later by signing in with that provider again. + sso_confirm_button: Disconnect + sso_warning_message: This is your only login method. You should set a password in your security settings before disconnecting, otherwise you may be locked out of your account. settings_nav: accounts_label: Accounts advanced_section_title: Advanced diff --git a/config/locales/views/settings/sso_identities/en.yml b/config/locales/views/settings/sso_identities/en.yml deleted file mode 100644 index c989ee974..000000000 --- a/config/locales/views/settings/sso_identities/en.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -en: - settings: - sso_identities: - show: - page_title: "Connected Accounts" - identities_title: "SSO Connections" - identities_subtitle: "Manage your single sign-on account connections" - disconnect: "Disconnect" - last_used: "Last used" - never: "Never" - no_email: "No email" - no_identities: "No SSO accounts connected" - connect_hint: "Log out and sign in with an SSO provider to connect an account." - confirm_title: "Disconnect Account?" - confirm_body: "Are you sure you want to disconnect your %{provider} account? You can reconnect it later by signing in with that provider again." - confirm_button: "Disconnect" - warning_title: "Important" - warning_message: "This is your only login method. You should set a password in your security settings before disconnecting, otherwise you may be locked out of your account." - destroy: - success: "Successfully disconnected %{provider}" - cannot_unlink_last: "Cannot disconnect your only login method. Please set a password first." diff --git a/config/routes.rb b/config/routes.rb index 9d0e8fa4a..f591ea2f4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -91,7 +91,6 @@ Rails.application.routes.draw do end resource :billing, only: :show resource :security, only: :show - resource :sso_identities, only: :show resources :sso_identities, only: :destroy resource :api_key, only: [ :show, :new, :create, :destroy ] resource :ai_prompts, only: :show From b2ecc6bc678609a37b95be7621ca9a4873dc6619 Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Sat, 3 Jan 2026 21:13:24 -0500 Subject: [PATCH 03/54] refactor: improve SSO provider management and logging - Simplified `name_id_format` selection logic in SSO provider form. - Switched raw database query to sanitized SQL in client secret tests. - Added condition to log JIT account creation only when identity persists. - Sanitized failure reasons in SSO login failure handling. - Added SSO provider connection test policy tests for super admin and regular users. --- app/controllers/oidc_accounts_controller.rb | 16 +++++++++------- app/controllers/sessions_controller.rb | 6 +++++- app/views/admin/sso_providers/_form.html.erb | 3 +-- test/models/sso_provider_test.rb | 8 +++++--- test/policies/sso_provider_policy_test.rb | 8 ++++++++ 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/app/controllers/oidc_accounts_controller.rb b/app/controllers/oidc_accounts_controller.rb index bfc6597ab..8e346fe79 100644 --- a/app/controllers/oidc_accounts_controller.rb +++ b/app/controllers/oidc_accounts_controller.rb @@ -118,17 +118,19 @@ class OidcAccountsController < ApplicationController if @user.save # Create the OIDC (or other SSO) identity - OidcIdentity.create_from_omniauth( + identity = OidcIdentity.create_from_omniauth( build_auth_hash(@pending_auth), @user ) - # Log JIT account creation - SsoAuditLog.log_jit_account_created!( - user: @user, - provider: @pending_auth["provider"], - request: request - ) + # Only log JIT account creation if identity was successfully created + if identity.persisted? + SsoAuditLog.log_jit_account_created!( + user: @user, + provider: @pending_auth["provider"], + request: request + ) + end # Clear pending auth from session session.delete(:pending_oidc_auth) diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index cb375d752..337447ee3 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -144,11 +144,15 @@ class SessionsController < ApplicationController end def failure + # Sanitize reason to known values only + known_reasons = %w[sso_provider_unavailable sso_invalid_response sso_failed] + sanitized_reason = known_reasons.include?(params[:message]) ? params[:message] : "sso_failed" + # Log failed SSO attempt SsoAuditLog.log_login_failed!( provider: params[:strategy], request: request, - reason: params[:message] + reason: sanitized_reason ) message = case params[:message] diff --git a/app/views/admin/sso_providers/_form.html.erb b/app/views/admin/sso_providers/_form.html.erb index 88de8c6c6..61b8a6c17 100644 --- a/app/views/admin/sso_providers/_form.html.erb +++ b/app/views/admin/sso_providers/_form.html.erb @@ -158,8 +158,7 @@ + <%= currency.iso_code %> +
+ + +

+ = <%= currency.symbol %><%= number_with_precision(current_per_share, precision: 2) || "0.00" %> <%= t(".per_share") %> +

+ + +
+ +
+ <%= currency.symbol %> + + <%= currency.iso_code %> +
+
+ +
+ + <%= f.submit t(".save"), class: "inline-flex items-center gap-1 px-2 py-1 rounded-md text-sm font-medium text-inverse bg-inverse hover:bg-inverse-hover" %> +
+ <% end %> + + <% end %> + <% end %> + <% end %> +<% end %> diff --git a/app/views/holdings/_holding.html.erb b/app/views/holdings/_holding.html.erb index 45ff4bf81..85d9474ac 100644 --- a/app/views/holdings/_holding.html.erb +++ b/app/views/holdings/_holding.html.erb @@ -31,7 +31,7 @@
- <%= tag.p holding.avg_cost ? format_money(holding.avg_cost) : t(".unknown"), class: holding.avg_cost ? nil : "text-secondary" %> + <%= render "holdings/cost_basis_cell", holding: holding, editable: false %> <%= tag.p t(".per_share"), class: "font-normal text-secondary" %>
@@ -45,13 +45,13 @@
- <%# Show Total Return (unrealized G/L) when cost basis exists %> - <% if holding.trades.any? && holding.trend %> + <%# Show Total Return (unrealized G/L) when cost basis exists (from trades or manual) %> + <% if holding.trend %> <%= tag.p format_money(holding.trend.value), style: "color: #{holding.trend.color};" %> <%= tag.p "(#{number_to_percentage(holding.trend.percent, precision: 1)})", style: "color: #{holding.trend.color};" %> <% else %> <%= tag.p "--", class: "text-secondary" %> - <%= tag.p "No cost basis", class: "text-xs text-secondary" %> + <%= tag.p t(".no_cost_basis"), class: "text-xs text-secondary" %> <% end %>
diff --git a/app/views/holdings/show.html.erb b/app/views/holdings/show.html.erb index 32918b49e..868b77f1d 100644 --- a/app/views/holdings/show.html.erb +++ b/app/views/holdings/show.html.erb @@ -35,16 +35,107 @@
<%= @holding.weight ? number_to_percentage(@holding.weight, precision: 2) : t(".unknown") %>
-
-
<%= t(".avg_cost_label") %>
-
<%= @holding.avg_cost ? format_money(@holding.avg_cost) : t(".unknown") %>
+ <%# Average Cost with inline editor %> + <% + currency = Money::Currency.new(@holding.currency) + current_per_share = @holding.cost_basis.present? && @holding.cost_basis.positive? ? @holding.cost_basis : nil + current_total = current_per_share && @holding.qty.positive? ? (current_per_share * @holding.qty).round(2) : nil + %> +
+
+
<%= t(".avg_cost_label") %>
+
+ <%= @holding.avg_cost ? format_money(@holding.avg_cost) : t(".unknown") %> + <% if @holding.cost_basis_locked? %> + <%= icon "lock", size: "xs", class: "text-secondary" %> + <% end %> + <% if @holding.cost_basis_source.present? %> + (<%= @holding.cost_basis_source_label %>) + <% end %> + +
+
+ + <%# Inline cost basis editor (hidden by default) %> +
<%= t(".total_return_label") %>
-
- <%= @holding.trend ? render("shared/trend_change", trend: @holding.trend) : t(".unknown") %> -
+ <% if @holding.trend %> +
+ <%= render("shared/trend_change", trend: @holding.trend) %> +
+ <% else %> +
<%= t(".unknown") %>
+ <% end %>
@@ -85,21 +176,39 @@ <% end %> - <% if @holding.account.can_delete_holdings? %> + <% if @holding.cost_basis_locked? || @holding.account.can_delete_holdings? %> <% dialog.with_section(title: t(".settings"), open: true) do %>
-
-
-

<%= t(".delete_title") %>

-

<%= t(".delete_subtitle") %>

-
+ <% if @holding.cost_basis_locked? %> +
+
+

<%= t(".cost_basis_locked_label") %>

+

<%= t(".cost_basis_locked_description") %>

+
- <%= button_to t(".delete"), - holding_path(@holding), - method: :delete, - class: "rounded-lg px-3 py-2 text-red-500 text-sm font-medium border border-secondary", - data: { turbo_confirm: true } %> -
+ <%= button_to t(".unlock_cost_basis"), + unlock_cost_basis_holding_path(@holding), + method: :post, + class: "inline-flex items-center gap-1 px-3 py-2 rounded-lg text-sm font-medium text-primary bg-gray-200 hover:bg-gray-300 theme-dark:bg-gray-700 theme-dark:hover:bg-gray-600", + form: { data: { turbo: false } }, + data: { turbo_confirm: { title: t(".unlock_confirm_title"), body: t(".unlock_confirm_body") } } %> +
+ <% end %> + + <% if @holding.account.can_delete_holdings? %> +
+
+

<%= t(".delete_title") %>

+

<%= t(".delete_subtitle") %>

+
+ + <%= button_to t(".delete"), + holding_path(@holding), + method: :delete, + class: "rounded-lg px-3 py-2 text-red-500 text-sm font-medium border border-secondary", + data: { turbo_confirm: true } %> +
+ <% end %>
<% end %> <% end %> diff --git a/app/views/reports/_investment_performance.html.erb b/app/views/reports/_investment_performance.html.erb index f88fa9fd2..c727c819a 100644 --- a/app/views/reports/_investment_performance.html.erb +++ b/app/views/reports/_investment_performance.html.erb @@ -119,7 +119,7 @@
<% investment_metrics[:accounts].each do |account| %> -
+ <%= link_to account_path(account), class: "bg-container-inset rounded-lg p-4 flex items-center justify-between hover:bg-container-hover transition-colors" do %>
<%= render "accounts/logo", account: account, size: "sm" %>
@@ -128,7 +128,7 @@

<%= format_money(account.balance_money) %>

-
+ <% end %> <% end %>
diff --git a/config/locales/views/holdings/en.yml b/config/locales/views/holdings/en.yml index 10b648965..0e6d588d4 100644 --- a/config/locales/views/holdings/en.yml +++ b/config/locales/views/holdings/en.yml @@ -5,10 +5,30 @@ en: brokerage_cash: Brokerage cash destroy: success: Holding deleted + update: + success: Cost basis saved. + error: Invalid cost basis value. + unlock_cost_basis: + success: Cost basis unlocked. It may be updated on next sync. + cost_basis_sources: + manual: User set + calculated: From trades + provider: From provider + cost_basis_cell: + unknown: "--" + set_cost_basis_header: "Set cost basis for %{ticker} (%{qty} shares)" + total_cost_basis_label: Total cost basis + or_per_share_label: "Or enter per share:" + per_share: per share + cancel: Cancel + save: Save + overwrite_confirm_title: Overwrite cost basis? + overwrite_confirm_body: "This will replace the current cost basis of %{current}." holding: per_share: per share shares: "%{qty} shares" unknown: "--" + no_cost_basis: No cost basis index: average_cost: Average cost holdings: Holdings @@ -36,3 +56,8 @@ en: trade_history_entry: "%{qty} shares of %{security} at %{price}" total_return_label: Total Return unknown: Unknown + cost_basis_locked_label: Cost basis is locked + cost_basis_locked_description: Your manually set cost basis won't be changed by syncs. + unlock_cost_basis: Unlock + unlock_confirm_title: Unlock cost basis? + unlock_confirm_body: This will allow the cost basis to be updated by provider syncs or trade calculations. diff --git a/config/routes.rb b/config/routes.rb index f6f6fdd66..ced0f676a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -168,7 +168,11 @@ Rails.application.routes.draw do resources :mappings, only: :update, module: :import end - resources :holdings, only: %i[index new show destroy] + resources :holdings, only: %i[index new show update destroy] do + member do + post :unlock_cost_basis + end + end resources :trades, only: %i[show new create update destroy] resources :valuations, only: %i[show new create update destroy] do post :confirm_create, on: :collection diff --git a/db/migrate/20260112011546_add_cost_basis_source_tracking_to_holdings.rb b/db/migrate/20260112011546_add_cost_basis_source_tracking_to_holdings.rb new file mode 100644 index 000000000..f805dfbe4 --- /dev/null +++ b/db/migrate/20260112011546_add_cost_basis_source_tracking_to_holdings.rb @@ -0,0 +1,6 @@ +class AddCostBasisSourceTrackingToHoldings < ActiveRecord::Migration[7.2] + def change + add_column :holdings, :cost_basis_source, :string + add_column :holdings, :cost_basis_locked, :boolean, default: false, null: false + end +end diff --git a/db/migrate/20260112065106_backfill_cost_basis_source_for_holdings.rb b/db/migrate/20260112065106_backfill_cost_basis_source_for_holdings.rb new file mode 100644 index 000000000..b81f92156 --- /dev/null +++ b/db/migrate/20260112065106_backfill_cost_basis_source_for_holdings.rb @@ -0,0 +1,42 @@ +class BackfillCostBasisSourceForHoldings < ActiveRecord::Migration[7.2] + disable_ddl_transaction! + + def up + # Backfill cost_basis_source for existing holdings that have cost_basis but no source + # This is safe - it only adds metadata, doesn't change actual cost_basis values + # Locks existing data by default to protect it - users can unlock if they want syncs to update + + say_with_time "Backfilling cost_basis_source for holdings" do + updated = 0 + + # Process in batches to avoid locking issues + Holding.where.not(cost_basis: nil) + .where(cost_basis_source: nil) + .where("cost_basis > 0") + .find_each do |holding| + # Heuristic: If holding's account has buy trades for this security, likely calculated + # Otherwise, likely from provider (SimpleFIN/Plaid/Lunchflow) + has_trades = holding.account.trades + .where(security_id: holding.security_id) + .where("qty > 0") + .exists? + + source = has_trades ? "calculated" : "provider" + + # Lock existing data to protect it - users can unlock via UI if they want syncs to update + holding.update_columns(cost_basis_source: source, cost_basis_locked: true) + updated += 1 + end + + updated + end + end + + def down + # Reversible: clear the source and unlock for holdings that were backfilled + # We can't know for sure which ones were backfilled vs manually set, + # but clearing all non-manual sources is safe since they'd be re-detected + Holding.where(cost_basis_source: %w[calculated provider]) + .update_all(cost_basis_source: nil, cost_basis_locked: false) + end +end diff --git a/db/schema.rb b/db/schema.rb index 6465a2c16..067d2c9f6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_01_10_122603) do +ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -49,6 +49,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_10_122603) do t.string "institution_name" t.string "institution_domain" t.text "notes" + t.jsonb "holdings_snapshot_data" + t.datetime "holdings_snapshot_at" t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type" t.index ["accountable_type"], name: "index_accounts_on_accountable_type" t.index ["currency"], name: "index_accounts_on_currency" @@ -340,12 +342,14 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_10_122603) do t.jsonb "locked_attributes", default: {} t.string "external_id" t.string "source" + t.boolean "exclude_from_cashflow", default: false, null: false t.index "lower((name)::text)", name: "index_entries_on_lower_name" t.index ["account_id", "date"], name: "index_entries_on_account_id_and_date" t.index ["account_id", "source", "external_id"], name: "index_entries_on_account_source_and_external_id", unique: true, where: "((external_id IS NOT NULL) AND (source IS NOT NULL))" t.index ["account_id"], name: "index_entries_on_account_id" t.index ["date"], name: "index_entries_on_date" t.index ["entryable_type"], name: "index_entries_on_entryable_type" + t.index ["exclude_from_cashflow"], name: "index_entries_on_exclude_from_cashflow" t.index ["import_id"], name: "index_entries_on_import_id" end @@ -485,6 +489,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_10_122603) do t.string "external_id" t.decimal "cost_basis", precision: 19, scale: 4 t.uuid "account_provider_id" + t.string "cost_basis_source" + t.boolean "cost_basis_locked", default: false, null: false t.index ["account_id", "external_id"], name: "idx_holdings_on_account_id_external_id_unique", unique: true, where: "(external_id IS NOT NULL)" t.index ["account_id", "security_id", "date", "currency"], name: "idx_on_account_id_security_id_date_currency_5323e39f8b", unique: true t.index ["account_id"], name: "index_holdings_on_account_id" @@ -1125,7 +1131,14 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_10_122603) do t.string "currency" t.jsonb "locked_attributes", default: {} t.uuid "category_id" + t.decimal "realized_gain", precision: 19, scale: 4 + t.decimal "cost_basis_amount", precision: 19, scale: 4 + t.string "cost_basis_currency" + t.integer "holding_period_days" + t.string "realized_gain_confidence" + t.string "realized_gain_currency" t.index ["category_id"], name: "index_trades_on_category_id" + t.index ["realized_gain"], name: "index_trades_on_realized_gain_not_null", where: "(realized_gain IS NOT NULL)" t.index ["security_id"], name: "index_trades_on_security_id" end @@ -1138,9 +1151,11 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_10_122603) do t.string "kind", default: "standard", null: false t.string "external_id" t.jsonb "extra", default: {}, null: false + t.string "investment_activity_label" t.index ["category_id"], name: "index_transactions_on_category_id" t.index ["external_id"], name: "index_transactions_on_external_id" t.index ["extra"], name: "index_transactions_on_extra", using: :gin + t.index ["investment_activity_label"], name: "index_transactions_on_investment_activity_label" t.index ["kind"], name: "index_transactions_on_kind" t.index ["merchant_id"], name: "index_transactions_on_merchant_id" end diff --git a/test/controllers/holdings_controller_test.rb b/test/controllers/holdings_controller_test.rb index 2e03b9f46..a73680d54 100644 --- a/test/controllers/holdings_controller_test.rb +++ b/test/controllers/holdings_controller_test.rb @@ -27,4 +27,38 @@ class HoldingsControllerTest < ActionDispatch::IntegrationTest assert_redirected_to account_path(@holding.account) assert_empty @holding.account.entries.where(entryable: @holding.account.trades.where(security: @holding.security)) end + + test "updates cost basis with total amount divided by qty" do + # Given: holding with 10 shares + @holding.update!(qty: 10, cost_basis: nil, cost_basis_source: nil, cost_basis_locked: false) + + # When: user submits total cost basis of $100 (should become $10 per share) + patch holding_path(@holding), params: { holding: { cost_basis: "100.00" } } + + # Redirects to account page holdings tab to refresh list + assert_redirected_to account_path(@holding.account, tab: "holdings") + @holding.reload + + # Then: cost_basis should be per-share ($10), not total + assert_equal 10.0, @holding.cost_basis.to_f + assert_equal "manual", @holding.cost_basis_source + assert @holding.cost_basis_locked? + end + + test "unlock_cost_basis removes lock" do + # Given: locked holding + @holding.update!(cost_basis: 50.0, cost_basis_source: "manual", cost_basis_locked: true) + + # When: user unlocks + post unlock_cost_basis_holding_path(@holding) + + # Redirects to account page holdings tab to refresh list + assert_redirected_to account_path(@holding.account, tab: "holdings") + @holding.reload + + # Then: lock is removed but cost_basis and source remain + assert_not @holding.cost_basis_locked? + assert_equal 50.0, @holding.cost_basis.to_f + assert_equal "manual", @holding.cost_basis_source + end end diff --git a/test/models/holding/cost_basis_reconciler_test.rb b/test/models/holding/cost_basis_reconciler_test.rb new file mode 100644 index 000000000..c8413b0c1 --- /dev/null +++ b/test/models/holding/cost_basis_reconciler_test.rb @@ -0,0 +1,171 @@ +require "test_helper" + +class Holding::CostBasisReconcilerTest < ActiveSupport::TestCase + setup do + @family = families(:empty) + @account = @family.accounts.create!( + name: "Test Investment", + balance: 20000, + currency: "USD", + accountable: Investment.new + ) + @security = securities(:aapl) + end + + test "new holding uses incoming cost_basis" do + result = Holding::CostBasisReconciler.reconcile( + existing_holding: nil, + incoming_cost_basis: BigDecimal("150"), + incoming_source: "provider" + ) + + assert result[:should_update] + assert_equal BigDecimal("150"), result[:cost_basis] + assert_equal "provider", result[:cost_basis_source] + end + + test "new holding with nil cost_basis gets nil source" do + result = Holding::CostBasisReconciler.reconcile( + existing_holding: nil, + incoming_cost_basis: nil, + incoming_source: "provider" + ) + + assert result[:should_update] + assert_nil result[:cost_basis] + assert_nil result[:cost_basis_source] + end + + test "locked holding is never overwritten" do + holding = @account.holdings.create!( + security: @security, + date: Date.current, + qty: 10, + price: 200, + amount: 2000, + currency: "USD", + cost_basis: BigDecimal("175"), + cost_basis_source: "manual", + cost_basis_locked: true + ) + + result = Holding::CostBasisReconciler.reconcile( + existing_holding: holding, + incoming_cost_basis: BigDecimal("200"), + incoming_source: "calculated" + ) + + assert_not result[:should_update] + assert_equal BigDecimal("175"), result[:cost_basis] + assert_equal "manual", result[:cost_basis_source] + end + + test "calculated overwrites provider" do + holding = @account.holdings.create!( + security: @security, + date: Date.current, + qty: 10, + price: 200, + amount: 2000, + currency: "USD", + cost_basis: BigDecimal("150"), + cost_basis_source: "provider", + cost_basis_locked: false + ) + + result = Holding::CostBasisReconciler.reconcile( + existing_holding: holding, + incoming_cost_basis: BigDecimal("175"), + incoming_source: "calculated" + ) + + assert result[:should_update] + assert_equal BigDecimal("175"), result[:cost_basis] + assert_equal "calculated", result[:cost_basis_source] + end + + test "provider does not overwrite calculated" do + holding = @account.holdings.create!( + security: @security, + date: Date.current, + qty: 10, + price: 200, + amount: 2000, + currency: "USD", + cost_basis: BigDecimal("175"), + cost_basis_source: "calculated", + cost_basis_locked: false + ) + + result = Holding::CostBasisReconciler.reconcile( + existing_holding: holding, + incoming_cost_basis: BigDecimal("150"), + incoming_source: "provider" + ) + + assert_not result[:should_update] + assert_equal BigDecimal("175"), result[:cost_basis] + assert_equal "calculated", result[:cost_basis_source] + end + + test "provider does not overwrite manual" do + holding = @account.holdings.create!( + security: @security, + date: Date.current, + qty: 10, + price: 200, + amount: 2000, + currency: "USD", + cost_basis: BigDecimal("175"), + cost_basis_source: "manual", + cost_basis_locked: false + ) + + result = Holding::CostBasisReconciler.reconcile( + existing_holding: holding, + incoming_cost_basis: BigDecimal("150"), + incoming_source: "provider" + ) + + assert_not result[:should_update] + assert_equal BigDecimal("175"), result[:cost_basis] + assert_equal "manual", result[:cost_basis_source] + end + + test "zero provider cost_basis treated as unknown" do + result = Holding::CostBasisReconciler.reconcile( + existing_holding: nil, + incoming_cost_basis: BigDecimal("0"), + incoming_source: "provider" + ) + + assert result[:should_update] + assert_nil result[:cost_basis] + assert_nil result[:cost_basis_source] + end + + test "nil incoming cost_basis does not overwrite existing" do + holding = @account.holdings.create!( + security: @security, + date: Date.current, + qty: 10, + price: 200, + amount: 2000, + currency: "USD", + cost_basis: BigDecimal("175"), + cost_basis_source: "provider", + cost_basis_locked: false + ) + + result = Holding::CostBasisReconciler.reconcile( + existing_holding: holding, + incoming_cost_basis: nil, + incoming_source: "calculated" + ) + + # Even though calculated > provider, nil incoming shouldn't overwrite existing value + assert_not result[:should_update] + assert_equal BigDecimal("175"), result[:cost_basis] + assert_equal "provider", result[:cost_basis_source] + end +end diff --git a/test/models/holding_test.rb b/test/models/holding_test.rb index a3f4e2039..7e49fe6ba 100644 --- a/test/models/holding_test.rb +++ b/test/models/holding_test.rb @@ -112,6 +112,132 @@ class HoldingTest < ActiveSupport::TestCase assert_equal Money.new(30), @amzn.trend.value end + # Cost basis source tracking tests + + test "cost_basis_replaceable_by? returns false when locked" do + @amzn.update!(cost_basis: 200, cost_basis_source: "manual", cost_basis_locked: true) + + assert_not @amzn.cost_basis_replaceable_by?("calculated") + assert_not @amzn.cost_basis_replaceable_by?("provider") + assert_not @amzn.cost_basis_replaceable_by?("manual") + end + + test "cost_basis_replaceable_by? respects priority hierarchy" do + # Provider data can be replaced by calculated or manual + @amzn.update!(cost_basis: 200, cost_basis_source: "provider", cost_basis_locked: false) + assert @amzn.cost_basis_replaceable_by?("calculated") + assert @amzn.cost_basis_replaceable_by?("manual") + assert_not @amzn.cost_basis_replaceable_by?("provider") + + # Calculated data can be replaced by manual only + @amzn.update!(cost_basis: 200, cost_basis_source: "calculated", cost_basis_locked: false) + assert @amzn.cost_basis_replaceable_by?("manual") + assert_not @amzn.cost_basis_replaceable_by?("calculated") + assert_not @amzn.cost_basis_replaceable_by?("provider") + + # Manual data when LOCKED cannot be replaced by anything + @amzn.update!(cost_basis: 200, cost_basis_source: "manual", cost_basis_locked: true) + assert_not @amzn.cost_basis_replaceable_by?("manual") + assert_not @amzn.cost_basis_replaceable_by?("calculated") + assert_not @amzn.cost_basis_replaceable_by?("provider") + + # Manual data when UNLOCKED can be replaced by calculated (enables recalculation) + @amzn.update!(cost_basis: 200, cost_basis_source: "manual", cost_basis_locked: false) + assert_not @amzn.cost_basis_replaceable_by?("manual") + assert @amzn.cost_basis_replaceable_by?("calculated") + assert_not @amzn.cost_basis_replaceable_by?("provider") + end + + test "set_manual_cost_basis! sets value and locks" do + @amzn.set_manual_cost_basis!(BigDecimal("175.50")) + + assert_equal BigDecimal("175.50"), @amzn.cost_basis + assert_equal "manual", @amzn.cost_basis_source + assert @amzn.cost_basis_locked? + end + + test "unlock_cost_basis! allows future updates" do + @amzn.set_manual_cost_basis!(BigDecimal("175.50")) + @amzn.unlock_cost_basis! + + assert_not @amzn.cost_basis_locked? + # Source remains manual but since unlocked, calculated could now overwrite + assert @amzn.cost_basis_replaceable_by?("calculated") + end + + test "cost_basis_source_label returns correct translation" do + @amzn.update!(cost_basis_source: "manual") + assert_equal I18n.t("holdings.cost_basis_sources.manual"), @amzn.cost_basis_source_label + + @amzn.update!(cost_basis_source: "calculated") + assert_equal I18n.t("holdings.cost_basis_sources.calculated"), @amzn.cost_basis_source_label + + @amzn.update!(cost_basis_source: "provider") + assert_equal I18n.t("holdings.cost_basis_sources.provider"), @amzn.cost_basis_source_label + + @amzn.update!(cost_basis_source: nil) + assert_nil @amzn.cost_basis_source_label + end + + test "cost_basis_known? returns true only when source and positive value exist" do + @amzn.update!(cost_basis: nil, cost_basis_source: nil) + assert_not @amzn.cost_basis_known? + + @amzn.update!(cost_basis: 200, cost_basis_source: nil) + assert_not @amzn.cost_basis_known? + + @amzn.update!(cost_basis: nil, cost_basis_source: "provider") + assert_not @amzn.cost_basis_known? + + @amzn.update!(cost_basis: 0, cost_basis_source: "provider") + assert_not @amzn.cost_basis_known? + + @amzn.update!(cost_basis: 200, cost_basis_source: "provider") + assert @amzn.cost_basis_known? + end + + # Precision and edge case tests + + test "cost_basis precision is maintained with fractional shares" do + @amzn.update!(qty: BigDecimal("0.123456")) + @amzn.set_manual_cost_basis!(BigDecimal("100.123456")) + @amzn.reload + + assert_in_delta 100.123456, @amzn.cost_basis.to_f, 0.0001 + end + + test "set_manual_cost_basis! with zero qty does not raise but saves the value" do + @amzn.update!(qty: 0) + @amzn.set_manual_cost_basis!(BigDecimal("100")) + + # Value is stored but effectively meaningless with zero qty + assert_equal BigDecimal("100"), @amzn.cost_basis + assert @amzn.cost_basis_locked? + end + + test "cost_basis_locked prevents all sources from overwriting" do + @amzn.set_manual_cost_basis!(BigDecimal("100")) + assert @amzn.cost_basis_locked? + + # Verify all sources are blocked when locked + assert_not @amzn.cost_basis_replaceable_by?("provider") + assert_not @amzn.cost_basis_replaceable_by?("calculated") + assert_not @amzn.cost_basis_replaceable_by?("manual") + + # Value should remain unchanged + assert_equal BigDecimal("100"), @amzn.cost_basis + end + + test "unlocked manual allows only calculated to replace" do + @amzn.set_manual_cost_basis!(BigDecimal("100")) + @amzn.unlock_cost_basis! + + assert_not @amzn.cost_basis_locked? + assert @amzn.cost_basis_replaceable_by?("calculated") + assert_not @amzn.cost_basis_replaceable_by?("provider") + assert_not @amzn.cost_basis_replaceable_by?("manual") + end + private def load_holdings From a8bdc4469be9b32a11f12b762bdeb8eb58ab6932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Mon, 12 Jan 2026 14:21:40 +0100 Subject: [PATCH 33/54] Add badges for DeepWiki and Dosu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added badges for DeepWiki and Dosu to README. Signed-off-by: Juan José Mata --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 48654668d..876a0f452 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/we-promise/sure) +[![Dosu](https://raw.githubusercontent.com/dosu-ai/assets/main/dosu-badge.svg)](https://app.dosu.dev/a72bdcfd-15f5-4edc-bd85-ea0daa6c3adc/ask) sure_shot From 25ac8223083dcdd23b66f3b7e3f6489b439e776a Mon Sep 17 00:00:00 2001 From: soky srm Date: Mon, 12 Jan 2026 14:40:30 +0100 Subject: [PATCH 34/54] Reports print functionality (#622) * Print initial impl * Try to keep the bigger section together * /* Tufte-inspired Print Report Styles */ * styling * I8n * Move print styling out. * FIX unrelated test ordering on line 53 - import.rows.first doesn't guarantee ordering. Without an explicit ORDER BY, the database may return rows in any order. * Update print-report.css * Update print.html.erb * pass data to view * Update index.html.erb * Fix ERB helpers * Update reports_helper.rb --- app/assets/tailwind/application.css | 1 + app/assets/tailwind/print-report.css | 296 ++++++++++++++++++++++ app/controllers/reports_controller.rb | 77 +++--- app/helpers/reports_helper.rb | 34 +++ app/views/layouts/print.html.erb | 28 +++ app/views/reports/index.html.erb | 74 +++--- app/views/reports/print.html.erb | 345 ++++++++++++++++++++++++++ config/locales/views/reports/en.yml | 55 ++++ config/routes.rb | 1 + test/models/import_encoding_test.rb | 7 +- 10 files changed, 852 insertions(+), 66 deletions(-) create mode 100644 app/assets/tailwind/print-report.css create mode 100644 app/helpers/reports_helper.rb create mode 100644 app/views/layouts/print.html.erb create mode 100644 app/views/reports/print.html.erb diff --git a/app/assets/tailwind/application.css b/app/assets/tailwind/application.css index 938f57d5c..c7439e481 100644 --- a/app/assets/tailwind/application.css +++ b/app/assets/tailwind/application.css @@ -12,6 +12,7 @@ @import "./google-sign-in.css"; @import "./date-picker-dark-mode.css"; +@import "./print-report.css"; @layer components { .pcr-app{ diff --git a/app/assets/tailwind/print-report.css b/app/assets/tailwind/print-report.css new file mode 100644 index 000000000..1917349a9 --- /dev/null +++ b/app/assets/tailwind/print-report.css @@ -0,0 +1,296 @@ +/* + Print Report Styles + Tufte-inspired styling for the printable financial report. + Uses design system tokens where applicable. +*/ + +/* Print Body & Container */ +.print-body { + background: var(--color-white); + color: var(--color-gray-900); + font-family: var(--font-sans); + line-height: 1.5; +} + +.print-container { + max-width: 680px; + margin: 0 auto; + padding: 32px 24px; +} + +.tufte-report { + font-size: 11px; + color: var(--color-gray-900); +} + +/* Header */ +.tufte-header { + margin-bottom: 24px; + padding-bottom: 12px; + border-bottom: 2px solid var(--color-gray-900); +} + +.tufte-title { + font-size: 20px; + font-weight: 700; + margin: 0 0 4px 0; + color: var(--color-gray-900); + letter-spacing: -0.3px; +} + +.tufte-period { + display: block; + font-size: 13px; + font-weight: 500; + color: var(--color-gray-600); + margin-top: 2px; +} + +.tufte-meta { + font-size: 10px; + color: var(--color-gray-500); + margin: 8px 0 0 0; +} + +/* Sections */ +.tufte-section { + margin-bottom: 24px; +} + +.tufte-section-title { + font-size: 11px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 1px; + color: var(--color-gray-900); + margin: 0 0 12px 0; + border-bottom: 1px solid var(--color-gray-200); + padding-bottom: 6px; +} + +.tufte-subsection { + font-size: 11px; + font-weight: 600; + margin: 16px 0 8px 0; + padding-bottom: 4px; + border-bottom: 1px solid var(--color-gray-100); +} + +/* Metric Cards */ +.tufte-metric-card { + display: inline-block; + min-width: 100px; +} + +.tufte-metric-card-main { + display: block; +} + +.tufte-metric-card-label { + display: block; + font-size: 10px; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--color-gray-500); + margin-bottom: 4px; +} + +.tufte-metric-card-value { + display: block; + font-size: 20px; + font-weight: 700; + font-variant-numeric: tabular-nums; + line-height: 1.1; + letter-spacing: -0.5px; +} + +.tufte-metric-card-change { + display: inline-block; + font-size: 10px; + font-weight: 500; + margin-top: 4px; + padding: 1px 4px; + border-radius: 2px; +} + +.tufte-metric-card-sm .tufte-metric-card-value { + font-size: 16px; +} + +.tufte-metric-card-sm .tufte-metric-card-label { + font-size: 9px; +} + +/* Metric Row (horizontal layout) */ +.tufte-metric-row { + display: flex; + gap: 32px; + flex-wrap: wrap; + margin-bottom: 8px; +} + +/* Semantic Colors */ +.tufte-income { color: var(--color-green-700); } +.tufte-expense { color: var(--color-red-700); } +.tufte-muted { color: var(--color-gray-500); font-size: 10px; } +.tufte-up { color: var(--color-green-700); } +.tufte-down { color: var(--color-red-700); } + +/* Two Column Layout */ +.tufte-two-col { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 32px; + margin-top: 12px; +} + +/* Tables - Clean, readable style */ +.tufte-table { + width: 100%; + border-collapse: collapse; + font-size: 11px; + font-variant-numeric: tabular-nums; +} + +.tufte-table thead th { + text-align: left; + font-weight: 600; + font-size: 9px; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--color-gray-600); + padding: 8px 12px 8px 0; + border-bottom: 2px solid var(--color-gray-900); +} + +.tufte-table tbody td { + padding: 6px 12px 6px 0; + border-bottom: 1px solid var(--color-gray-200); + vertical-align: middle; +} + +.tufte-table tbody tr:last-child td { + border-bottom: none; +} + +.tufte-table tfoot td { + padding: 8px 12px 6px 0; + border-top: 2px solid var(--color-gray-900); + font-weight: 600; +} + +.tufte-table.tufte-compact thead th { + padding: 6px 8px 6px 0; +} + +.tufte-table.tufte-compact tbody td { + padding: 5px 8px 5px 0; +} + +.tufte-right { + text-align: right; + padding-right: 0 !important; +} + +.tufte-highlight { + background: var(--color-yellow-100); +} + +.tufte-highlight td:first-child { + font-weight: 600; +} + +/* Category Dots */ +.tufte-dot { + display: inline-block; + width: 8px; + height: 8px; + border-radius: 50%; + margin-right: 6px; + vertical-align: middle; +} + +/* Footnotes */ +.tufte-footnote { + font-size: 10px; + color: var(--color-gray-500); + margin-top: 8px; + font-style: italic; +} + +/* Footer */ +.tufte-footer { + margin-top: 32px; + padding-top: 12px; + border-top: 1px solid var(--color-gray-200); + font-size: 10px; + color: var(--color-gray-500); + text-align: center; +} + +/* Print-specific overrides */ +@media print { + @page { + size: A4; + margin: 15mm 18mm; + } + + /* Scoped to .print-body to avoid affecting other pages when printing */ + .print-body { + font-size: 10px; + -webkit-print-color-adjust: exact; + print-color-adjust: exact; + } + + .print-container { + max-width: none; + padding: 0; + } + + .tufte-section { + page-break-inside: auto; + } + + .tufte-section-title { + page-break-after: avoid; + } + + .tufte-table { + page-break-inside: auto; + } + + .tufte-table thead { + display: table-header-group; + } + + .tufte-table tr { + page-break-inside: avoid; + } + + .tufte-two-col { + page-break-inside: avoid; + } + + .tufte-keep-together { + page-break-inside: avoid; + } + + .tufte-header { + page-break-after: avoid; + } + + /* Force colors in print */ + .tufte-income { color: var(--color-green-700) !important; } + .tufte-expense { color: var(--color-red-700) !important; } + .tufte-up { color: var(--color-green-700) !important; } + .tufte-down { color: var(--color-red-700) !important; } + + .tufte-footer { + page-break-before: avoid; + } + + .tufte-highlight { + background: var(--color-yellow-100) !important; + } +} diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb index bfb833acb..91593f801 100644 --- a/app/controllers/reports_controller.rb +++ b/app/controllers/reports_controller.rb @@ -7,38 +7,7 @@ class ReportsController < ApplicationController before_action :authenticate_for_export, only: :export_transactions def index - @period_type = params[:period_type]&.to_sym || :monthly - @start_date = parse_date_param(:start_date) || default_start_date - @end_date = parse_date_param(:end_date) || default_end_date - - # Validate and fix date range if end_date is before start_date - validate_and_fix_date_range(show_flash: true) - - # Build the period - @period = Period.custom(start_date: @start_date, end_date: @end_date) - @previous_period = build_previous_period - - # Get aggregated data - @current_income_totals = Current.family.income_statement.income_totals(period: @period) - @current_expense_totals = Current.family.income_statement.expense_totals(period: @period) - - @previous_income_totals = Current.family.income_statement.income_totals(period: @previous_period) - @previous_expense_totals = Current.family.income_statement.expense_totals(period: @previous_period) - - # Calculate summary metrics - @summary_metrics = build_summary_metrics - - # Build trend data (last 6 months) - @trends_data = build_trends_data - - # Net worth metrics - @net_worth_metrics = build_net_worth_metrics - - # Transactions breakdown - @transactions = build_transactions_breakdown - - # Investment metrics (must be before build_reports_sections) - @investment_metrics = build_investment_metrics + setup_report_data(show_flash: true) # Build reports sections for collapsible/reorderable UI @reports_sections = build_reports_sections @@ -46,6 +15,12 @@ class ReportsController < ApplicationController @breadcrumbs = [ [ "Home", root_path ], [ "Reports", nil ] ] end + def print + setup_report_data(show_flash: false) + + render layout: "print" + end + def update_preferences if Current.user.update_reports_preferences(preferences_params) head :ok @@ -114,6 +89,44 @@ class ReportsController < ApplicationController end private + def setup_report_data(show_flash: false) + @period_type = params[:period_type]&.to_sym || :monthly + @start_date = parse_date_param(:start_date) || default_start_date + @end_date = parse_date_param(:end_date) || default_end_date + + # Validate and fix date range if end_date is before start_date + validate_and_fix_date_range(show_flash: show_flash) + + # Build the period + @period = Period.custom(start_date: @start_date, end_date: @end_date) + @previous_period = build_previous_period + + # Get aggregated data + @current_income_totals = Current.family.income_statement.income_totals(period: @period) + @current_expense_totals = Current.family.income_statement.expense_totals(period: @period) + + @previous_income_totals = Current.family.income_statement.income_totals(period: @previous_period) + @previous_expense_totals = Current.family.income_statement.expense_totals(period: @previous_period) + + # Calculate summary metrics + @summary_metrics = build_summary_metrics + + # Build trend data (last 6 months) + @trends_data = build_trends_data + + # Net worth metrics + @net_worth_metrics = build_net_worth_metrics + + # Transactions breakdown + @transactions = build_transactions_breakdown + + # Investment metrics + @investment_metrics = build_investment_metrics + + # Flags for view rendering + @has_accounts = Current.family.accounts.any? + end + def preferences_params prefs = params.require(:preferences) {}.tap do |permitted| diff --git a/app/helpers/reports_helper.rb b/app/helpers/reports_helper.rb new file mode 100644 index 000000000..52b3016b5 --- /dev/null +++ b/app/helpers/reports_helper.rb @@ -0,0 +1,34 @@ +module ReportsHelper + # Generate SVG polyline points for a sparkline chart + # Returns empty string if fewer than 2 data points (can't draw a line with 1 point) + def sparkline_points(values, width: 60, height: 16) + return "" if values.nil? || values.length < 2 || values.all? { |v| v.nil? || v.zero? } + + nums = values.map(&:to_f) + max_val = nums.max + min_val = nums.min + range = max_val - min_val + range = 1.0 if range.zero? + + points = nums.each_with_index.map do |val, i| + x = (i.to_f / [ nums.length - 1, 1 ].max) * width + y = height - ((val - min_val) / range * (height - 2)) - 1 + "#{x.round(1)},#{y.round(1)}" + end + + points.join(" ") + end + + # Calculate cumulative net values from trends data + def cumulative_net_values(trends) + return [] if trends.nil? + + running = 0 + trends.map { |t| running += t[:net].to_i; running } + end + + # Check if trends data has enough points for sparklines (need at least 2) + def has_sparkline_data?(trends_data) + trends_data&.length.to_i >= 2 + end +end diff --git a/app/views/layouts/print.html.erb b/app/views/layouts/print.html.erb new file mode 100644 index 000000000..582f139bc --- /dev/null +++ b/app/views/layouts/print.html.erb @@ -0,0 +1,28 @@ + + + + <%= content_for(:title) || t("reports.print.document_title") %> + + <%= csrf_meta_tags %> + <%= stylesheet_link_tag "tailwind", "data-turbo-track": "reload" %> + + + + + + +
+ <%= yield %> +
+ + + + diff --git a/app/views/reports/index.html.erb b/app/views/reports/index.html.erb index 57cd109ba..8940b7551 100644 --- a/app/views/reports/index.html.erb +++ b/app/views/reports/index.html.erb @@ -17,37 +17,49 @@ <% end %> <%# Period Navigation Tabs %> -
- <%= render DS::Link.new( - text: t("reports.index.periods.monthly"), - variant: @period_type == :monthly ? "secondary" : "ghost", - href: reports_path(period_type: :monthly), - size: :sm - ) %> - <%= render DS::Link.new( - text: t("reports.index.periods.quarterly"), - variant: @period_type == :quarterly ? "secondary" : "ghost", - href: reports_path(period_type: :quarterly), - size: :sm - ) %> - <%= render DS::Link.new( - text: t("reports.index.periods.ytd"), - variant: @period_type == :ytd ? "secondary" : "ghost", - href: reports_path(period_type: :ytd), - size: :sm - ) %> - <%= render DS::Link.new( - text: t("reports.index.periods.last_6_months"), - variant: @period_type == :last_6_months ? "secondary" : "ghost", - href: reports_path(period_type: :last_6_months), - size: :sm - ) %> - <%= render DS::Link.new( - text: t("reports.index.periods.custom"), - variant: @period_type == :custom ? "secondary" : "ghost", - href: reports_path(period_type: :custom), - size: :sm - ) %> +
+
+ <%= render DS::Link.new( + text: t("reports.index.periods.monthly"), + variant: @period_type == :monthly ? "secondary" : "ghost", + href: reports_path(period_type: :monthly), + size: :sm + ) %> + <%= render DS::Link.new( + text: t("reports.index.periods.quarterly"), + variant: @period_type == :quarterly ? "secondary" : "ghost", + href: reports_path(period_type: :quarterly), + size: :sm + ) %> + <%= render DS::Link.new( + text: t("reports.index.periods.ytd"), + variant: @period_type == :ytd ? "secondary" : "ghost", + href: reports_path(period_type: :ytd), + size: :sm + ) %> + <%= render DS::Link.new( + text: t("reports.index.periods.last_6_months"), + variant: @period_type == :last_6_months ? "secondary" : "ghost", + href: reports_path(period_type: :last_6_months), + size: :sm + ) %> + <%= render DS::Link.new( + text: t("reports.index.periods.custom"), + variant: @period_type == :custom ? "secondary" : "ghost", + href: reports_path(period_type: :custom), + size: :sm + ) %> +
+ + <%# Print Report Button %> + <%= link_to print_reports_path(period_type: @period_type, start_date: @start_date, end_date: @end_date), + target: "_blank", + rel: "noopener", + aria: { label: t("reports.index.print_report") }, + class: "inline-flex items-center gap-2 px-3 py-1.5 text-sm font-medium text-secondary bg-surface-inset hover:bg-surface-hover rounded-lg transition-colors flex-shrink-0" do %> + <%= icon("printer", size: "sm") %> + + <% end %>
<%# Custom Date Range Picker (only shown when custom is selected) %> diff --git a/app/views/reports/print.html.erb b/app/views/reports/print.html.erb new file mode 100644 index 000000000..3add245f0 --- /dev/null +++ b/app/views/reports/print.html.erb @@ -0,0 +1,345 @@ +<% content_for :title do %> + <%= t("reports.print.document_title") %> - <%= @start_date.strftime("%B %d, %Y") %> to <%= @end_date.strftime("%B %d, %Y") %> +<% end %> + +
+ <%# Header %> +
+

<%= t("reports.print.title") %>

+ <%= @start_date.strftime("%B %d, %Y") %> – <%= @end_date.strftime("%B %d, %Y") %> +

<%= Current.family.name %> · <%= t("reports.print.generated_on", date: Time.current.strftime("%B %d, %Y")) %>

+
+ + <%# Summary %> +
+

<%= t("reports.print.summary.title") %>

+
+
+ <%= t("reports.print.summary.income") %> + <%= @summary_metrics[:current_income].format %> + <% if @summary_metrics[:income_change] && @summary_metrics[:income_change] != 0 %> + "> + <%= t("reports.print.summary.vs_prior", percent: (@summary_metrics[:income_change] >= 0 ? "+#{@summary_metrics[:income_change]}" : @summary_metrics[:income_change].to_s)) %> + + <% end %> + <% if has_sparkline_data?(@trends_data) %> + + + + <% end %> +
+ +
+ <%= t("reports.print.summary.expenses") %> + <%= @summary_metrics[:current_expenses].format %> + <% if @summary_metrics[:expense_change] && @summary_metrics[:expense_change] != 0 %> + "> + <%= t("reports.print.summary.vs_prior", percent: (@summary_metrics[:expense_change] >= 0 ? "+#{@summary_metrics[:expense_change]}" : @summary_metrics[:expense_change].to_s)) %> + + <% end %> + <% if has_sparkline_data?(@trends_data) %> + + + + <% end %> +
+ +
+ <%= t("reports.print.summary.net_savings") %> + "><%= @summary_metrics[:net_savings].format %> + <% + # Calculate savings rate + savings_rate = @summary_metrics[:current_income].amount > 0 ? ((@summary_metrics[:net_savings].amount / @summary_metrics[:current_income].amount) * 100).round(0) : 0 + %> + <% if savings_rate != 0 %> + <%= t("reports.print.summary.of_income", percent: savings_rate) %> + <% end %> + <% if has_sparkline_data?(@trends_data) %> + + " stroke-width="1.5" /> + + <% end %> +
+ + <% if @summary_metrics[:budget_percent] %> +
+ <%= t("reports.print.summary.budget") %> + <%= @summary_metrics[:budget_percent] %>% + <%= t("reports.print.summary.used") %> +
+ <% end %> +
+
+ + <%# Net Worth %> + <% if @has_accounts %> +
+

<%= t("reports.print.net_worth.title") %>

+
+
+ <%= t("reports.print.net_worth.current_balance") %> + "> + <%= @net_worth_metrics[:current_net_worth].format %> + + <% if @net_worth_metrics[:trend] %> + + <%= @net_worth_metrics[:trend].value >= 0 ? "+" : "" %><%= @net_worth_metrics[:trend].value.format %> (<%= @net_worth_metrics[:trend].percent_formatted %>) <%= t("reports.print.net_worth.this_period") %> + + <% end %> + <% if has_sparkline_data?(@trends_data) %> + + " stroke-width="1.5" /> + + <% end %> +
+
+ +
+
+

<%= t("reports.print.net_worth.assets") %> <%= @net_worth_metrics[:total_assets].format %>

+ <% if @net_worth_metrics[:asset_groups].any? %> + + + <% @net_worth_metrics[:asset_groups].each do |group| %> + + + + + <% end %> + +
<%= group[:name] %><%= group[:total].format %>
+ <% end %> +
+
+

<%= t("reports.print.net_worth.liabilities") %> <%= @net_worth_metrics[:total_liabilities].format %>

+ <% if @net_worth_metrics[:liability_groups].any? %> + + + <% @net_worth_metrics[:liability_groups].each do |group| %> + + + + + <% end %> + +
<%= group[:name] %><%= group[:total].format %>
+ <% else %> +

<%= t("reports.print.net_worth.no_liabilities") %>

+ <% end %> +
+
+
+ <% end %> + + <%# Monthly Trends %> + <% if has_sparkline_data?(@trends_data) %> +
+

<%= t("reports.print.trends.title") %>

+ + + + + + + + + + + + <% @trends_data.each do |trend| %> + "> + + + + + + + <% end %> + + + <% + total_income = @trends_data.sum { |t| t[:income].to_d } + total_expenses = @trends_data.sum { |t| t[:expenses].to_d } + total_net = @trends_data.sum { |t| t[:net].to_d } + trends_count = @trends_data.length + avg_income = trends_count > 0 ? (total_income / trends_count) : 0 + avg_expenses = trends_count > 0 ? (total_expenses / trends_count) : 0 + avg_net = trends_count > 0 ? (total_net / trends_count) : 0 + overall_savings_rate = total_income > 0 ? ((total_net / total_income) * 100).round(0) : 0 + %> + + + + + + + + +
<%= t("reports.print.trends.month") %><%= t("reports.print.trends.income") %><%= t("reports.print.trends.expenses") %><%= t("reports.print.trends.net") %><%= t("reports.print.trends.savings_rate") %>
<%= trend[:month] %><%= trend[:is_current_month] ? " *" : "" %><%= Money.new(trend[:income], Current.family.currency).format %><%= Money.new(trend[:expenses], Current.family.currency).format %>"><%= Money.new(trend[:net], Current.family.currency).format %> + <% month_savings_rate = trend[:income] > 0 ? ((trend[:net].to_f / trend[:income].to_f) * 100).round(0) : 0 %> + <%= month_savings_rate %>% +
<%= t("reports.print.trends.average") %><%= Money.new(avg_income, Current.family.currency).format %><%= Money.new(avg_expenses, Current.family.currency).format %>"><%= Money.new(avg_net, Current.family.currency).format %><%= overall_savings_rate %>%
+ <% if @trends_data.any? { |t| t[:is_current_month] } %> +

<%= t("reports.print.trends.current_month_note") %>

+ <% end %> +
+ <% end %> + + <%# Investments %> + <% if @investment_metrics[:has_investments] %> +
+

<%= t("reports.print.investments.title") %>

+
+
+ <%= t("reports.print.investments.portfolio_value") %> + <%= format_money(@investment_metrics[:portfolio_value]) %> + <% if has_sparkline_data?(@trends_data) %> + + + + <% end %> +
+ <% if @investment_metrics[:unrealized_trend] %> +
+ <%= t("reports.print.investments.total_return") %> + + <%= @investment_metrics[:unrealized_trend].value >= 0 ? "+" : "" %><%= format_money(Money.new(@investment_metrics[:unrealized_trend].value, Current.family.currency)) %> + + + <%= @investment_metrics[:unrealized_trend].percent_formatted %> + +
+ <% end %> +
+ <%= t("reports.print.investments.contributions") %> + <%= format_money(@investment_metrics[:period_contributions]) %> + <%= t("reports.print.investments.this_period") %> +
+
+ <%= t("reports.print.investments.withdrawals") %> + <%= format_money(@investment_metrics[:period_withdrawals]) %> + <%= t("reports.print.investments.this_period") %> +
+
+ + <% if @investment_metrics[:top_holdings].any? %> +

<%= t("reports.print.investments.top_holdings") %>

+ + + + + + + + + + + <% @investment_metrics[:top_holdings].each do |holding| %> + + + + + + + <% end %> + +
<%= t("reports.print.investments.holding") %><%= t("reports.print.investments.weight") %><%= t("reports.print.investments.value") %><%= t("reports.print.investments.return") %>
<%= holding.ticker %> <%= truncate(holding.name, length: 25) %><%= number_to_percentage(holding.weight || 0, precision: 1) %><%= format_money(holding.amount_money) %> + <% if holding.trend %> + <%= holding.trend.percent_formatted %> + <% else %> + — + <% end %> +
+ <% end %> +
+ <% end %> + + <%# Spending by Category %> + <% if @transactions.any? %> +
+

<%= t("reports.print.spending.title") %>

+ <% + income_groups = @transactions.select { |g| g[:type] == "income" } + expense_groups = @transactions.select { |g| g[:type] == "expense" } + income_total = income_groups.sum { |g| g[:total] } + expense_total = expense_groups.sum { |g| g[:total] } + %> + +
+ <% if income_groups.any? %> +
+

<%= t("reports.print.spending.income") %> <%= Money.new(income_total, Current.family.currency).format %>

+ + + + + + + + + + <% income_groups.first(8).each do |group| %> + <% percentage = income_total.zero? ? 0 : (group[:total].to_f / income_total * 100).round(0) %> + + + + + + <% end %> + <% if income_groups.length > 8 %> + + + + + + <% end %> + +
<%= t("reports.print.spending.category") %><%= t("reports.print.spending.amount") %><%= t("reports.print.spending.percent") %>
+ + <%= group[:category_name] %> + <%= Money.new(group[:total], Current.family.currency).format %><%= percentage %>%
<%= t("reports.print.spending.more_categories", count: income_groups.length - 8) %>
+
+ <% end %> + + <% if expense_groups.any? %> +
+

<%= t("reports.print.spending.expenses") %> <%= Money.new(expense_total, Current.family.currency).format %>

+ + + + + + + + + + <% expense_groups.first(8).each do |group| %> + <% percentage = expense_total.zero? ? 0 : (group[:total].to_f / expense_total * 100).round(0) %> + + + + + + <% end %> + <% if expense_groups.length > 8 %> + + + + + + <% end %> + +
<%= t("reports.print.spending.category") %><%= t("reports.print.spending.amount") %><%= t("reports.print.spending.percent") %>
+ + <%= group[:category_name] %> + <%= Money.new(group[:total], Current.family.currency).format %><%= percentage %>%
<%= t("reports.print.spending.more_categories", count: expense_groups.length - 8) %>
+
+ <% end %> +
+
+ <% end %> + +
+ <%= product_name %> · <%= @start_date.strftime("%B %Y") %> – <%= @end_date.strftime("%B %Y") %> +
+
diff --git a/config/locales/views/reports/en.yml b/config/locales/views/reports/en.yml index 2d34539f7..ceebbe74c 100644 --- a/config/locales/views/reports/en.yml +++ b/config/locales/views/reports/en.yml @@ -5,6 +5,7 @@ en: title: Reports subtitle: Comprehensive insights into your financial health export: Export CSV + print_report: Print Report drag_to_reorder: "Drag to reorder section" toggle_section: "Toggle section visibility" periods: @@ -149,3 +150,57 @@ en: open_sheets: Open Google Sheets go_to_api_keys: Go to API Keys close: Got it + print: + document_title: Financial Report + title: Financial Report + generated_on: "Generated %{date}" + # Summary section + summary: + title: Summary + income: Income + expenses: Expenses + net_savings: Net Savings + budget: Budget + vs_prior: "%{percent}% vs prior" + of_income: "%{percent}% of income" + used: used + # Net Worth section + net_worth: + title: Net Worth + current_balance: Current Balance + this_period: this period + assets: Assets + liabilities: Liabilities + no_liabilities: No liabilities + # Monthly Trends section + trends: + title: Monthly Trends + month: Month + income: Income + expenses: Expenses + net: Net + savings_rate: Savings Rate + average: Average + current_month_note: "* Current month (partial data)" + # Investments section + investments: + title: Investments + portfolio_value: Portfolio Value + total_return: Total Return + contributions: Contributions + withdrawals: Withdrawals + this_period: this period + top_holdings: Top Holdings + holding: Holding + weight: Weight + value: Value + return: Return + # Spending by Category section + spending: + title: Spending by Category + income: Income + expenses: Expenses + category: Category + amount: Amount + percent: "%" + more_categories: "+ %{count} more categories" diff --git a/config/routes.rb b/config/routes.rb index ced0f676a..b2c7b84d0 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -135,6 +135,7 @@ Rails.application.routes.draw do patch :update_preferences, on: :collection get :export_transactions, on: :collection get :google_sheets_instructions, on: :collection + get :print, on: :collection end resources :budgets, only: %i[index show edit update], param: :month_year do diff --git a/test/models/import_encoding_test.rb b/test/models/import_encoding_test.rb index 0052e8dfc..7770da992 100644 --- a/test/models/import_encoding_test.rb +++ b/test/models/import_encoding_test.rb @@ -50,9 +50,10 @@ class ImportEncodingTest < ActiveSupport::TestCase assert_equal 3, import.rows_count, "Expected 3 data rows" # Verify Polish characters were preserved correctly - first_row = import.rows.first - assert_not_nil first_row, "Expected first row to exist" - assert_includes first_row.name, "spożywczy", "Polish characters should be preserved" + # Check that any row contains the Polish characters (test is about encoding, not ordering) + assert import.rows.any? { |row| row.name&.include?("spożywczy") }, "Polish characters should be preserved" + # Also verify other Polish characters from different rows + assert import.rows.any? { |row| row.name&.include?("Café") }, "Extended Latin characters should be preserved" end test "handles UTF-8 files without modification" do From 30923b72963e51618b89484b98c2ed8a08c422b9 Mon Sep 17 00:00:00 2001 From: sokie Date: Mon, 12 Jan 2026 15:05:28 +0100 Subject: [PATCH 35/54] FIX address comments --- .../settings/hostings_controller.rb | 31 +++++++++++++++++-- app/models/setting.rb | 13 ++++++++ app/services/auto_sync_scheduler.rb | 22 +++++++++++-- config/locales/views/settings/hostings/en.yml | 2 ++ 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/app/controllers/settings/hostings_controller.rb b/app/controllers/settings/hostings_controller.rb index e3bf49850..dd4024d7b 100644 --- a/app/controllers/settings/hostings_controller.rb +++ b/app/controllers/settings/hostings_controller.rb @@ -62,14 +62,27 @@ class Settings::HostingsController < ApplicationController Setting.syncs_include_pending = hosting_params[:syncs_include_pending] == "1" end + sync_settings_changed = false + if hosting_params.key?(:auto_sync_enabled) Setting.auto_sync_enabled = hosting_params[:auto_sync_enabled] == "1" - AutoSyncScheduler.sync! + sync_settings_changed = true end if hosting_params.key?(:auto_sync_time) - Setting.auto_sync_time = hosting_params[:auto_sync_time] - AutoSyncScheduler.sync! + time_value = hosting_params[:auto_sync_time] + unless Setting.valid_auto_sync_time?(time_value) + flash[:alert] = t(".invalid_sync_time") + return redirect_to settings_hosting_path + end + + Setting.auto_sync_time = time_value + Setting.auto_sync_timezone = current_user_timezone + sync_settings_changed = true + end + + if sync_settings_changed + sync_auto_sync_scheduler! end if hosting_params.key?(:openai_access_token) @@ -119,4 +132,16 @@ class Settings::HostingsController < ApplicationController def ensure_admin redirect_to settings_hosting_path, alert: t(".not_authorized") unless Current.user.admin? end + + def sync_auto_sync_scheduler! + AutoSyncScheduler.sync! + rescue StandardError => error + Rails.logger.error("[AutoSyncScheduler] Failed to sync scheduler: #{error.message}") + Rails.logger.error(error.backtrace.join("\n")) + flash[:alert] = t(".scheduler_sync_failed") + end + + def current_user_timezone + Current.family&.timezone.presence || "UTC" + end end diff --git a/app/models/setting.rb b/app/models/setting.rb index c28c611f3..4407d7293 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -26,6 +26,19 @@ class Setting < RailsSettings::Base field :syncs_include_pending, type: :boolean, default: SYNCS_INCLUDE_PENDING_DEFAULT field :auto_sync_enabled, type: :boolean, default: ENV.fetch("AUTO_SYNC_ENABLED", "1") == "1" field :auto_sync_time, type: :string, default: ENV.fetch("AUTO_SYNC_TIME", "02:22") + field :auto_sync_timezone, type: :string, default: ENV.fetch("AUTO_SYNC_TIMEZONE", "UTC") + + AUTO_SYNC_TIME_FORMAT = /\A([01]?\d|2[0-3]):([0-5]\d)\z/ + + def self.valid_auto_sync_time?(time_str) + return false if time_str.blank? + AUTO_SYNC_TIME_FORMAT.match?(time_str.to_s.strip) + end + + def self.valid_auto_sync_timezone?(timezone_str) + return false if timezone_str.blank? + ActiveSupport::TimeZone[timezone_str].present? + end # Dynamic fields are now stored as individual entries with "dynamic:" prefix # This prevents race conditions and ensures each field is independently managed diff --git a/app/services/auto_sync_scheduler.rb b/app/services/auto_sync_scheduler.rb index a6bbbdd74..44adea3b0 100644 --- a/app/services/auto_sync_scheduler.rb +++ b/app/services/auto_sync_scheduler.rb @@ -12,20 +12,36 @@ class AutoSyncScheduler def self.upsert_job time_str = Setting.auto_sync_time || "02:22" - hour, minute = time_str.split(":").map(&:to_i) + timezone_str = Setting.auto_sync_timezone || "UTC" - local_time = Time.zone.now.change(hour: hour, min: minute, sec: 0) + unless Setting.valid_auto_sync_time?(time_str) + Rails.logger.error("[AutoSyncScheduler] Invalid time format: #{time_str}, using default 02:22") + time_str = "02:22" + end + + hour, minute = time_str.split(":").map(&:to_i) + timezone = ActiveSupport::TimeZone[timezone_str] || ActiveSupport::TimeZone["UTC"] + local_time = timezone.now.change(hour: hour, min: minute, sec: 0) utc_time = local_time.utc cron = "#{utc_time.min} #{utc_time.hour} * * *" - Sidekiq::Cron::Job.create( + job = Sidekiq::Cron::Job.create( name: JOB_NAME, cron: cron, class: "SyncAllJob", queue: "scheduled", description: "Syncs all accounts for all families" ) + + if job.nil? || (job.respond_to?(:valid?) && !job.valid?) + error_msg = job.respond_to?(:errors) ? job.errors.to_a.join(", ") : "unknown error" + Rails.logger.error("[AutoSyncScheduler] Failed to create cron job: #{error_msg}") + raise StandardError, "Failed to create sync schedule: #{error_msg}" + end + + Rails.logger.info("[AutoSyncScheduler] Created cron job with schedule: #{cron} (#{time_str} #{timezone_str})") + job end def self.remove_job diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index aaa7246d6..4742bc809 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -75,6 +75,8 @@ en: failure: Invalid setting value success: Settings updated invalid_onboarding_state: Invalid onboarding state + invalid_sync_time: Invalid sync time format. Please use HH:MM format (e.g., 02:30). + scheduler_sync_failed: Settings saved, but failed to update the sync schedule. Please try again or check the server logs. clear_cache: cache_cleared: Data cache has been cleared. This may take a few moments to complete. not_authorized: You are not authorized to perform this action From ebdaafaf954e6c31ad2b61bd39c1d3a9ef03a4d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Mon, 12 Jan 2026 15:13:19 +0000 Subject: [PATCH 36/54] Remove deprecated code --- app/services/simplefin_item/unlinker.rb | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 app/services/simplefin_item/unlinker.rb diff --git a/app/services/simplefin_item/unlinker.rb b/app/services/simplefin_item/unlinker.rb deleted file mode 100644 index d676af999..000000000 --- a/app/services/simplefin_item/unlinker.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -# DEPRECATED: This thin wrapper remains only for backward compatibility. -# Business logic has moved into `SimplefinItem::Unlinking` (model concern). -# Prefer calling `item.unlink_all!(dry_run: ...)` directly. -class SimplefinItem::Unlinker - attr_reader :item, :dry_run - - def initialize(item, dry_run: false) - @item = item - @dry_run = dry_run - end - - def unlink_all! - item.unlink_all!(dry_run: dry_run) - end -end From 8b6392e1d136f4761696fd8f8771a1ec30ba1159 Mon Sep 17 00:00:00 2001 From: Andrei Onel Date: Mon, 12 Jan 2026 16:32:19 +0000 Subject: [PATCH 37/54] Updated documentation for release v0.6.7-alpha.1 (#563) * Update charts/sure/values.yaml * Update charts/sure/README.md * Update charts/sure/CHANGELOG.md * Revert changelog changes Signed-off-by: Andrei Onel * Update documentation to use nested strategy.rollingUpdate Signed-off-by: Andrei Onel * Update values.yaml to use nested strategy.rollingUpdate Signed-off-by: Andrei Onel --------- Signed-off-by: Andrei Onel --- charts/sure/README.md | 76 ++++++++++++++++++++++++++++++++++++++--- charts/sure/values.yaml | 28 +++++++++++++++ 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/charts/sure/README.md b/charts/sure/README.md index 17c6d64b0..a1dfb8903 100644 --- a/charts/sure/README.md +++ b/charts/sure/README.md @@ -310,6 +310,74 @@ Security note on label selectors: - CNPG: `cnpg.io/cluster: ` (CNPG labels its pods) - RedisReplication: `app.kubernetes.io/instance: ` or `app.kubernetes.io/name: ` +#### Rolling update strategy + +When using topology spread constraints with `whenUnsatisfiable: DoNotSchedule`, you must configure the Kubernetes rolling update strategy to prevent deployment deadlocks. + +The chart now makes the rolling update strategy configurable for web and worker deployments. The defaults have been changed from Kubernetes defaults (`maxUnavailable=0`, `maxSurge=25%`) to: + +```yaml +web: + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + +worker: + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 +``` + +**Why these defaults?** + +With `maxSurge=0`, Kubernetes will terminate an old pod before creating a new one. This ensures that when all nodes are occupied (due to strict topology spreading), there is always space for the new pod to be scheduled. + +If you use `maxSurge > 0` with `DoNotSchedule` topology constraints and all nodes are occupied, Kubernetes cannot create the new pod (no space available) and cannot terminate the old pod (new pod must be ready first), resulting in a deployment deadlock. + +**Configuration examples:** + +For faster rollouts when not using strict topology constraints: + +```yaml +web: + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + +worker: + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 +``` + +For HA setups with topology spreading: + +```yaml +web: + replicas: 3 + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: sure + app.kubernetes.io/component: web +``` + +**Warning:** Using `maxSurge > 0` with `whenUnsatisfiable: DoNotSchedule` can cause deployment deadlocks when all nodes are occupied. If you need faster rollouts, either: +- Use `whenUnsatisfiable: ScheduleAnyway` instead of `DoNotSchedule` +- Ensure you have spare capacity on your nodes +- Keep `maxSurge: 0` and accept slower rollouts + Compatibility: - CloudNativePG v1.27.1 supports `minSyncReplicas`/`maxSyncReplicas` and standard k8s scheduling fields under `spec`. - OT redis-operator v0.21.0 supports scheduling under `spec.kubernetesConfig`. @@ -370,13 +438,13 @@ stringData: # password: "__SET_SECRET__" ``` -Note: These are non-sensitive placeholder values. Do not commit real secrets to version control. Prefer External Secrets, Sealed Secrets, or your platform’s secret manager to source these at runtime. +Note: These are non-sensitive placeholder values. Do not commit real secrets to version control. Prefer External Secrets, Sealed Secrets, or your platform's secret manager to source these at runtime. ### Linting Helm templates and YAML Helm template files under `charts/**/templates/**` contain template delimiters like `{{- ... }}` that raw YAML linters will flag as invalid. To avoid false positives in CI: -- Use Helm’s linter for charts: +- Use Helm's linter for charts: - `helm lint charts/sure` - Configure your YAML linter (e.g., yamllint) to ignore Helm template directories (exclude `charts/**/templates/**`), or use a Helm-aware plugin that preprocesses templates before linting. @@ -588,7 +656,7 @@ See `values.yaml` for the complete configuration surface, including: - `redis-ha.*`: enable dandydev/redis-ha subchart and configure replicas/auth (Sentinel/HA); supports `existingSecret` and `existingSecretPasswordKey` - `redisOperator.*`: optionally install OT redis-operator (`redisOperator.enabled`) and/or render a `RedisSentinel` CR (`redisOperator.managed.enabled`); configure `name`, `replicas`, `auth.existingSecret/passwordKey`, `persistence.className/size`, scheduling knobs, and `operator.resources` (controller) / `workloadResources` (Redis pods) - `redisSimple.*`: optional single‑pod Redis (non‑HA) when `redis-ha.enabled=false` -- `web.*`, `worker.*`: replicas, probes, resources, scheduling +- `web.*`, `worker.*`: replicas, probes, resources, scheduling, **strategy** (rolling update configuration) - `migrations.*`: strategy job or initContainer - `simplefin.encryption.*`: enable + backfill options - `cronjobs.*`: custom CronJobs @@ -635,7 +703,7 @@ helm uninstall sure -n sure ## Cleanup & reset (k3s) -For local k3s experimentation it’s sometimes useful to completely reset the `sure` namespace, especially if CR finalizers or PVCs get stuck. +For local k3s experimentation it's sometimes useful to completely reset the `sure` namespace, especially if CR finalizers or PVCs get stuck. The script below is a **last-resort tool** for cleaning the namespace. It: diff --git a/charts/sure/values.yaml b/charts/sure/values.yaml index 23b5ad8d7..d2fb97eba 100644 --- a/charts/sure/values.yaml +++ b/charts/sure/values.yaml @@ -266,6 +266,20 @@ web: # Optional command/args override command: [] args: [] + # Kubernetes rolling update strategy for the web Deployment. + # Controls how pods are replaced during updates. + # Default: maxUnavailable=1, maxSurge=0 + # This prevents deployment deadlocks when using topology spread constraints with DoNotSchedule. + # If you are not using strict topology constraints, you can increase maxSurge for faster rollouts. + # Example for faster rollouts (when not using DoNotSchedule): + # strategy: + # rollingUpdate: + # maxUnavailable: 0 + # maxSurge: 1 + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 resources: requests: cpu: 100m @@ -312,6 +326,20 @@ worker: # Optional command/args override for Sidekiq command: [] args: [] + # Kubernetes rolling update strategy for the worker Deployment. + # Controls how pods are replaced during updates. + # Default: maxUnavailable=1, maxSurge=0 + # This prevents deployment deadlocks when using topology spread constraints with DoNotSchedule. + # If you are not using strict topology constraints, you can increase maxSurge for faster rollouts. + # Example for faster rollouts (when not using DoNotSchedule): + # strategy: + # rollingUpdate: + # maxUnavailable: 0 + # maxSurge: 1 + strategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 resources: requests: cpu: 100m From 70e7a5f2d63d3197dae4f53515e645e17cc17aa0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:39:02 +0000 Subject: [PATCH 38/54] Initial plan From 52588784d0c1cad7fd0cd7baa6b6c512a8053bf4 Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Sat, 10 Jan 2026 19:48:04 -0500 Subject: [PATCH 39/54] Add investment activity detection, labels, and exclusions - Introduced `InvestmentActivityDetector` to mark internal investment activity as excluded from cashflow and assign appropriate labels. - Added `exclude_from_cashflow` flag to `entries` and `investment_activity_label` to `transactions` with migrations. - Implemented rake tasks to backfill and clear investment activity labels. - Updated `PlaidAccount::Investments::TransactionsProcessor` to map Plaid transaction types to labels. - Included comprehensive test coverage for new functionality. --- app/controllers/reports_controller.rb | 10 +- app/controllers/transactions_controller.rb | 4 +- app/models/account/provider_import_adapter.rb | 11 +- app/models/income_statement/category_stats.rb | 3 +- app/models/income_statement/family_stats.rb | 3 +- app/models/income_statement/totals.rb | 7 +- app/models/investment_activity_detector.rb | 346 ++++++++++++++++++ app/models/lunchflow_account/processor.rb | 27 ++ .../investments/transactions_processor.rb | 25 +- app/models/plaid_account/processor.rb | 40 ++ app/models/simplefin_account/processor.rb | 27 ++ app/models/simplefin_entry/processor.rb | 10 +- app/models/transaction.rb | 29 +- app/models/transaction/search.rb | 8 +- app/models/transfer.rb | 4 + app/views/transactions/_transaction.html.erb | 13 + app/views/transactions/show.html.erb | 53 ++- config/locales/views/transactions/en.yml | 11 + ...0120000_add_investment_cashflow_support.rb | 13 + ...vestment_activity_label_to_transactions.rb | 8 + db/schema.rb | 62 +--- lib/tasks/investment_labels.rake | 185 ++++++++++ test/models/income_statement_test.rb | 66 ++++ .../investment_activity_detector_test.rb | 299 +++++++++++++++ test/models/transaction_test.rb | 33 ++ test/models/transfer_test.rb | 20 + 26 files changed, 1235 insertions(+), 82 deletions(-) create mode 100644 app/models/investment_activity_detector.rb create mode 100644 db/migrate/20260110120000_add_investment_cashflow_support.rb create mode 100644 db/migrate/20260110180000_add_investment_activity_label_to_transactions.rb create mode 100644 lib/tasks/investment_labels.rake create mode 100644 test/models/investment_activity_detector_test.rb diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb index 91593f801..842ff50e8 100644 --- a/app/controllers/reports_controller.rb +++ b/app/controllers/reports_controller.rb @@ -338,7 +338,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Transaction", excluded: false, date: @period.date_range }) + .where(entries: { entryable_type: "Transaction", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) .where.not(kind: [ "funds_movement", "one_time", "cc_payment" ]) .includes(entry: :account, category: []) @@ -350,7 +350,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Trade", excluded: false, date: @period.date_range }) + .where(entries: { entryable_type: "Trade", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) .includes(entry: :account, category: []) # Get sort parameters @@ -519,7 +519,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Transaction", excluded: false, date: @period.date_range }) + .where(entries: { entryable_type: "Transaction", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) .where.not(kind: [ "funds_movement", "one_time", "cc_payment" ]) .includes(entry: :account, category: []) @@ -556,7 +556,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Transaction", excluded: false, date: @period.date_range }) + .where(entries: { entryable_type: "Transaction", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) .where.not(kind: [ "funds_movement", "one_time", "cc_payment" ]) .includes(entry: :account, category: []) @@ -567,7 +567,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Trade", excluded: false, date: @period.date_range }) + .where(entries: { entryable_type: "Trade", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) .includes(entry: :account, category: []) # Group by category, type, and month diff --git a/app/controllers/transactions_controller.rb b/app/controllers/transactions_controller.rb index a0fc9aaee..03dcbc2db 100644 --- a/app/controllers/transactions_controller.rb +++ b/app/controllers/transactions_controller.rb @@ -217,8 +217,8 @@ class TransactionsController < ApplicationController def entry_params entry_params = params.require(:entry).permit( - :name, :date, :amount, :currency, :excluded, :notes, :nature, :entryable_type, - entryable_attributes: [ :id, :category_id, :merchant_id, :kind, { tag_ids: [] } ] + :name, :date, :amount, :currency, :excluded, :exclude_from_cashflow, :notes, :nature, :entryable_type, + entryable_attributes: [ :id, :category_id, :merchant_id, :kind, :investment_activity_label, { tag_ids: [] } ] ) nature = entry_params.delete(:nature) diff --git a/app/models/account/provider_import_adapter.rb b/app/models/account/provider_import_adapter.rb index e74d1f5e6..31d65a6ae 100644 --- a/app/models/account/provider_import_adapter.rb +++ b/app/models/account/provider_import_adapter.rb @@ -18,8 +18,9 @@ class Account::ProviderImportAdapter # @param notes [String, nil] Optional transaction notes/memo # @param pending_transaction_id [String, nil] Plaid's linking ID for pending→posted reconciliation # @param extra [Hash, nil] Optional provider-specific metadata to merge into transaction.extra + # @param investment_activity_label [String, nil] Optional activity type label (e.g., "Buy", "Dividend") # @return [Entry] The created or updated entry - def import_transaction(external_id:, amount:, currency:, date:, name:, source:, category_id: nil, merchant: nil, notes: nil, pending_transaction_id: nil, extra: nil) + def import_transaction(external_id:, amount:, currency:, date:, name:, source:, category_id: nil, merchant: nil, notes: nil, pending_transaction_id: nil, extra: nil, investment_activity_label: nil) raise ArgumentError, "external_id is required" if external_id.blank? raise ArgumentError, "source is required" if source.blank? @@ -114,6 +115,14 @@ class Account::ProviderImportAdapter entry.transaction.extra = existing.deep_merge(incoming) entry.transaction.save! end + + # Set investment activity label if provided and not already set + if investment_activity_label.present? && entry.entryable.is_a?(Transaction) + if entry.transaction.investment_activity_label.blank? + entry.transaction.update!(investment_activity_label: investment_activity_label) + end + end + entry.save! # AFTER save: For NEW posted transactions, check for fuzzy matches to SUGGEST (not auto-claim) diff --git a/app/models/income_statement/category_stats.rb b/app/models/income_statement/category_stats.rb index 3bc91b839..31679a662 100644 --- a/app/models/income_statement/category_stats.rb +++ b/app/models/income_statement/category_stats.rb @@ -47,8 +47,9 @@ class IncomeStatement::CategoryStats er.to_currency = :target_currency ) WHERE a.family_id = :family_id - AND t.kind NOT IN ('funds_movement', 'one_time', 'cc_payment') + AND t.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false + AND ae.exclude_from_cashflow = false AND (t.extra -> 'simplefin' ->> 'pending')::boolean IS DISTINCT FROM true AND (t.extra -> 'plaid' ->> 'pending')::boolean IS DISTINCT FROM true GROUP BY c.id, period, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END diff --git a/app/models/income_statement/family_stats.rb b/app/models/income_statement/family_stats.rb index c2a3c8f8e..c96f13b35 100644 --- a/app/models/income_statement/family_stats.rb +++ b/app/models/income_statement/family_stats.rb @@ -44,8 +44,9 @@ class IncomeStatement::FamilyStats er.to_currency = :target_currency ) WHERE a.family_id = :family_id - AND t.kind NOT IN ('funds_movement', 'one_time', 'cc_payment') + AND t.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false + AND ae.exclude_from_cashflow = false AND (t.extra -> 'simplefin' ->> 'pending')::boolean IS DISTINCT FROM true AND (t.extra -> 'plaid' ->> 'pending')::boolean IS DISTINCT FROM true GROUP BY period, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END diff --git a/app/models/income_statement/totals.rb b/app/models/income_statement/totals.rb index 355212486..2b5bd07b7 100644 --- a/app/models/income_statement/totals.rb +++ b/app/models/income_statement/totals.rb @@ -69,8 +69,9 @@ class IncomeStatement::Totals er.from_currency = ae.currency AND er.to_currency = :target_currency ) - WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment') + WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false + AND ae.exclude_from_cashflow = false AND a.family_id = :family_id AND a.status IN ('draft', 'active') GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END; @@ -95,8 +96,9 @@ class IncomeStatement::Totals er.from_currency = ae.currency AND er.to_currency = :target_currency ) - WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment') + WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false + AND ae.exclude_from_cashflow = false AND a.family_id = :family_id AND a.status IN ('draft', 'active') GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END @@ -126,6 +128,7 @@ class IncomeStatement::Totals WHERE a.family_id = :family_id AND a.status IN ('draft', 'active') AND ae.excluded = false + AND ae.exclude_from_cashflow = false AND ae.date BETWEEN :start_date AND :end_date GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END, CASE WHEN t.category_id IS NULL THEN true ELSE false END SQL diff --git a/app/models/investment_activity_detector.rb b/app/models/investment_activity_detector.rb new file mode 100644 index 000000000..c030d9cdf --- /dev/null +++ b/app/models/investment_activity_detector.rb @@ -0,0 +1,346 @@ +# Detects internal investment activity (fund swaps, reinvestments) by comparing +# holdings snapshots between syncs and marks matching transactions as excluded +# from cashflow. This is provider-agnostic and works with any holdings data. +# +# Usage: +# detector = InvestmentActivityDetector.new(account) +# detector.detect_and_mark_internal_activity(current_holdings, recent_transactions) +# +class InvestmentActivityDetector + def initialize(account) + @account = account + end + + # Class method for inferring activity label from description and amount + # without needing a full detector instance + # @param name [String] Transaction name/description + # @param amount [Numeric] Transaction amount + # @param account [Account, nil] Optional account for context (e.g., retirement plan detection) + # @return [String, nil] Activity label or nil if unknown + def self.infer_label_from_description(name, amount, account = nil) + new(nil).send(:infer_from_description, name, amount, account) + end + + # Call this after syncing transactions for an investment/crypto account + # @param current_holdings [Array] Array of holding objects/hashes from provider + # @param recent_transactions [Array] Recently imported transactions + def detect_and_mark_internal_activity(current_holdings, recent_transactions) + return unless @account.investment? || @account.crypto? + return if current_holdings.blank? + + previous_snapshot = @account.holdings_snapshot_data || [] + + # Find holdings changes that indicate buys/sells + changes = detect_holdings_changes(previous_snapshot, current_holdings) + + # Match changes to transactions and mark them as excluded + changes.each do |change| + matched_entry = find_matching_entry(change, recent_transactions) + next unless matched_entry + + transaction = matched_entry.entryable + + # Only auto-set if not already manually set by user (respect user overrides) + unless matched_entry.locked?(:exclude_from_cashflow) + matched_entry.update!(exclude_from_cashflow: true) + matched_entry.lock_attr!(:exclude_from_cashflow) + + Rails.logger.info( + "InvestmentActivityDetector: Auto-excluded entry #{matched_entry.id} " \ + "(#{matched_entry.name}) as internal #{change[:type]} of #{change[:symbol] || change[:description]}" + ) + end + + # Set activity label if not already set + if transaction.is_a?(Transaction) && transaction.investment_activity_label.blank? + label = infer_activity_label(matched_entry, change[:type]) + transaction.update!(investment_activity_label: label) if label.present? + end + end + + # Store current snapshot for next comparison + save_holdings_snapshot(current_holdings) + end + + private + + # Infer activity label from change type and transaction description + def infer_activity_label(entry, change_type) + # If we know it's a buy or sell from holdings comparison + return "Buy" if change_type == :buy + return "Sell" if change_type == :sell + + # Otherwise try to infer from description + infer_from_description(entry) + end + + # Infer activity label from transaction description + # Can be called with an Entry or with name/amount directly + # @param entry_or_name [Entry, String] Entry object or transaction name + # @param amount [Numeric, nil] Transaction amount (required if entry_or_name is String) + # @param account [Account, nil] Optional account for context (e.g., retirement plan detection) + def infer_from_description(entry_or_name, amount = nil, account = nil) + if entry_or_name.respond_to?(:name) + description = (entry_or_name.name || "").upcase + amount = entry_or_name.amount || 0 + account ||= entry_or_name.try(:account) + else + description = (entry_or_name || "").upcase + amount ||= 0 + end + + # Check if this is a retirement plan account (401k, 403b, etc.) + account_name = (account&.name || "").upcase + retirement_indicators = %w[401K 403B RETIREMENT TOTALSOURCE NETBENEFITS] + retirement_phrases = [ "SAVINGS PLAN", "THRIFT PLAN", "PENSION" ] + is_retirement_plan = retirement_indicators.any? { |ind| account_name.include?(ind) } || + retirement_phrases.any? { |phrase| account_name.include?(phrase) } + + # Check for sweep/money market patterns (but NOT money market FUND purchases) + # INVESTOR CL indicates this is a money market fund, not a sweep + sweep_patterns = %w[SWEEP SETTLEMENT] + money_market_sweep = description.include?("MONEY MARKET") && !description.include?("INVESTOR") + common_money_market_tickers = %w[VMFXX SPAXX FDRXX SWVXX SPRXX] + + if sweep_patterns.any? { |p| description.include?(p) } || + money_market_sweep || + common_money_market_tickers.any? { |t| description == t } + return amount.positive? ? "Sweep Out" : "Sweep In" + end + + # Check for likely interest/dividend on money market funds + # Small amounts (under $5) on money market funds are typically interest income + money_market_fund_patterns = %w[MONEY\ MARKET VMFXX SPAXX FDRXX SWVXX SPRXX VUSXX] + is_money_market_fund = money_market_fund_patterns.any? { |p| description.include?(p) } + + if is_money_market_fund && amount.abs < 5 + # Small money market amounts are interest, not buys/sells + return "Interest" + end + + # Check for dividend patterns + if description == "CASH" || description.include?("DIVIDEND") || + description.include?("DISTRIBUTION") + return "Dividend" + end + + # Check for interest + return "Interest" if description.include?("INTEREST") + + # Check for fees + return "Fee" if description.include?("FEE") || description.include?("CHARGE") + + # Check for reinvestment + return "Reinvestment" if description.include?("REINVEST") + + # Check for exchange/conversion + return "Exchange" if description.include?("EXCHANGE") || description.include?("CONVERSION") + + # Check for contribution patterns + return "Contribution" if description.include?("CONTRIBUTION") || description.include?("DEPOSIT") + + # Check for withdrawal patterns + return "Withdrawal" if description.include?("WITHDRAWAL") || description.include?("DISBURSEMENT") + + # Check for fund names that indicate buy/sell activity + # Positive amount = money out from account perspective = buying securities + # Negative amount = money in = selling securities + fund_patterns = %w[ + INDEX FUND ADMIRAL ETF SHARES TRUST + VANGUARD FIDELITY SCHWAB ISHARES SPDR + 500\ INDEX TOTAL\ MARKET GROWTH BOND + ] + + # Common fund ticker patterns + fund_ticker_patterns = %w[ + VFIAX VTSAX VXUS VBTLX VTIAX VTTVX + VTI VOO VGT VIG VYM VGIT + FXAIX FZROX FSKAX FBALX + SWTSX SWPPX SCHD SCHX + SPY QQQ IVV AGG + IBIT GBTC ETHE + ] + + is_fund_transaction = fund_patterns.any? { |p| description.include?(p) } || + fund_ticker_patterns.any? { |t| description.include?(t) } + + if is_fund_transaction + if is_retirement_plan && amount.negative? + # Negative amount in retirement plan = payroll contribution buying shares + return "Contribution" + else + return amount.positive? ? "Buy" : "Sell" + end + end + + nil # Unknown - user can set manually + end + + def detect_holdings_changes(previous, current) + changes = [] + + current.each do |holding| + prev = find_previous_holding(previous, holding) + + if prev.nil? + # New holding appeared = BUY + changes << { + type: :buy, + symbol: holding_symbol(holding), + description: holding_description(holding), + shares: holding_shares(holding), + cost_basis: holding_cost_basis(holding), + created_at: holding_created_at(holding) + } + elsif holding_shares(holding) > prev_shares(prev) + # Shares increased = BUY + changes << { + type: :buy, + symbol: holding_symbol(holding), + description: holding_description(holding), + shares_delta: holding_shares(holding) - prev_shares(prev), + cost_basis_delta: holding_cost_basis(holding) - prev_cost_basis(prev) + } + elsif holding_shares(holding) < prev_shares(prev) + # Shares decreased = SELL + changes << { + type: :sell, + symbol: holding_symbol(holding), + description: holding_description(holding), + shares_delta: prev_shares(prev) - holding_shares(holding) + } + end + end + + # Check for holdings that completely disappeared = SELL ALL + previous.each do |prev| + unless current.any? { |h| same_holding?(h, prev) } + changes << { + type: :sell, + symbol: prev_symbol(prev), + description: prev_description(prev), + shares: prev_shares(prev) + } + end + end + + changes + end + + def find_matching_entry(change, transactions) + transactions.each do |txn| + entry = txn.respond_to?(:entry) ? txn.entry : txn + next unless entry + next if entry.exclude_from_cashflow? # Already excluded + + # Match by cost_basis amount (for buys with known cost) + if change[:cost_basis].present? && change[:cost_basis].to_d > 0 + amount_diff = (entry.amount.to_d.abs - change[:cost_basis].to_d.abs).abs + return entry if amount_diff < 0.01 + end + + # Match by cost_basis delta (for additional buys) + if change[:cost_basis_delta].present? && change[:cost_basis_delta].to_d > 0 + amount_diff = (entry.amount.to_d.abs - change[:cost_basis_delta].to_d.abs).abs + return entry if amount_diff < 0.01 + end + + # Match by description containing security name/symbol + entry_desc = entry.name&.downcase || "" + + if change[:symbol].present? + return entry if entry_desc.include?(change[:symbol].downcase) + end + + if change[:description].present? + # Match first few words of description for fuzzy matching + desc_words = change[:description].downcase.split.first(3).join(" ") + return entry if desc_words.present? && entry_desc.include?(desc_words) + end + end + + nil + end + + def find_previous_holding(previous, current) + symbol = holding_symbol(current) + return previous.find { |p| prev_symbol(p) == symbol } if symbol.present? + + # Fallback to description matching if no symbol + desc = holding_description(current) + previous.find { |p| prev_description(p) == desc } if desc.present? + end + + def same_holding?(current, previous) + current_symbol = holding_symbol(current) + prev_sym = prev_symbol(previous) + + if current_symbol.present? && prev_sym.present? + current_symbol == prev_sym + else + holding_description(current) == prev_description(previous) + end + end + + def save_holdings_snapshot(holdings) + snapshot_data = holdings.map do |h| + { + "symbol" => holding_symbol(h), + "description" => holding_description(h), + "shares" => holding_shares(h).to_s, + "cost_basis" => holding_cost_basis(h).to_s, + "market_value" => holding_market_value(h).to_s + } + end + + @account.update!( + holdings_snapshot_data: snapshot_data, + holdings_snapshot_at: Time.current + ) + end + + # Normalize access - holdings could be AR objects or hashes from different providers + def holding_symbol(h) + h.try(:symbol) || h.try(:ticker) || h["symbol"] || h[:symbol] || h["ticker"] || h[:ticker] + end + + def holding_description(h) + h.try(:description) || h.try(:name) || h["description"] || h[:description] || h["name"] || h[:name] + end + + def holding_shares(h) + val = h.try(:shares) || h.try(:qty) || h["shares"] || h[:shares] || h["qty"] || h[:qty] + val.to_d + end + + def holding_cost_basis(h) + val = h.try(:cost_basis) || h["cost_basis"] || h[:cost_basis] + val.to_d + end + + def holding_market_value(h) + val = h.try(:market_value) || h.try(:amount) || h["market_value"] || h[:market_value] || h["amount"] || h[:amount] + val.to_d + end + + def holding_created_at(h) + h.try(:created_at) || h["created"] || h[:created] || h["created_at"] || h[:created_at] + end + + # Previous snapshot accessor methods (snapshot is always a hash) + def prev_symbol(p) + p["symbol"] || p[:symbol] + end + + def prev_description(p) + p["description"] || p[:description] + end + + def prev_shares(p) + (p["shares"] || p[:shares]).to_d + end + + def prev_cost_basis(p) + (p["cost_basis"] || p[:cost_basis]).to_d + end +end diff --git a/app/models/lunchflow_account/processor.rb b/app/models/lunchflow_account/processor.rb index b9c6b2184..d82301dbd 100644 --- a/app/models/lunchflow_account/processor.rb +++ b/app/models/lunchflow_account/processor.rb @@ -74,10 +74,37 @@ class LunchflowAccount::Processor return unless [ "Investment", "Crypto" ].include?(lunchflow_account.current_account&.accountable_type) LunchflowAccount::Investments::HoldingsProcessor.new(lunchflow_account).process + + # Detect and mark internal investment activity (fund swaps, reinvestments) + detect_internal_investment_activity rescue => e report_exception(e, "holdings") end + def detect_internal_investment_activity + account = lunchflow_account.current_account + return unless account&.investment? || account&.crypto? + + # Get current holdings from raw payload + current_holdings = lunchflow_account.raw_holdings_payload || [] + return if current_holdings.blank? + + # Get recent transactions (last 30 days to catch any we might have missed) + recent_transactions = account.entries + .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") + .where(date: 30.days.ago.to_date..Date.current) + .where(exclude_from_cashflow: false) + .map(&:entryable) + .compact + + InvestmentActivityDetector.new(account).detect_and_mark_internal_activity( + current_holdings, + recent_transactions + ) + rescue => e + Rails.logger.warn("InvestmentActivityDetector failed for Lunchflow account #{lunchflow_account.id}: #{e.message}") + end + def report_exception(error, context) Sentry.capture_exception(error) do |scope| scope.set_tags( diff --git a/app/models/plaid_account/investments/transactions_processor.rb b/app/models/plaid_account/investments/transactions_processor.rb index 922a3f2a6..ded2a473f 100644 --- a/app/models/plaid_account/investments/transactions_processor.rb +++ b/app/models/plaid_account/investments/transactions_processor.rb @@ -1,6 +1,23 @@ class PlaidAccount::Investments::TransactionsProcessor SecurityNotFoundError = Class.new(StandardError) + # Map Plaid investment transaction types to activity labels + PLAID_TYPE_TO_LABEL = { + "buy" => "Buy", + "sell" => "Sell", + "cancel" => "Cancelled", + "cash" => "Cash", + "fee" => "Fee", + "transfer" => "Transfer", + "dividend" => "Dividend", + "interest" => "Interest", + "contribution" => "Contribution", + "withdrawal" => "Withdrawal", + "dividend reinvestment" => "Reinvestment", + "spin off" => "Other", + "split" => "Other" + }.freeze + def initialize(plaid_account, security_resolver:) @plaid_account = plaid_account @security_resolver = security_resolver @@ -68,10 +85,16 @@ class PlaidAccount::Investments::TransactionsProcessor currency: transaction["iso_currency_code"], date: transaction["date"], name: transaction["name"], - source: "plaid" + source: "plaid", + investment_activity_label: label_from_plaid_type(transaction) ) end + def label_from_plaid_type(transaction) + plaid_type = transaction["type"]&.downcase + PLAID_TYPE_TO_LABEL[plaid_type] || plaid_type&.titleize + end + def transactions plaid_account.raw_investments_payload["transactions"] || [] end diff --git a/app/models/plaid_account/processor.rb b/app/models/plaid_account/processor.rb index 4faead9d9..e282d5e9d 100644 --- a/app/models/plaid_account/processor.rb +++ b/app/models/plaid_account/processor.rb @@ -103,10 +103,50 @@ class PlaidAccount::Processor def process_investments PlaidAccount::Investments::TransactionsProcessor.new(plaid_account, security_resolver: security_resolver).process PlaidAccount::Investments::HoldingsProcessor.new(plaid_account, security_resolver: security_resolver).process + + # Detect and mark internal investment activity (fund swaps, reinvestments) + # Note: Plaid already creates Trade entries for buy/sell, but this catches cash transactions + detect_internal_investment_activity rescue => e report_exception(e) end + def detect_internal_investment_activity + account = AccountProvider.find_by(provider: plaid_account)&.account + return unless account&.investment? || account&.crypto? + + # Get current holdings from raw payload + raw_holdings = plaid_account.raw_investments_payload&.dig("holdings") || [] + return if raw_holdings.blank? + + # Transform to common format + current_holdings = raw_holdings.map do |h| + security = security_resolver.resolve(h["security_id"]) + { + "symbol" => security&.ticker, + "description" => security&.name, + "shares" => h["quantity"], + "cost_basis" => h["cost_basis"], + "market_value" => h["institution_value"] + } + end + + # Get recent transactions (last 30 days to catch any we might have missed) + recent_transactions = account.entries + .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") + .where(date: 30.days.ago.to_date..Date.current) + .where(exclude_from_cashflow: false) + .map(&:entryable) + .compact + + InvestmentActivityDetector.new(account).detect_and_mark_internal_activity( + current_holdings, + recent_transactions + ) + rescue => e + Rails.logger.warn("InvestmentActivityDetector failed for Plaid account #{plaid_account.id}: #{e.message}") + end + def process_liabilities case [ plaid_account.plaid_type, plaid_account.plaid_subtype ] when [ "credit", "credit card" ] diff --git a/app/models/simplefin_account/processor.rb b/app/models/simplefin_account/processor.rb index 569f515cd..68b679772 100644 --- a/app/models/simplefin_account/processor.rb +++ b/app/models/simplefin_account/processor.rb @@ -151,10 +151,37 @@ class SimplefinAccount::Processor return unless simplefin_account.current_account&.accountable_type == "Investment" SimplefinAccount::Investments::TransactionsProcessor.new(simplefin_account).process SimplefinAccount::Investments::HoldingsProcessor.new(simplefin_account).process + + # Detect and mark internal investment activity (fund swaps, reinvestments) + detect_internal_investment_activity rescue => e report_exception(e, "investments") end + def detect_internal_investment_activity + account = simplefin_account.current_account + return unless account&.investment? || account&.crypto? + + # Get current holdings from raw payload + current_holdings = simplefin_account.raw_holdings_payload || [] + return if current_holdings.blank? + + # Get recent transactions (last 30 days to catch any we might have missed) + recent_transactions = account.entries + .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") + .where(date: 30.days.ago.to_date..Date.current) + .where(exclude_from_cashflow: false) + .map(&:entryable) + .compact + + InvestmentActivityDetector.new(account).detect_and_mark_internal_activity( + current_holdings, + recent_transactions + ) + rescue => e + Rails.logger.warn("InvestmentActivityDetector failed for account #{simplefin_account.current_account&.id}: #{e.message}") + end + def process_liabilities case simplefin_account.current_account&.accountable_type when "CreditCard" diff --git a/app/models/simplefin_entry/processor.rb b/app/models/simplefin_entry/processor.rb index db4b5689b..8e6cd88f5 100644 --- a/app/models/simplefin_entry/processor.rb +++ b/app/models/simplefin_entry/processor.rb @@ -18,7 +18,8 @@ class SimplefinEntry::Processor source: "simplefin", merchant: merchant, notes: notes, - extra: extra_metadata + extra: extra_metadata, + investment_activity_label: inferred_activity_label ) end @@ -204,4 +205,11 @@ class SimplefinEntry::Processor end parts.presence&.join(" | ") end + + # Infer investment activity label from transaction description + # Only returns a label for investment/crypto accounts + def inferred_activity_label + return nil unless account&.investment? || account&.crypto? + InvestmentActivityDetector.infer_label_from_description(name, amount, account) + end end diff --git a/app/models/transaction.rb b/app/models/transaction.rb index 70ca49da0..42e3a2677 100644 --- a/app/models/transaction.rb +++ b/app/models/transaction.rb @@ -16,9 +16,23 @@ class Transaction < ApplicationRecord funds_movement: "funds_movement", # Movement of funds between accounts, excluded from budget analytics cc_payment: "cc_payment", # A CC payment, excluded from budget analytics (CC payments offset the sum of expense transactions) loan_payment: "loan_payment", # A payment to a Loan account, treated as an expense in budgets - one_time: "one_time" # A one-time expense/income, excluded from budget analytics + one_time: "one_time", # A one-time expense/income, excluded from budget analytics + investment_contribution: "investment_contribution" # Transfer to investment/crypto account, included in budget as investment expense } + # Labels for internal investment activity (auto-exclude from cashflow) + # Only internal shuffling should be excluded, not contributions/dividends/withdrawals + INTERNAL_ACTIVITY_LABELS = %w[Buy Sell Reinvestment Exchange].freeze + + # All valid investment activity labels (for UI dropdown) + ACTIVITY_LABELS = [ + "Buy", "Sell", "Sweep In", "Sweep Out", "Dividend", "Reinvestment", + "Interest", "Fee", "Transfer", "Contribution", "Withdrawal", "Exchange", "Other" + ].freeze + + after_save :sync_exclude_from_cashflow_with_activity_label, + if: :saved_change_to_investment_activity_label? + # Pending transaction scopes - filter based on provider pending flags in extra JSONB # Works with any provider that stores pending status in extra["provider_name"]["pending"] scope :pending, -> { @@ -145,4 +159,17 @@ class Transaction < ApplicationRecord FamilyMerchantAssociation.where(family: family, merchant: merchant).delete_all end + + # Sync exclude_from_cashflow based on activity label + # Internal activities (Buy, Sell, etc.) should be excluded from cashflow + def sync_exclude_from_cashflow_with_activity_label + return unless entry&.account&.investment? || entry&.account&.crypto? + return if entry.locked?(:exclude_from_cashflow) # Respect user's manual setting + + should_exclude = INTERNAL_ACTIVITY_LABELS.include?(investment_activity_label) + + if entry.exclude_from_cashflow != should_exclude + entry.update!(exclude_from_cashflow: should_exclude) + end + end end diff --git a/app/models/transaction/search.rb b/app/models/transaction/search.rb index 6c401e0f0..3ece18e45 100644 --- a/app/models/transaction/search.rb +++ b/app/models/transaction/search.rb @@ -49,8 +49,8 @@ class Transaction::Search Rails.cache.fetch("transaction_search_totals/#{cache_key_base}") do result = transactions_scope .select( - "COALESCE(SUM(CASE WHEN entries.amount >= 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment') THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as expense_total", - "COALESCE(SUM(CASE WHEN entries.amount < 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment') THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as income_total", + "COALESCE(SUM(CASE WHEN entries.amount >= 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution') AND entries.exclude_from_cashflow = false THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as expense_total", + "COALESCE(SUM(CASE WHEN entries.amount < 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution') AND entries.exclude_from_cashflow = false THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as income_total", "COUNT(entries.id) as transactions_count" ) .joins( @@ -100,14 +100,14 @@ class Transaction::Search if parent_category_ids.empty? query = query.left_joins(:category).where( "categories.name IN (?) OR ( - categories.id IS NULL AND (transactions.kind NOT IN ('funds_movement', 'cc_payment')) + categories.id IS NULL AND (transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution')) )", categories ) else query = query.left_joins(:category).where( "categories.name IN (?) OR categories.parent_id IN (?) OR ( - categories.id IS NULL AND (transactions.kind NOT IN ('funds_movement', 'cc_payment')) + categories.id IS NULL AND (transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution')) )", categories, parent_category_ids ) diff --git a/app/models/transfer.rb b/app/models/transfer.rb index 93de0a068..d2dcbf667 100644 --- a/app/models/transfer.rb +++ b/app/models/transfer.rb @@ -16,6 +16,10 @@ class Transfer < ApplicationRecord def kind_for_account(account) if account.loan? "loan_payment" + elsif account.credit_card? + "cc_payment" + elsif account.investment? || account.crypto? + "investment_contribution" elsif account.liability? "cc_payment" else diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index ec89bb308..a4593a56d 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -78,6 +78,19 @@ <% end %> + <% if entry.exclude_from_cashflow? %> + + <%= icon "eye-off", size: "sm", color: "current" %> + + <% end %> + + <%# Investment activity label badge %> + <% if transaction.investment_activity_label.present? %> + "> + <%= transaction.investment_activity_label %> + + <% end %> + <%# Pending indicator %> <% if transaction.pending? %> "> diff --git a/app/views/transactions/show.html.erb b/app/views/transactions/show.html.erb index 0b6397494..85e770509 100644 --- a/app/views/transactions/show.html.erb +++ b/app/views/transactions/show.html.erb @@ -194,8 +194,8 @@ data: { controller: "auto-submit-form" } do |f| %>
-

Exclude

-

Excluded transactions will be removed from budgeting calculations and reports.

+

<%= t(".exclude") %>

+

<%= t(".exclude_description") %>

<%= f.toggle :excluded, { data: { auto_submit_form_target: "auto" } } %> @@ -203,6 +203,55 @@ <% end %>
+
+ <%= styled_form_with model: @entry, + url: transaction_path(@entry), + class: "p-3", + data: { controller: "auto-submit-form" } do |f| %> +
+
+

<%= t(".exclude_from_cashflow") %>

+

+ <% if @entry.account.investment? || @entry.account.crypto? %> + <%= t(".exclude_from_cashflow_description_investment") %> + <% else %> + <%= t(".exclude_from_cashflow_description") %> + <% end %> +

+
+ + <%= f.toggle :exclude_from_cashflow, { data: { auto_submit_form_target: "auto" } } %> +
+ <% end %> +
+ + <% if @entry.account.investment? || @entry.account.crypto? %> +
+ <%= styled_form_with model: @entry, + url: transaction_path(@entry), + class: "p-3", + data: { controller: "auto-submit-form" } do |f| %> + <%= f.fields_for :entryable do |ef| %> +
+
+

<%= t(".activity_type") %>

+

<%= t(".activity_type_description") %>

+
+ + <%= ef.select :investment_activity_label, + options_for_select( + [["—", nil]] + Transaction::ACTIVITY_LABELS.map { |l| [l, l] }, + @entry.entryable.investment_activity_label + ), + { label: false }, + { class: "form-field__input border border-secondary rounded-lg px-3 py-1.5 max-w-40 text-sm", + data: { auto_submit_form_target: "auto" } } %> +
+ <% end %> + <% end %> +
+ <% end %> +
<%= styled_form_with model: @entry, url: transaction_path(@entry), diff --git a/config/locales/views/transactions/en.yml b/config/locales/views/transactions/en.yml index 717d146b1..0a393d835 100644 --- a/config/locales/views/transactions/en.yml +++ b/config/locales/views/transactions/en.yml @@ -31,6 +31,13 @@ en: balances, and cannot be undone. delete_title: Delete transaction details: Details + exclude: Exclude + exclude_description: Excluded transactions will be removed from budgeting calculations and reports. + exclude_from_cashflow: Exclude from Cashflow + exclude_from_cashflow_description: Hide from income/expense reports and Sankey chart. Useful for transactions you don't want in cashflow analysis. + exclude_from_cashflow_description_investment: Hide from income/expense reports and Sankey chart. Use for internal investment activity like fund swaps, reinvestments, or money market sweeps. + activity_type: Activity Type + activity_type_description: Type of investment activity (Buy, Sell, Dividend, etc.). Auto-detected or set manually. mark_recurring: Mark as Recurring mark_recurring_subtitle: Track this as a recurring transaction. Amount variance is automatically calculated from past 6 months of similar transactions. mark_recurring_title: Recurring Transaction @@ -48,9 +55,13 @@ en: potential_duplicate_description: This pending transaction may be the same as the posted transaction below. If so, merge them to avoid double-counting. merge_duplicate: Yes, merge them keep_both: No, keep both + loan_payment: Loan Payment + transfer: Transfer transaction: pending: Pending pending_tooltip: Pending transaction — may change when posted + excluded_from_cashflow_tooltip: Excluded from cashflow reports + activity_type_tooltip: Investment activity type possible_duplicate: Duplicate? potential_duplicate_tooltip: This may be a duplicate of another transaction review_recommended: Review diff --git a/db/migrate/20260110120000_add_investment_cashflow_support.rb b/db/migrate/20260110120000_add_investment_cashflow_support.rb new file mode 100644 index 000000000..26cffbc55 --- /dev/null +++ b/db/migrate/20260110120000_add_investment_cashflow_support.rb @@ -0,0 +1,13 @@ +class AddInvestmentCashflowSupport < ActiveRecord::Migration[7.2] + def change + # Flag for excluding from cashflow (user-controllable) + # Used for internal investment activity like fund swaps + add_column :entries, :exclude_from_cashflow, :boolean, default: false, null: false + add_index :entries, :exclude_from_cashflow + + # Holdings snapshot for comparison (provider-agnostic) + # Used to detect internal investment activity by comparing holdings between syncs + add_column :accounts, :holdings_snapshot_data, :jsonb + add_column :accounts, :holdings_snapshot_at, :datetime + end +end diff --git a/db/migrate/20260110180000_add_investment_activity_label_to_transactions.rb b/db/migrate/20260110180000_add_investment_activity_label_to_transactions.rb new file mode 100644 index 000000000..658900bc2 --- /dev/null +++ b/db/migrate/20260110180000_add_investment_activity_label_to_transactions.rb @@ -0,0 +1,8 @@ +class AddInvestmentActivityLabelToTransactions < ActiveRecord::Migration[7.2] + def change + # Label for investment activity type (Buy, Sell, Sweep In, Dividend, etc.) + # Provides human-readable context for why a transaction is excluded from cashflow + add_column :transactions, :investment_activity_label, :string + add_index :transactions, :investment_activity_label + end +end diff --git a/db/schema.rb b/db/schema.rb index a4b314661..04b9920ae 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do +ActiveRecord::Schema[7.2].define(version: 2026_01_10_180000) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -476,22 +476,6 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.index ["merchant_id"], name: "index_family_merchant_associations_on_merchant_id" end - create_table "flipper_features", force: :cascade do |t| - t.string "key", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["key"], name: "index_flipper_features_on_key", unique: true - end - - create_table "flipper_gates", force: :cascade do |t| - t.string "feature_key", null: false - t.string "key", null: false - t.text "value" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["feature_key", "key", "value"], name: "index_flipper_gates_on_feature_key_and_key_and_value", unique: true - end - create_table "holdings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "account_id", null: false t.uuid "security_id", null: false @@ -505,8 +489,6 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.string "external_id" t.decimal "cost_basis", precision: 19, scale: 4 t.uuid "account_provider_id" - t.string "cost_basis_source" - t.boolean "cost_basis_locked", default: false, null: false t.index ["account_id", "external_id"], name: "idx_holdings_on_account_id_external_id_unique", unique: true, where: "(external_id IS NOT NULL)" t.index ["account_id", "security_id", "date", "currency"], name: "idx_on_account_id_security_id_date_currency_5323e39f8b", unique: true t.index ["account_id"], name: "index_holdings_on_account_id" @@ -816,8 +798,6 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.datetime "last_authenticated_at" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "issuer" - t.index ["issuer"], name: "index_oidc_identities_on_issuer" t.index ["provider", "uid"], name: "index_oidc_identities_on_provider_and_uid", unique: true t.index ["user_id"], name: "index_oidc_identities_on_user_id" end @@ -1073,38 +1053,6 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.index ["status"], name: "index_simplefin_items_on_status" end - create_table "sso_audit_logs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.uuid "user_id" - t.string "event_type", null: false - t.string "provider" - t.string "ip_address" - t.string "user_agent" - t.jsonb "metadata", default: {}, null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["created_at"], name: "index_sso_audit_logs_on_created_at" - t.index ["event_type"], name: "index_sso_audit_logs_on_event_type" - t.index ["user_id", "created_at"], name: "index_sso_audit_logs_on_user_id_and_created_at" - t.index ["user_id"], name: "index_sso_audit_logs_on_user_id" - end - - create_table "sso_providers", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "strategy", null: false - t.string "name", null: false - t.string "label", null: false - t.string "icon" - t.boolean "enabled", default: true, null: false - t.string "issuer" - t.string "client_id" - t.string "client_secret" - t.string "redirect_uri" - t.jsonb "settings", default: {}, null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["enabled"], name: "index_sso_providers_on_enabled" - t.index ["name"], name: "index_sso_providers_on_name", unique: true - end - create_table "subscriptions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "family_id", null: false t.string "status", null: false @@ -1181,14 +1129,7 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.string "currency" t.jsonb "locked_attributes", default: {} t.uuid "category_id" - t.decimal "realized_gain", precision: 19, scale: 4 - t.decimal "cost_basis_amount", precision: 19, scale: 4 - t.string "cost_basis_currency" - t.integer "holding_period_days" - t.string "realized_gain_confidence" - t.string "realized_gain_currency" t.index ["category_id"], name: "index_trades_on_category_id" - t.index ["realized_gain"], name: "index_trades_on_realized_gain_not_null", where: "(realized_gain IS NOT NULL)" t.index ["security_id"], name: "index_trades_on_security_id" end @@ -1339,7 +1280,6 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do add_foreign_key "sessions", "users" add_foreign_key "simplefin_accounts", "simplefin_items" add_foreign_key "simplefin_items", "families" - add_foreign_key "sso_audit_logs", "users" add_foreign_key "subscriptions", "families" add_foreign_key "syncs", "syncs", column: "parent_id" add_foreign_key "taggings", "tags" diff --git a/lib/tasks/investment_labels.rake b/lib/tasks/investment_labels.rake new file mode 100644 index 000000000..31741574a --- /dev/null +++ b/lib/tasks/investment_labels.rake @@ -0,0 +1,185 @@ +# frozen_string_literal: true + +# Backfill investment activity labels for existing transactions +# +# Usage examples: +# # Preview (dry run) - show what labels would be set +# bin/rails 'sure:investments:backfill_labels[dry_run=true]' +# +# # Execute the backfill for all investment/crypto accounts +# bin/rails 'sure:investments:backfill_labels[dry_run=false]' +# +# # Backfill for a specific account +# bin/rails 'sure:investments:backfill_labels[account_id=8b46387c-5aa4-4a92-963a-4392c10999c9,dry_run=false]' +# +# # Force re-label already-labeled transactions +# bin/rails 'sure:investments:backfill_labels[dry_run=false,force=true]' + +namespace :sure do + namespace :investments do + desc "Backfill activity labels for existing investment transactions. Args: account_id (optional), dry_run=true, force=false" + task :backfill_labels, [ :account_id, :dry_run, :force ] => :environment do |_, args| + # Support named args (key=value) - parse all positional args for key=value pairs + kv = {} + [ args[:account_id], args[:dry_run], args[:force] ].each do |raw| + next unless raw.is_a?(String) && raw.include?("=") + k, v = raw.split("=", 2) + kv[k.to_s] = v + end + + # Only use positional args if they don't contain "=" (otherwise they're named args in wrong position) + positional_account_id = args[:account_id] unless args[:account_id].to_s.include?("=") + positional_dry_run = args[:dry_run] unless args[:dry_run].to_s.include?("=") + positional_force = args[:force] unless args[:force].to_s.include?("=") + + account_id = (kv["account_id"] || positional_account_id).presence + dry_raw = (kv["dry_run"] || positional_dry_run).to_s.downcase + force_raw = (kv["force"] || positional_force).to_s.downcase + force = %w[true yes 1].include?(force_raw) + + # Default to dry_run=true unless explicitly disabled + dry_run = if dry_raw.blank? + true + elsif %w[1 true yes y].include?(dry_raw) + true + elsif %w[0 false no n].include?(dry_raw) + false + else + puts({ ok: false, error: "invalid_argument", message: "dry_run must be one of: true/yes/1 or false/no/0" }.to_json) + exit 1 + end + + # Build account scope + accounts = if account_id.present? + Account.where(id: account_id) + else + Account.where(accountable_type: %w[Investment Crypto]) + end + + if accounts.none? + puts({ ok: false, error: "no_accounts", message: "No investment/crypto accounts found" }.to_json) + exit 1 + end + + total_processed = 0 + total_labeled = 0 + total_skipped = 0 + total_errors = 0 + + accounts.find_each do |account| + # Skip non-investment/crypto accounts if processing all + next unless account.investment? || account.crypto? + + acct_processed = 0 + acct_labeled = 0 + acct_skipped = 0 + acct_errors = 0 + + # Find transactions (optionally include already-labeled if force=true) + entries = account.entries + .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") + + unless force + entries = entries.where("transactions.investment_activity_label IS NULL OR transactions.investment_activity_label = ''") + end + + entries.find_each do |entry| + acct_processed += 1 + total_processed += 1 + + begin + transaction = entry.transaction + current_label = transaction.investment_activity_label + label = InvestmentActivityDetector.infer_label_from_description(entry.name, entry.amount, account) + + # Skip if no label can be inferred + if label.blank? + acct_skipped += 1 + total_skipped += 1 + next + end + + # Skip if label unchanged (when force=true) + if current_label == label + acct_skipped += 1 + total_skipped += 1 + next + end + + if dry_run + if current_label.present? + puts " [DRY RUN] Would relabel '#{entry.name}' (#{entry.amount}) from '#{current_label}' to '#{label}'" + else + puts " [DRY RUN] Would label '#{entry.name}' (#{entry.amount}) as '#{label}'" + end + else + transaction.update!(investment_activity_label: label) + if current_label.present? + puts " Relabeled '#{entry.name}' (#{entry.amount}) from '#{current_label}' to '#{label}'" + else + puts " Labeled '#{entry.name}' (#{entry.amount}) as '#{label}'" + end + end + acct_labeled += 1 + total_labeled += 1 + rescue => e + acct_errors += 1 + total_errors += 1 + puts({ error: e.class.name, message: e.message, entry_id: entry.id }.to_json) + end + end + + puts({ account_id: account.id, account_name: account.name, accountable_type: account.accountable_type, processed: acct_processed, labeled: acct_labeled, skipped: acct_skipped, errors: acct_errors, dry_run: dry_run, force: force }.to_json) + end + + puts({ ok: true, total_processed: total_processed, total_labeled: total_labeled, total_skipped: total_skipped, total_errors: total_errors, dry_run: dry_run }.to_json) + end + + desc "Clear all investment activity labels (for testing). Args: account_id (required), dry_run=true" + task :clear_labels, [ :account_id, :dry_run ] => :environment do |_, args| + kv = {} + [ args[:account_id], args[:dry_run] ].each do |raw| + next unless raw.is_a?(String) && raw.include?("=") + k, v = raw.split("=", 2) + kv[k.to_s] = v + end + + # Only use positional args if they don't contain "=" + positional_account_id = args[:account_id] unless args[:account_id].to_s.include?("=") + positional_dry_run = args[:dry_run] unless args[:dry_run].to_s.include?("=") + + account_id = (kv["account_id"] || positional_account_id).presence + dry_raw = (kv["dry_run"] || positional_dry_run).to_s.downcase + + unless account_id.present? + puts({ ok: false, error: "usage", message: "Provide account_id" }.to_json) + exit 1 + end + + dry_run = if dry_raw.blank? + true + elsif %w[1 true yes y].include?(dry_raw) + true + else + false + end + + account = Account.find(account_id) + + count = account.entries + .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") + .where("transactions.investment_activity_label IS NOT NULL AND transactions.investment_activity_label != ''") + .count + + if dry_run + puts({ ok: true, message: "Would clear #{count} labels", dry_run: true }.to_json) + else + Transaction.joins(:entry) + .where(entries: { account_id: account_id }) + .where("investment_activity_label IS NOT NULL AND investment_activity_label != ''") + .update_all(investment_activity_label: nil) + puts({ ok: true, message: "Cleared #{count} labels", dry_run: false }.to_json) + end + end + end +end diff --git a/test/models/income_statement_test.rb b/test/models/income_statement_test.rb index b152917c6..ba5fa3e81 100644 --- a/test/models/income_statement_test.rb +++ b/test/models/income_statement_test.rb @@ -285,4 +285,70 @@ class IncomeStatementTest < ActiveSupport::TestCase assert_equal 5, totals.transactions_count assert_equal Money.new(1050, @family.currency), totals.expense_money # 900 + 150 end + + # NEW TESTS: exclude_from_cashflow Feature + test "excludes transactions with exclude_from_cashflow flag from totals" do + # Create an expense transaction and mark it as excluded from cashflow + excluded_entry = create_transaction(account: @checking_account, amount: 250, category: @groceries_category) + excluded_entry.update!(exclude_from_cashflow: true) + + income_statement = IncomeStatement.new(@family) + totals = income_statement.totals(date_range: Period.last_30_days.date_range) + + # Should NOT include the excluded transaction + assert_equal 4, totals.transactions_count # Only original 4 transactions + assert_equal Money.new(1000, @family.currency), totals.income_money + assert_equal Money.new(900, @family.currency), totals.expense_money + end + + test "excludes income transactions with exclude_from_cashflow flag" do + # Create income and mark as excluded from cashflow + excluded_income = create_transaction(account: @checking_account, amount: -500, category: @income_category) + excluded_income.update!(exclude_from_cashflow: true) + + income_statement = IncomeStatement.new(@family) + totals = income_statement.totals(date_range: Period.last_30_days.date_range) + + # Should NOT include the excluded income + assert_equal 4, totals.transactions_count + assert_equal Money.new(1000, @family.currency), totals.income_money # Original income only + assert_equal Money.new(900, @family.currency), totals.expense_money + end + + test "excludes investment_contribution transactions from income statement" do + # Create a transfer to investment account (marked as investment_contribution) + investment_contribution = create_transaction( + account: @checking_account, + amount: 1000, + category: nil, + kind: "investment_contribution" + ) + + income_statement = IncomeStatement.new(@family) + totals = income_statement.totals(date_range: Period.last_30_days.date_range) + + # investment_contribution should be excluded (it's in the exclusion list) + assert_equal 4, totals.transactions_count # Only original 4 transactions + assert_equal Money.new(1000, @family.currency), totals.income_money + assert_equal Money.new(900, @family.currency), totals.expense_money + end + + test "exclude_from_cashflow works with median calculations" do + # Clear existing transactions + Entry.joins(:account).where(accounts: { family_id: @family.id }).destroy_all + + # Create expenses: 100, 200, 300 + create_transaction(account: @checking_account, amount: 100, category: @groceries_category) + create_transaction(account: @checking_account, amount: 200, category: @groceries_category) + excluded_entry = create_transaction(account: @checking_account, amount: 300, category: @groceries_category) + + # Exclude the 300 transaction from cashflow + excluded_entry.update!(exclude_from_cashflow: true) + + income_statement = IncomeStatement.new(@family) + + # Median should only consider non-excluded transactions (100, 200) + # Monthly total = 300, so median = 300.0 + assert_equal 300.0, income_statement.median_expense(interval: "month") + end end diff --git a/test/models/investment_activity_detector_test.rb b/test/models/investment_activity_detector_test.rb new file mode 100644 index 000000000..68ea44ac2 --- /dev/null +++ b/test/models/investment_activity_detector_test.rb @@ -0,0 +1,299 @@ +require "test_helper" + +class InvestmentActivityDetectorTest < ActiveSupport::TestCase + include EntriesTestHelper + + setup do + @family = families(:empty) + @investment_account = @family.accounts.create!( + name: "Brokerage", + balance: 10000, + cash_balance: 2000, + currency: "USD", + accountable: Investment.new + ) + @detector = InvestmentActivityDetector.new(@investment_account) + end + + test "detects new holding purchase and marks matching transaction" do + # Create a transaction that matches a new holding purchase + entry = create_transaction( + account: @investment_account, + amount: 1000, + name: "Buy VFIAX" + ) + transaction = entry.transaction + + # Simulate holdings snapshot showing a new holding + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + + # No previous snapshot + @investment_account.update!(holdings_snapshot_data: nil, holdings_snapshot_at: nil) + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + entry.reload + assert entry.exclude_from_cashflow?, "Transaction matching new holding should be excluded from cashflow" + end + + test "detects holding sale and marks matching transaction" do + # Set up previous holdings + previous_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 2000.0, "shares" => 20 } + ] + @investment_account.update!( + holdings_snapshot_data: previous_holdings, + holdings_snapshot_at: 1.day.ago + ) + + # Create a transaction for the sale proceeds (negative = inflow) + entry = create_transaction( + account: @investment_account, + amount: -1000, + name: "Sell VFIAX" + ) + transaction = entry.transaction + + # Current holdings show reduced position + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + entry.reload + assert entry.exclude_from_cashflow?, "Transaction matching holding sale should be excluded from cashflow" + end + + test "respects locked exclude_from_cashflow attribute" do + # Create a transaction and lock the attribute + entry = create_transaction( + account: @investment_account, + amount: 1000, + name: "Buy VFIAX" + ) + transaction = entry.transaction + + # User explicitly set to NOT exclude (and locked it) + entry.update!(exclude_from_cashflow: false) + entry.lock_attr!(:exclude_from_cashflow) + + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + entry.reload + assert_not entry.exclude_from_cashflow?, "Locked attribute should not be overwritten" + end + + test "updates holdings snapshot after detection" do + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 }, + { "symbol" => "IBIT", "cost_basis" => 500.0, "shares" => 5 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, []) + + @investment_account.reload + # Snapshot is normalized with string values and additional fields + snapshot = @investment_account.holdings_snapshot_data + assert_equal 2, snapshot.size + assert_equal "VFIAX", snapshot[0]["symbol"] + assert_equal "1000.0", snapshot[0]["cost_basis"] + assert_equal "10.0", snapshot[0]["shares"] + assert_equal "IBIT", snapshot[1]["symbol"] + assert_not_nil @investment_account.holdings_snapshot_at + end + + test "matches transaction by cost_basis amount within tolerance" do + entry = create_transaction( + account: @investment_account, + amount: 1000.005, # Very close - within 0.01 tolerance + name: "Investment purchase" + ) + transaction = entry.transaction + + # Holding with cost basis close to transaction amount (within 0.01) + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + entry.reload + assert entry.exclude_from_cashflow?, "Should match transaction within tolerance" + end + + test "does not mark unrelated transactions" do + # Create a regular expense transaction + entry = create_transaction( + account: @investment_account, + amount: 50, + name: "Account fee" + ) + transaction = entry.transaction + + # Holdings that don't match + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + entry.reload + assert_not entry.exclude_from_cashflow?, "Unrelated transaction should not be excluded" + end + + test "works with crypto accounts" do + crypto_account = @family.accounts.create!( + name: "Crypto Wallet", + balance: 5000, + currency: "USD", + accountable: Crypto.new + ) + detector = InvestmentActivityDetector.new(crypto_account) + + entry = create_transaction( + account: crypto_account, + amount: 1000, + name: "Buy BTC" + ) + transaction = entry.transaction + + current_holdings = [ + { "symbol" => "BTC", "cost_basis" => 1000.0, "shares" => 0.02 } + ] + + detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + entry.reload + assert entry.exclude_from_cashflow?, "Should work with crypto accounts" + end + + test "handles empty holdings gracefully" do + entry = create_transaction( + account: @investment_account, + amount: 1000, + name: "Some transaction" + ) + transaction = entry.transaction + + # Should not raise, just do nothing + assert_nothing_raised do + @detector.detect_and_mark_internal_activity([], [ transaction ]) + end + + entry.reload + assert_not entry.exclude_from_cashflow? + end + + test "handles nil holdings gracefully" do + entry = create_transaction( + account: @investment_account, + amount: 1000, + name: "Some transaction" + ) + transaction = entry.transaction + + assert_nothing_raised do + @detector.detect_and_mark_internal_activity(nil, [ transaction ]) + end + + entry.reload + assert_not entry.exclude_from_cashflow? + end + + test "sets Buy label for new holding purchase" do + entry = create_transaction( + account: @investment_account, + amount: 1000, + name: "Some investment" + ) + transaction = entry.transaction + + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + transaction.reload + assert_equal "Buy", transaction.investment_activity_label + end + + test "sets Sell label for holding sale" do + previous_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 2000.0, "shares" => 20 } + ] + @investment_account.update!( + holdings_snapshot_data: previous_holdings, + holdings_snapshot_at: 1.day.ago + ) + + entry = create_transaction( + account: @investment_account, + amount: -1000, + name: "VFIAX Sale" + ) + transaction = entry.transaction + + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + transaction.reload + assert_equal "Sell", transaction.investment_activity_label + end + + test "infers Sweep In label from money market description" do + entry = create_transaction( + account: @investment_account, + amount: -500, + name: "VANGUARD FEDERAL MONEY MARKET" + ) + transaction = entry.transaction + + # Call with empty holdings but simulate it being a sweep + # This tests the infer_from_description fallback + current_holdings = [ + { "symbol" => "VMFXX", "cost_basis" => 500.0, "shares" => 500 } + ] + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + transaction.reload + # Should be either "Buy" (from holdings match) or "Sweep In" (from description) + assert transaction.investment_activity_label.present? + end + + test "infers Dividend label from CASH description" do + entry = create_transaction( + account: @investment_account, + amount: -50, + name: "CASH" + ) + transaction = entry.transaction + + # No holdings change, but description-based inference + current_holdings = [ + { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } + ] + @investment_account.update!( + holdings_snapshot_data: current_holdings, + holdings_snapshot_at: 1.day.ago + ) + + @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) + + # Since there's no holdings change, no label gets set via holdings match + # But if we manually test the infer_from_description method... + label = @detector.send(:infer_from_description, entry) + assert_equal "Dividend", label + end +end diff --git a/test/models/transaction_test.rb b/test/models/transaction_test.rb index 7b5f2faeb..3d7b3236b 100644 --- a/test/models/transaction_test.rb +++ b/test/models/transaction_test.rb @@ -18,4 +18,37 @@ class TransactionTest < ActiveSupport::TestCase assert_not transaction.pending? end + + test "investment_contribution is a valid kind" do + transaction = Transaction.new(kind: "investment_contribution") + + assert_equal "investment_contribution", transaction.kind + assert transaction.investment_contribution? + end + + test "all transaction kinds are valid" do + valid_kinds = %w[standard funds_movement cc_payment loan_payment one_time investment_contribution] + + valid_kinds.each do |kind| + transaction = Transaction.new(kind: kind) + assert_equal kind, transaction.kind, "#{kind} should be a valid transaction kind" + end + end + + test "INTERNAL_ACTIVITY_LABELS contains expected labels" do + assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Buy" + assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Sell" + assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Reinvestment" + assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Exchange" + end + + test "ACTIVITY_LABELS contains all valid labels" do + assert_includes Transaction::ACTIVITY_LABELS, "Buy" + assert_includes Transaction::ACTIVITY_LABELS, "Sell" + assert_includes Transaction::ACTIVITY_LABELS, "Sweep In" + assert_includes Transaction::ACTIVITY_LABELS, "Sweep Out" + assert_includes Transaction::ACTIVITY_LABELS, "Dividend" + assert_includes Transaction::ACTIVITY_LABELS, "Interest" + assert_includes Transaction::ACTIVITY_LABELS, "Fee" + end end diff --git a/test/models/transfer_test.rb b/test/models/transfer_test.rb index 331638165..e47d200d2 100644 --- a/test/models/transfer_test.rb +++ b/test/models/transfer_test.rb @@ -104,4 +104,24 @@ class TransferTest < ActiveSupport::TestCase Transfer.create!(inflow_transaction: inflow_entry2.transaction, outflow_transaction: outflow_entry.transaction) end end + + test "kind_for_account returns investment_contribution for investment accounts" do + assert_equal "investment_contribution", Transfer.kind_for_account(accounts(:investment)) + end + + test "kind_for_account returns investment_contribution for crypto accounts" do + assert_equal "investment_contribution", Transfer.kind_for_account(accounts(:crypto)) + end + + test "kind_for_account returns loan_payment for loan accounts" do + assert_equal "loan_payment", Transfer.kind_for_account(accounts(:loan)) + end + + test "kind_for_account returns cc_payment for credit card accounts" do + assert_equal "cc_payment", Transfer.kind_for_account(accounts(:credit_card)) + end + + test "kind_for_account returns funds_movement for depository accounts" do + assert_equal "funds_movement", Transfer.kind_for_account(accounts(:depository)) + end end From 96022cbe9a76324eb32a946e0ee706e560a3a9a3 Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Sat, 10 Jan 2026 20:03:51 -0500 Subject: [PATCH 40/54] Update `PlaidAccount::Processor` to use resolved security from response - Refactored to extract `security` from `security_resolver.resolve` response for better clarity and consistency. --- app/models/plaid_account/processor.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/plaid_account/processor.rb b/app/models/plaid_account/processor.rb index e282d5e9d..3dc09c710 100644 --- a/app/models/plaid_account/processor.rb +++ b/app/models/plaid_account/processor.rb @@ -121,7 +121,8 @@ class PlaidAccount::Processor # Transform to common format current_holdings = raw_holdings.map do |h| - security = security_resolver.resolve(h["security_id"]) + response = security_resolver.resolve(plaid_security_id: h["security_id"]) + security = response.security { "symbol" => security&.ticker, "description" => security&.name, From e5fbdfb593dcb1c5ec5fbd8993b94a8f853d26f8 Mon Sep 17 00:00:00 2001 From: LPW Date: Mon, 12 Jan 2026 08:05:46 -0500 Subject: [PATCH 41/54] Add cost basis source tracking with manual override and lock protection (#623) * Add cost basis tracking and management to holdings - Added migration to introduce `cost_basis_source` and `cost_basis_locked` fields to `holdings`. - Implemented backfill for existing holdings to set `cost_basis_source` based on heuristics. - Introduced `Holding::CostBasisReconciler` to manage cost basis resolution logic. - Added user interface components for editing and locking cost basis in holdings. - Updated `materializer` to integrate reconciliation logic and respect locked holdings. - Extended tests for cost basis-related workflows to ensure accuracy and reliability. * Fix cost basis calculation in holdings controller - Ensure `cost_basis` is converted to decimal for accurate arithmetic. - Fix conditional check to properly validate positive `cost_basis`. * Improve cost basis validation and error handling in holdings controller - Allow zero as a valid cost basis for gifted/inherited shares. - Add error handling with user feedback for invalid cost basis values. --------- Co-authored-by: Josh Waldrep --- db/schema.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 04b9920ae..067d2c9f6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_01_10_180000) do +ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -489,6 +489,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_10_180000) do t.string "external_id" t.decimal "cost_basis", precision: 19, scale: 4 t.uuid "account_provider_id" + t.string "cost_basis_source" + t.boolean "cost_basis_locked", default: false, null: false t.index ["account_id", "external_id"], name: "idx_holdings_on_account_id_external_id_unique", unique: true, where: "(external_id IS NOT NULL)" t.index ["account_id", "security_id", "date", "currency"], name: "idx_on_account_id_security_id_date_currency_5323e39f8b", unique: true t.index ["account_id"], name: "index_holdings_on_account_id" @@ -1129,7 +1131,14 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_10_180000) do t.string "currency" t.jsonb "locked_attributes", default: {} t.uuid "category_id" + t.decimal "realized_gain", precision: 19, scale: 4 + t.decimal "cost_basis_amount", precision: 19, scale: 4 + t.string "cost_basis_currency" + t.integer "holding_period_days" + t.string "realized_gain_confidence" + t.string "realized_gain_currency" t.index ["category_id"], name: "index_trades_on_category_id" + t.index ["realized_gain"], name: "index_trades_on_realized_gain_not_null", where: "(realized_gain IS NOT NULL)" t.index ["security_id"], name: "index_trades_on_security_id" end From 307a8bb760456961d1cd7a8ddf9f7fbf5b7ee51a Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Mon, 12 Jan 2026 09:19:09 -0500 Subject: [PATCH 42/54] Localize investment activity labels and improve transaction processing - Replaced hardcoded activity labels with `I18n` translations for better localization. - Updated `transactions` views to display localized labels dynamically. - Fixed `InvestmentActivityDetector` to enhance dividend detection. - Refined `Account::ProviderImportAdapter` to prevent unnecessary updates and ensure transactional consistency. - Improved error handling and feedback in rake tasks for invalid arguments. --- app/models/account/provider_import_adapter.rb | 3 ++- app/models/investment_activity_detector.rb | 5 +++-- app/views/transactions/_transaction.html.erb | 2 +- app/views/transactions/show.html.erb | 6 +++--- config/locales/views/transactions/en.yml | 16 ++++++++++++++++ lib/tasks/investment_labels.rake | 6 +++++- 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/app/models/account/provider_import_adapter.rb b/app/models/account/provider_import_adapter.rb index 31d65a6ae..dac0da8d4 100644 --- a/app/models/account/provider_import_adapter.rb +++ b/app/models/account/provider_import_adapter.rb @@ -119,11 +119,12 @@ class Account::ProviderImportAdapter # Set investment activity label if provided and not already set if investment_activity_label.present? && entry.entryable.is_a?(Transaction) if entry.transaction.investment_activity_label.blank? - entry.transaction.update!(investment_activity_label: investment_activity_label) + entry.transaction.assign_attributes(investment_activity_label: investment_activity_label) end end entry.save! + entry.transaction.save! if entry.transaction.changed? # AFTER save: For NEW posted transactions, check for fuzzy matches to SUGGEST (not auto-claim) # This handles tip adjustments where auto-matching is too risky diff --git a/app/models/investment_activity_detector.rb b/app/models/investment_activity_detector.rb index c030d9cdf..2ef6c2f71 100644 --- a/app/models/investment_activity_detector.rb +++ b/app/models/investment_activity_detector.rb @@ -119,8 +119,9 @@ class InvestmentActivityDetector end # Check for dividend patterns - if description == "CASH" || description.include?("DIVIDEND") || - description.include?("DISTRIBUTION") + # "CASH" alone typically indicates dividend payout in brokerage feeds (only for inflows) + if description.include?("DIVIDEND") || description.include?("DISTRIBUTION") || + (description == "CASH" && amount < 0) return "Dividend" end diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index a4593a56d..c645d515d 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -87,7 +87,7 @@ <%# Investment activity label badge %> <% if transaction.investment_activity_label.present? %> "> - <%= transaction.investment_activity_label %> + <%= t("transactions.activity_labels.#{transaction.investment_activity_label.parameterize(separator: '_')}") %> <% end %> diff --git a/app/views/transactions/show.html.erb b/app/views/transactions/show.html.erb index 85e770509..4af991f1a 100644 --- a/app/views/transactions/show.html.erb +++ b/app/views/transactions/show.html.erb @@ -240,7 +240,7 @@ <%= ef.select :investment_activity_label, options_for_select( - [["—", nil]] + Transaction::ACTIVITY_LABELS.map { |l| [l, l] }, + [["—", nil]] + Transaction::ACTIVITY_LABELS.map { |l| [t("transactions.activity_labels.#{l.parameterize(separator: '_')}"), l] }, @entry.entryable.investment_activity_label ), { label: false }, @@ -260,8 +260,8 @@ <%= f.fields_for :entryable do |ef| %>
-

One-time <%= @entry.amount.negative? ? "Income" : "Expense" %>

-

One-time transactions will be excluded from certain budgeting calculations and reports to help you see what's really important.

+

<%= t(".one_time_title", type: @entry.amount.negative? ? t("transactions.form.income") : t("transactions.form.expense")) %>

+

<%= t(".one_time_description") %>

<%= ef.toggle :kind, { diff --git a/config/locales/views/transactions/en.yml b/config/locales/views/transactions/en.yml index 0a393d835..267adce43 100644 --- a/config/locales/views/transactions/en.yml +++ b/config/locales/views/transactions/en.yml @@ -38,6 +38,22 @@ en: exclude_from_cashflow_description_investment: Hide from income/expense reports and Sankey chart. Use for internal investment activity like fund swaps, reinvestments, or money market sweeps. activity_type: Activity Type activity_type_description: Type of investment activity (Buy, Sell, Dividend, etc.). Auto-detected or set manually. + one_time_title: One-time %{type} + one_time_description: One-time transactions will be excluded from certain budgeting calculations and reports to help you see what's really important. + activity_labels: + buy: Buy + sell: Sell + sweep_in: Sweep In + sweep_out: Sweep Out + dividend: Dividend + reinvestment: Reinvestment + interest: Interest + fee: Fee + transfer: Transfer + contribution: Contribution + withdrawal: Withdrawal + exchange: Exchange + other: Other mark_recurring: Mark as Recurring mark_recurring_subtitle: Track this as a recurring transaction. Amount variance is automatically calculated from past 6 months of similar transactions. mark_recurring_title: Recurring Transaction diff --git a/lib/tasks/investment_labels.rake b/lib/tasks/investment_labels.rake index 31741574a..9cefae7c9 100644 --- a/lib/tasks/investment_labels.rake +++ b/lib/tasks/investment_labels.rake @@ -78,6 +78,7 @@ namespace :sure do # Find transactions (optionally include already-labeled if force=true) entries = account.entries .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") + .includes(:entryable) unless force entries = entries.where("transactions.investment_activity_label IS NULL OR transactions.investment_activity_label = ''") @@ -160,8 +161,11 @@ namespace :sure do true elsif %w[1 true yes y].include?(dry_raw) true - else + elsif %w[0 false no n].include?(dry_raw) false + else + puts({ ok: false, error: "invalid_argument", message: "dry_run must be one of: true/yes/1 or false/no/0" }.to_json) + exit 1 end account = Account.find(account_id) From cfda5a6d3d2644ff96477389d1f0a1394840510e Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Mon, 12 Jan 2026 11:13:49 -0500 Subject: [PATCH 43/54] Remove `InvestmentActivityDetector` and related functionality - Deleted the `InvestmentActivityDetector` and associated tests. - Removed rake tasks for backfilling and clearing investment activity labels. - Simplified transaction processing in `SimplefinEntry::Processor` by removing inferred activity label logic. - Added new rule `SetInvestmentActivityLabel` for setting labels using rules. - Updated `Rule::Registry::TransactionResource` to include the new rule executor. --- app/models/investment_activity_detector.rb | 347 ------------------ app/models/lunchflow_account/processor.rb | 27 -- app/models/plaid_account/processor.rb | 41 --- .../set_investment_activity_label.rb | 31 ++ .../rule/registry/transaction_resource.rb | 1 + app/models/simplefin_account/processor.rb | 27 -- app/models/simplefin_entry/processor.rb | 10 +- app/models/transaction.rb | 20 - lib/tasks/investment_labels.rake | 189 ---------- .../investment_activity_detector_test.rb | 299 --------------- test/models/rule/action_test.rb | 32 ++ test/models/transaction_test.rb | 7 - 12 files changed, 65 insertions(+), 966 deletions(-) delete mode 100644 app/models/investment_activity_detector.rb create mode 100644 app/models/rule/action_executor/set_investment_activity_label.rb delete mode 100644 lib/tasks/investment_labels.rake delete mode 100644 test/models/investment_activity_detector_test.rb diff --git a/app/models/investment_activity_detector.rb b/app/models/investment_activity_detector.rb deleted file mode 100644 index 2ef6c2f71..000000000 --- a/app/models/investment_activity_detector.rb +++ /dev/null @@ -1,347 +0,0 @@ -# Detects internal investment activity (fund swaps, reinvestments) by comparing -# holdings snapshots between syncs and marks matching transactions as excluded -# from cashflow. This is provider-agnostic and works with any holdings data. -# -# Usage: -# detector = InvestmentActivityDetector.new(account) -# detector.detect_and_mark_internal_activity(current_holdings, recent_transactions) -# -class InvestmentActivityDetector - def initialize(account) - @account = account - end - - # Class method for inferring activity label from description and amount - # without needing a full detector instance - # @param name [String] Transaction name/description - # @param amount [Numeric] Transaction amount - # @param account [Account, nil] Optional account for context (e.g., retirement plan detection) - # @return [String, nil] Activity label or nil if unknown - def self.infer_label_from_description(name, amount, account = nil) - new(nil).send(:infer_from_description, name, amount, account) - end - - # Call this after syncing transactions for an investment/crypto account - # @param current_holdings [Array] Array of holding objects/hashes from provider - # @param recent_transactions [Array] Recently imported transactions - def detect_and_mark_internal_activity(current_holdings, recent_transactions) - return unless @account.investment? || @account.crypto? - return if current_holdings.blank? - - previous_snapshot = @account.holdings_snapshot_data || [] - - # Find holdings changes that indicate buys/sells - changes = detect_holdings_changes(previous_snapshot, current_holdings) - - # Match changes to transactions and mark them as excluded - changes.each do |change| - matched_entry = find_matching_entry(change, recent_transactions) - next unless matched_entry - - transaction = matched_entry.entryable - - # Only auto-set if not already manually set by user (respect user overrides) - unless matched_entry.locked?(:exclude_from_cashflow) - matched_entry.update!(exclude_from_cashflow: true) - matched_entry.lock_attr!(:exclude_from_cashflow) - - Rails.logger.info( - "InvestmentActivityDetector: Auto-excluded entry #{matched_entry.id} " \ - "(#{matched_entry.name}) as internal #{change[:type]} of #{change[:symbol] || change[:description]}" - ) - end - - # Set activity label if not already set - if transaction.is_a?(Transaction) && transaction.investment_activity_label.blank? - label = infer_activity_label(matched_entry, change[:type]) - transaction.update!(investment_activity_label: label) if label.present? - end - end - - # Store current snapshot for next comparison - save_holdings_snapshot(current_holdings) - end - - private - - # Infer activity label from change type and transaction description - def infer_activity_label(entry, change_type) - # If we know it's a buy or sell from holdings comparison - return "Buy" if change_type == :buy - return "Sell" if change_type == :sell - - # Otherwise try to infer from description - infer_from_description(entry) - end - - # Infer activity label from transaction description - # Can be called with an Entry or with name/amount directly - # @param entry_or_name [Entry, String] Entry object or transaction name - # @param amount [Numeric, nil] Transaction amount (required if entry_or_name is String) - # @param account [Account, nil] Optional account for context (e.g., retirement plan detection) - def infer_from_description(entry_or_name, amount = nil, account = nil) - if entry_or_name.respond_to?(:name) - description = (entry_or_name.name || "").upcase - amount = entry_or_name.amount || 0 - account ||= entry_or_name.try(:account) - else - description = (entry_or_name || "").upcase - amount ||= 0 - end - - # Check if this is a retirement plan account (401k, 403b, etc.) - account_name = (account&.name || "").upcase - retirement_indicators = %w[401K 403B RETIREMENT TOTALSOURCE NETBENEFITS] - retirement_phrases = [ "SAVINGS PLAN", "THRIFT PLAN", "PENSION" ] - is_retirement_plan = retirement_indicators.any? { |ind| account_name.include?(ind) } || - retirement_phrases.any? { |phrase| account_name.include?(phrase) } - - # Check for sweep/money market patterns (but NOT money market FUND purchases) - # INVESTOR CL indicates this is a money market fund, not a sweep - sweep_patterns = %w[SWEEP SETTLEMENT] - money_market_sweep = description.include?("MONEY MARKET") && !description.include?("INVESTOR") - common_money_market_tickers = %w[VMFXX SPAXX FDRXX SWVXX SPRXX] - - if sweep_patterns.any? { |p| description.include?(p) } || - money_market_sweep || - common_money_market_tickers.any? { |t| description == t } - return amount.positive? ? "Sweep Out" : "Sweep In" - end - - # Check for likely interest/dividend on money market funds - # Small amounts (under $5) on money market funds are typically interest income - money_market_fund_patterns = %w[MONEY\ MARKET VMFXX SPAXX FDRXX SWVXX SPRXX VUSXX] - is_money_market_fund = money_market_fund_patterns.any? { |p| description.include?(p) } - - if is_money_market_fund && amount.abs < 5 - # Small money market amounts are interest, not buys/sells - return "Interest" - end - - # Check for dividend patterns - # "CASH" alone typically indicates dividend payout in brokerage feeds (only for inflows) - if description.include?("DIVIDEND") || description.include?("DISTRIBUTION") || - (description == "CASH" && amount < 0) - return "Dividend" - end - - # Check for interest - return "Interest" if description.include?("INTEREST") - - # Check for fees - return "Fee" if description.include?("FEE") || description.include?("CHARGE") - - # Check for reinvestment - return "Reinvestment" if description.include?("REINVEST") - - # Check for exchange/conversion - return "Exchange" if description.include?("EXCHANGE") || description.include?("CONVERSION") - - # Check for contribution patterns - return "Contribution" if description.include?("CONTRIBUTION") || description.include?("DEPOSIT") - - # Check for withdrawal patterns - return "Withdrawal" if description.include?("WITHDRAWAL") || description.include?("DISBURSEMENT") - - # Check for fund names that indicate buy/sell activity - # Positive amount = money out from account perspective = buying securities - # Negative amount = money in = selling securities - fund_patterns = %w[ - INDEX FUND ADMIRAL ETF SHARES TRUST - VANGUARD FIDELITY SCHWAB ISHARES SPDR - 500\ INDEX TOTAL\ MARKET GROWTH BOND - ] - - # Common fund ticker patterns - fund_ticker_patterns = %w[ - VFIAX VTSAX VXUS VBTLX VTIAX VTTVX - VTI VOO VGT VIG VYM VGIT - FXAIX FZROX FSKAX FBALX - SWTSX SWPPX SCHD SCHX - SPY QQQ IVV AGG - IBIT GBTC ETHE - ] - - is_fund_transaction = fund_patterns.any? { |p| description.include?(p) } || - fund_ticker_patterns.any? { |t| description.include?(t) } - - if is_fund_transaction - if is_retirement_plan && amount.negative? - # Negative amount in retirement plan = payroll contribution buying shares - return "Contribution" - else - return amount.positive? ? "Buy" : "Sell" - end - end - - nil # Unknown - user can set manually - end - - def detect_holdings_changes(previous, current) - changes = [] - - current.each do |holding| - prev = find_previous_holding(previous, holding) - - if prev.nil? - # New holding appeared = BUY - changes << { - type: :buy, - symbol: holding_symbol(holding), - description: holding_description(holding), - shares: holding_shares(holding), - cost_basis: holding_cost_basis(holding), - created_at: holding_created_at(holding) - } - elsif holding_shares(holding) > prev_shares(prev) - # Shares increased = BUY - changes << { - type: :buy, - symbol: holding_symbol(holding), - description: holding_description(holding), - shares_delta: holding_shares(holding) - prev_shares(prev), - cost_basis_delta: holding_cost_basis(holding) - prev_cost_basis(prev) - } - elsif holding_shares(holding) < prev_shares(prev) - # Shares decreased = SELL - changes << { - type: :sell, - symbol: holding_symbol(holding), - description: holding_description(holding), - shares_delta: prev_shares(prev) - holding_shares(holding) - } - end - end - - # Check for holdings that completely disappeared = SELL ALL - previous.each do |prev| - unless current.any? { |h| same_holding?(h, prev) } - changes << { - type: :sell, - symbol: prev_symbol(prev), - description: prev_description(prev), - shares: prev_shares(prev) - } - end - end - - changes - end - - def find_matching_entry(change, transactions) - transactions.each do |txn| - entry = txn.respond_to?(:entry) ? txn.entry : txn - next unless entry - next if entry.exclude_from_cashflow? # Already excluded - - # Match by cost_basis amount (for buys with known cost) - if change[:cost_basis].present? && change[:cost_basis].to_d > 0 - amount_diff = (entry.amount.to_d.abs - change[:cost_basis].to_d.abs).abs - return entry if amount_diff < 0.01 - end - - # Match by cost_basis delta (for additional buys) - if change[:cost_basis_delta].present? && change[:cost_basis_delta].to_d > 0 - amount_diff = (entry.amount.to_d.abs - change[:cost_basis_delta].to_d.abs).abs - return entry if amount_diff < 0.01 - end - - # Match by description containing security name/symbol - entry_desc = entry.name&.downcase || "" - - if change[:symbol].present? - return entry if entry_desc.include?(change[:symbol].downcase) - end - - if change[:description].present? - # Match first few words of description for fuzzy matching - desc_words = change[:description].downcase.split.first(3).join(" ") - return entry if desc_words.present? && entry_desc.include?(desc_words) - end - end - - nil - end - - def find_previous_holding(previous, current) - symbol = holding_symbol(current) - return previous.find { |p| prev_symbol(p) == symbol } if symbol.present? - - # Fallback to description matching if no symbol - desc = holding_description(current) - previous.find { |p| prev_description(p) == desc } if desc.present? - end - - def same_holding?(current, previous) - current_symbol = holding_symbol(current) - prev_sym = prev_symbol(previous) - - if current_symbol.present? && prev_sym.present? - current_symbol == prev_sym - else - holding_description(current) == prev_description(previous) - end - end - - def save_holdings_snapshot(holdings) - snapshot_data = holdings.map do |h| - { - "symbol" => holding_symbol(h), - "description" => holding_description(h), - "shares" => holding_shares(h).to_s, - "cost_basis" => holding_cost_basis(h).to_s, - "market_value" => holding_market_value(h).to_s - } - end - - @account.update!( - holdings_snapshot_data: snapshot_data, - holdings_snapshot_at: Time.current - ) - end - - # Normalize access - holdings could be AR objects or hashes from different providers - def holding_symbol(h) - h.try(:symbol) || h.try(:ticker) || h["symbol"] || h[:symbol] || h["ticker"] || h[:ticker] - end - - def holding_description(h) - h.try(:description) || h.try(:name) || h["description"] || h[:description] || h["name"] || h[:name] - end - - def holding_shares(h) - val = h.try(:shares) || h.try(:qty) || h["shares"] || h[:shares] || h["qty"] || h[:qty] - val.to_d - end - - def holding_cost_basis(h) - val = h.try(:cost_basis) || h["cost_basis"] || h[:cost_basis] - val.to_d - end - - def holding_market_value(h) - val = h.try(:market_value) || h.try(:amount) || h["market_value"] || h[:market_value] || h["amount"] || h[:amount] - val.to_d - end - - def holding_created_at(h) - h.try(:created_at) || h["created"] || h[:created] || h["created_at"] || h[:created_at] - end - - # Previous snapshot accessor methods (snapshot is always a hash) - def prev_symbol(p) - p["symbol"] || p[:symbol] - end - - def prev_description(p) - p["description"] || p[:description] - end - - def prev_shares(p) - (p["shares"] || p[:shares]).to_d - end - - def prev_cost_basis(p) - (p["cost_basis"] || p[:cost_basis]).to_d - end -end diff --git a/app/models/lunchflow_account/processor.rb b/app/models/lunchflow_account/processor.rb index d82301dbd..b9c6b2184 100644 --- a/app/models/lunchflow_account/processor.rb +++ b/app/models/lunchflow_account/processor.rb @@ -74,37 +74,10 @@ class LunchflowAccount::Processor return unless [ "Investment", "Crypto" ].include?(lunchflow_account.current_account&.accountable_type) LunchflowAccount::Investments::HoldingsProcessor.new(lunchflow_account).process - - # Detect and mark internal investment activity (fund swaps, reinvestments) - detect_internal_investment_activity rescue => e report_exception(e, "holdings") end - def detect_internal_investment_activity - account = lunchflow_account.current_account - return unless account&.investment? || account&.crypto? - - # Get current holdings from raw payload - current_holdings = lunchflow_account.raw_holdings_payload || [] - return if current_holdings.blank? - - # Get recent transactions (last 30 days to catch any we might have missed) - recent_transactions = account.entries - .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") - .where(date: 30.days.ago.to_date..Date.current) - .where(exclude_from_cashflow: false) - .map(&:entryable) - .compact - - InvestmentActivityDetector.new(account).detect_and_mark_internal_activity( - current_holdings, - recent_transactions - ) - rescue => e - Rails.logger.warn("InvestmentActivityDetector failed for Lunchflow account #{lunchflow_account.id}: #{e.message}") - end - def report_exception(error, context) Sentry.capture_exception(error) do |scope| scope.set_tags( diff --git a/app/models/plaid_account/processor.rb b/app/models/plaid_account/processor.rb index 3dc09c710..4faead9d9 100644 --- a/app/models/plaid_account/processor.rb +++ b/app/models/plaid_account/processor.rb @@ -103,51 +103,10 @@ class PlaidAccount::Processor def process_investments PlaidAccount::Investments::TransactionsProcessor.new(plaid_account, security_resolver: security_resolver).process PlaidAccount::Investments::HoldingsProcessor.new(plaid_account, security_resolver: security_resolver).process - - # Detect and mark internal investment activity (fund swaps, reinvestments) - # Note: Plaid already creates Trade entries for buy/sell, but this catches cash transactions - detect_internal_investment_activity rescue => e report_exception(e) end - def detect_internal_investment_activity - account = AccountProvider.find_by(provider: plaid_account)&.account - return unless account&.investment? || account&.crypto? - - # Get current holdings from raw payload - raw_holdings = plaid_account.raw_investments_payload&.dig("holdings") || [] - return if raw_holdings.blank? - - # Transform to common format - current_holdings = raw_holdings.map do |h| - response = security_resolver.resolve(plaid_security_id: h["security_id"]) - security = response.security - { - "symbol" => security&.ticker, - "description" => security&.name, - "shares" => h["quantity"], - "cost_basis" => h["cost_basis"], - "market_value" => h["institution_value"] - } - end - - # Get recent transactions (last 30 days to catch any we might have missed) - recent_transactions = account.entries - .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") - .where(date: 30.days.ago.to_date..Date.current) - .where(exclude_from_cashflow: false) - .map(&:entryable) - .compact - - InvestmentActivityDetector.new(account).detect_and_mark_internal_activity( - current_holdings, - recent_transactions - ) - rescue => e - Rails.logger.warn("InvestmentActivityDetector failed for Plaid account #{plaid_account.id}: #{e.message}") - end - def process_liabilities case [ plaid_account.plaid_type, plaid_account.plaid_subtype ] when [ "credit", "credit card" ] diff --git a/app/models/rule/action_executor/set_investment_activity_label.rb b/app/models/rule/action_executor/set_investment_activity_label.rb new file mode 100644 index 000000000..21e421292 --- /dev/null +++ b/app/models/rule/action_executor/set_investment_activity_label.rb @@ -0,0 +1,31 @@ +class Rule::ActionExecutor::SetInvestmentActivityLabel < Rule::ActionExecutor + def label + "Set investment activity label" + end + + def type + "select" + end + + def options + Transaction::ACTIVITY_LABELS.map { |l| [ l, l ] } + end + + def execute(transaction_scope, value: nil, ignore_attribute_locks: false, rule_run: nil) + return 0 unless Transaction::ACTIVITY_LABELS.include?(value) + + scope = transaction_scope + + unless ignore_attribute_locks + scope = scope.enrichable(:investment_activity_label) + end + + count_modified_resources(scope) do |txn| + txn.enrich_attribute( + :investment_activity_label, + value, + source: "rule" + ) + end + end +end diff --git a/app/models/rule/registry/transaction_resource.rb b/app/models/rule/registry/transaction_resource.rb index d051b5837..fac1d6667 100644 --- a/app/models/rule/registry/transaction_resource.rb +++ b/app/models/rule/registry/transaction_resource.rb @@ -20,6 +20,7 @@ class Rule::Registry::TransactionResource < Rule::Registry Rule::ActionExecutor::SetTransactionTags.new(rule), Rule::ActionExecutor::SetTransactionMerchant.new(rule), Rule::ActionExecutor::SetTransactionName.new(rule), + Rule::ActionExecutor::SetInvestmentActivityLabel.new(rule), Rule::ActionExecutor::ExcludeTransaction.new(rule) ] diff --git a/app/models/simplefin_account/processor.rb b/app/models/simplefin_account/processor.rb index 68b679772..569f515cd 100644 --- a/app/models/simplefin_account/processor.rb +++ b/app/models/simplefin_account/processor.rb @@ -151,37 +151,10 @@ class SimplefinAccount::Processor return unless simplefin_account.current_account&.accountable_type == "Investment" SimplefinAccount::Investments::TransactionsProcessor.new(simplefin_account).process SimplefinAccount::Investments::HoldingsProcessor.new(simplefin_account).process - - # Detect and mark internal investment activity (fund swaps, reinvestments) - detect_internal_investment_activity rescue => e report_exception(e, "investments") end - def detect_internal_investment_activity - account = simplefin_account.current_account - return unless account&.investment? || account&.crypto? - - # Get current holdings from raw payload - current_holdings = simplefin_account.raw_holdings_payload || [] - return if current_holdings.blank? - - # Get recent transactions (last 30 days to catch any we might have missed) - recent_transactions = account.entries - .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") - .where(date: 30.days.ago.to_date..Date.current) - .where(exclude_from_cashflow: false) - .map(&:entryable) - .compact - - InvestmentActivityDetector.new(account).detect_and_mark_internal_activity( - current_holdings, - recent_transactions - ) - rescue => e - Rails.logger.warn("InvestmentActivityDetector failed for account #{simplefin_account.current_account&.id}: #{e.message}") - end - def process_liabilities case simplefin_account.current_account&.accountable_type when "CreditCard" diff --git a/app/models/simplefin_entry/processor.rb b/app/models/simplefin_entry/processor.rb index 8e6cd88f5..db4b5689b 100644 --- a/app/models/simplefin_entry/processor.rb +++ b/app/models/simplefin_entry/processor.rb @@ -18,8 +18,7 @@ class SimplefinEntry::Processor source: "simplefin", merchant: merchant, notes: notes, - extra: extra_metadata, - investment_activity_label: inferred_activity_label + extra: extra_metadata ) end @@ -205,11 +204,4 @@ class SimplefinEntry::Processor end parts.presence&.join(" | ") end - - # Infer investment activity label from transaction description - # Only returns a label for investment/crypto accounts - def inferred_activity_label - return nil unless account&.investment? || account&.crypto? - InvestmentActivityDetector.infer_label_from_description(name, amount, account) - end end diff --git a/app/models/transaction.rb b/app/models/transaction.rb index 42e3a2677..d1f65a704 100644 --- a/app/models/transaction.rb +++ b/app/models/transaction.rb @@ -20,19 +20,12 @@ class Transaction < ApplicationRecord investment_contribution: "investment_contribution" # Transfer to investment/crypto account, included in budget as investment expense } - # Labels for internal investment activity (auto-exclude from cashflow) - # Only internal shuffling should be excluded, not contributions/dividends/withdrawals - INTERNAL_ACTIVITY_LABELS = %w[Buy Sell Reinvestment Exchange].freeze - # All valid investment activity labels (for UI dropdown) ACTIVITY_LABELS = [ "Buy", "Sell", "Sweep In", "Sweep Out", "Dividend", "Reinvestment", "Interest", "Fee", "Transfer", "Contribution", "Withdrawal", "Exchange", "Other" ].freeze - after_save :sync_exclude_from_cashflow_with_activity_label, - if: :saved_change_to_investment_activity_label? - # Pending transaction scopes - filter based on provider pending flags in extra JSONB # Works with any provider that stores pending status in extra["provider_name"]["pending"] scope :pending, -> { @@ -159,17 +152,4 @@ class Transaction < ApplicationRecord FamilyMerchantAssociation.where(family: family, merchant: merchant).delete_all end - - # Sync exclude_from_cashflow based on activity label - # Internal activities (Buy, Sell, etc.) should be excluded from cashflow - def sync_exclude_from_cashflow_with_activity_label - return unless entry&.account&.investment? || entry&.account&.crypto? - return if entry.locked?(:exclude_from_cashflow) # Respect user's manual setting - - should_exclude = INTERNAL_ACTIVITY_LABELS.include?(investment_activity_label) - - if entry.exclude_from_cashflow != should_exclude - entry.update!(exclude_from_cashflow: should_exclude) - end - end end diff --git a/lib/tasks/investment_labels.rake b/lib/tasks/investment_labels.rake deleted file mode 100644 index 9cefae7c9..000000000 --- a/lib/tasks/investment_labels.rake +++ /dev/null @@ -1,189 +0,0 @@ -# frozen_string_literal: true - -# Backfill investment activity labels for existing transactions -# -# Usage examples: -# # Preview (dry run) - show what labels would be set -# bin/rails 'sure:investments:backfill_labels[dry_run=true]' -# -# # Execute the backfill for all investment/crypto accounts -# bin/rails 'sure:investments:backfill_labels[dry_run=false]' -# -# # Backfill for a specific account -# bin/rails 'sure:investments:backfill_labels[account_id=8b46387c-5aa4-4a92-963a-4392c10999c9,dry_run=false]' -# -# # Force re-label already-labeled transactions -# bin/rails 'sure:investments:backfill_labels[dry_run=false,force=true]' - -namespace :sure do - namespace :investments do - desc "Backfill activity labels for existing investment transactions. Args: account_id (optional), dry_run=true, force=false" - task :backfill_labels, [ :account_id, :dry_run, :force ] => :environment do |_, args| - # Support named args (key=value) - parse all positional args for key=value pairs - kv = {} - [ args[:account_id], args[:dry_run], args[:force] ].each do |raw| - next unless raw.is_a?(String) && raw.include?("=") - k, v = raw.split("=", 2) - kv[k.to_s] = v - end - - # Only use positional args if they don't contain "=" (otherwise they're named args in wrong position) - positional_account_id = args[:account_id] unless args[:account_id].to_s.include?("=") - positional_dry_run = args[:dry_run] unless args[:dry_run].to_s.include?("=") - positional_force = args[:force] unless args[:force].to_s.include?("=") - - account_id = (kv["account_id"] || positional_account_id).presence - dry_raw = (kv["dry_run"] || positional_dry_run).to_s.downcase - force_raw = (kv["force"] || positional_force).to_s.downcase - force = %w[true yes 1].include?(force_raw) - - # Default to dry_run=true unless explicitly disabled - dry_run = if dry_raw.blank? - true - elsif %w[1 true yes y].include?(dry_raw) - true - elsif %w[0 false no n].include?(dry_raw) - false - else - puts({ ok: false, error: "invalid_argument", message: "dry_run must be one of: true/yes/1 or false/no/0" }.to_json) - exit 1 - end - - # Build account scope - accounts = if account_id.present? - Account.where(id: account_id) - else - Account.where(accountable_type: %w[Investment Crypto]) - end - - if accounts.none? - puts({ ok: false, error: "no_accounts", message: "No investment/crypto accounts found" }.to_json) - exit 1 - end - - total_processed = 0 - total_labeled = 0 - total_skipped = 0 - total_errors = 0 - - accounts.find_each do |account| - # Skip non-investment/crypto accounts if processing all - next unless account.investment? || account.crypto? - - acct_processed = 0 - acct_labeled = 0 - acct_skipped = 0 - acct_errors = 0 - - # Find transactions (optionally include already-labeled if force=true) - entries = account.entries - .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") - .includes(:entryable) - - unless force - entries = entries.where("transactions.investment_activity_label IS NULL OR transactions.investment_activity_label = ''") - end - - entries.find_each do |entry| - acct_processed += 1 - total_processed += 1 - - begin - transaction = entry.transaction - current_label = transaction.investment_activity_label - label = InvestmentActivityDetector.infer_label_from_description(entry.name, entry.amount, account) - - # Skip if no label can be inferred - if label.blank? - acct_skipped += 1 - total_skipped += 1 - next - end - - # Skip if label unchanged (when force=true) - if current_label == label - acct_skipped += 1 - total_skipped += 1 - next - end - - if dry_run - if current_label.present? - puts " [DRY RUN] Would relabel '#{entry.name}' (#{entry.amount}) from '#{current_label}' to '#{label}'" - else - puts " [DRY RUN] Would label '#{entry.name}' (#{entry.amount}) as '#{label}'" - end - else - transaction.update!(investment_activity_label: label) - if current_label.present? - puts " Relabeled '#{entry.name}' (#{entry.amount}) from '#{current_label}' to '#{label}'" - else - puts " Labeled '#{entry.name}' (#{entry.amount}) as '#{label}'" - end - end - acct_labeled += 1 - total_labeled += 1 - rescue => e - acct_errors += 1 - total_errors += 1 - puts({ error: e.class.name, message: e.message, entry_id: entry.id }.to_json) - end - end - - puts({ account_id: account.id, account_name: account.name, accountable_type: account.accountable_type, processed: acct_processed, labeled: acct_labeled, skipped: acct_skipped, errors: acct_errors, dry_run: dry_run, force: force }.to_json) - end - - puts({ ok: true, total_processed: total_processed, total_labeled: total_labeled, total_skipped: total_skipped, total_errors: total_errors, dry_run: dry_run }.to_json) - end - - desc "Clear all investment activity labels (for testing). Args: account_id (required), dry_run=true" - task :clear_labels, [ :account_id, :dry_run ] => :environment do |_, args| - kv = {} - [ args[:account_id], args[:dry_run] ].each do |raw| - next unless raw.is_a?(String) && raw.include?("=") - k, v = raw.split("=", 2) - kv[k.to_s] = v - end - - # Only use positional args if they don't contain "=" - positional_account_id = args[:account_id] unless args[:account_id].to_s.include?("=") - positional_dry_run = args[:dry_run] unless args[:dry_run].to_s.include?("=") - - account_id = (kv["account_id"] || positional_account_id).presence - dry_raw = (kv["dry_run"] || positional_dry_run).to_s.downcase - - unless account_id.present? - puts({ ok: false, error: "usage", message: "Provide account_id" }.to_json) - exit 1 - end - - dry_run = if dry_raw.blank? - true - elsif %w[1 true yes y].include?(dry_raw) - true - elsif %w[0 false no n].include?(dry_raw) - false - else - puts({ ok: false, error: "invalid_argument", message: "dry_run must be one of: true/yes/1 or false/no/0" }.to_json) - exit 1 - end - - account = Account.find(account_id) - - count = account.entries - .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id AND entries.entryable_type = 'Transaction'") - .where("transactions.investment_activity_label IS NOT NULL AND transactions.investment_activity_label != ''") - .count - - if dry_run - puts({ ok: true, message: "Would clear #{count} labels", dry_run: true }.to_json) - else - Transaction.joins(:entry) - .where(entries: { account_id: account_id }) - .where("investment_activity_label IS NOT NULL AND investment_activity_label != ''") - .update_all(investment_activity_label: nil) - puts({ ok: true, message: "Cleared #{count} labels", dry_run: false }.to_json) - end - end - end -end diff --git a/test/models/investment_activity_detector_test.rb b/test/models/investment_activity_detector_test.rb deleted file mode 100644 index 68ea44ac2..000000000 --- a/test/models/investment_activity_detector_test.rb +++ /dev/null @@ -1,299 +0,0 @@ -require "test_helper" - -class InvestmentActivityDetectorTest < ActiveSupport::TestCase - include EntriesTestHelper - - setup do - @family = families(:empty) - @investment_account = @family.accounts.create!( - name: "Brokerage", - balance: 10000, - cash_balance: 2000, - currency: "USD", - accountable: Investment.new - ) - @detector = InvestmentActivityDetector.new(@investment_account) - end - - test "detects new holding purchase and marks matching transaction" do - # Create a transaction that matches a new holding purchase - entry = create_transaction( - account: @investment_account, - amount: 1000, - name: "Buy VFIAX" - ) - transaction = entry.transaction - - # Simulate holdings snapshot showing a new holding - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - - # No previous snapshot - @investment_account.update!(holdings_snapshot_data: nil, holdings_snapshot_at: nil) - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - entry.reload - assert entry.exclude_from_cashflow?, "Transaction matching new holding should be excluded from cashflow" - end - - test "detects holding sale and marks matching transaction" do - # Set up previous holdings - previous_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 2000.0, "shares" => 20 } - ] - @investment_account.update!( - holdings_snapshot_data: previous_holdings, - holdings_snapshot_at: 1.day.ago - ) - - # Create a transaction for the sale proceeds (negative = inflow) - entry = create_transaction( - account: @investment_account, - amount: -1000, - name: "Sell VFIAX" - ) - transaction = entry.transaction - - # Current holdings show reduced position - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - entry.reload - assert entry.exclude_from_cashflow?, "Transaction matching holding sale should be excluded from cashflow" - end - - test "respects locked exclude_from_cashflow attribute" do - # Create a transaction and lock the attribute - entry = create_transaction( - account: @investment_account, - amount: 1000, - name: "Buy VFIAX" - ) - transaction = entry.transaction - - # User explicitly set to NOT exclude (and locked it) - entry.update!(exclude_from_cashflow: false) - entry.lock_attr!(:exclude_from_cashflow) - - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - entry.reload - assert_not entry.exclude_from_cashflow?, "Locked attribute should not be overwritten" - end - - test "updates holdings snapshot after detection" do - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 }, - { "symbol" => "IBIT", "cost_basis" => 500.0, "shares" => 5 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, []) - - @investment_account.reload - # Snapshot is normalized with string values and additional fields - snapshot = @investment_account.holdings_snapshot_data - assert_equal 2, snapshot.size - assert_equal "VFIAX", snapshot[0]["symbol"] - assert_equal "1000.0", snapshot[0]["cost_basis"] - assert_equal "10.0", snapshot[0]["shares"] - assert_equal "IBIT", snapshot[1]["symbol"] - assert_not_nil @investment_account.holdings_snapshot_at - end - - test "matches transaction by cost_basis amount within tolerance" do - entry = create_transaction( - account: @investment_account, - amount: 1000.005, # Very close - within 0.01 tolerance - name: "Investment purchase" - ) - transaction = entry.transaction - - # Holding with cost basis close to transaction amount (within 0.01) - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - entry.reload - assert entry.exclude_from_cashflow?, "Should match transaction within tolerance" - end - - test "does not mark unrelated transactions" do - # Create a regular expense transaction - entry = create_transaction( - account: @investment_account, - amount: 50, - name: "Account fee" - ) - transaction = entry.transaction - - # Holdings that don't match - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - entry.reload - assert_not entry.exclude_from_cashflow?, "Unrelated transaction should not be excluded" - end - - test "works with crypto accounts" do - crypto_account = @family.accounts.create!( - name: "Crypto Wallet", - balance: 5000, - currency: "USD", - accountable: Crypto.new - ) - detector = InvestmentActivityDetector.new(crypto_account) - - entry = create_transaction( - account: crypto_account, - amount: 1000, - name: "Buy BTC" - ) - transaction = entry.transaction - - current_holdings = [ - { "symbol" => "BTC", "cost_basis" => 1000.0, "shares" => 0.02 } - ] - - detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - entry.reload - assert entry.exclude_from_cashflow?, "Should work with crypto accounts" - end - - test "handles empty holdings gracefully" do - entry = create_transaction( - account: @investment_account, - amount: 1000, - name: "Some transaction" - ) - transaction = entry.transaction - - # Should not raise, just do nothing - assert_nothing_raised do - @detector.detect_and_mark_internal_activity([], [ transaction ]) - end - - entry.reload - assert_not entry.exclude_from_cashflow? - end - - test "handles nil holdings gracefully" do - entry = create_transaction( - account: @investment_account, - amount: 1000, - name: "Some transaction" - ) - transaction = entry.transaction - - assert_nothing_raised do - @detector.detect_and_mark_internal_activity(nil, [ transaction ]) - end - - entry.reload - assert_not entry.exclude_from_cashflow? - end - - test "sets Buy label for new holding purchase" do - entry = create_transaction( - account: @investment_account, - amount: 1000, - name: "Some investment" - ) - transaction = entry.transaction - - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - transaction.reload - assert_equal "Buy", transaction.investment_activity_label - end - - test "sets Sell label for holding sale" do - previous_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 2000.0, "shares" => 20 } - ] - @investment_account.update!( - holdings_snapshot_data: previous_holdings, - holdings_snapshot_at: 1.day.ago - ) - - entry = create_transaction( - account: @investment_account, - amount: -1000, - name: "VFIAX Sale" - ) - transaction = entry.transaction - - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - transaction.reload - assert_equal "Sell", transaction.investment_activity_label - end - - test "infers Sweep In label from money market description" do - entry = create_transaction( - account: @investment_account, - amount: -500, - name: "VANGUARD FEDERAL MONEY MARKET" - ) - transaction = entry.transaction - - # Call with empty holdings but simulate it being a sweep - # This tests the infer_from_description fallback - current_holdings = [ - { "symbol" => "VMFXX", "cost_basis" => 500.0, "shares" => 500 } - ] - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - transaction.reload - # Should be either "Buy" (from holdings match) or "Sweep In" (from description) - assert transaction.investment_activity_label.present? - end - - test "infers Dividend label from CASH description" do - entry = create_transaction( - account: @investment_account, - amount: -50, - name: "CASH" - ) - transaction = entry.transaction - - # No holdings change, but description-based inference - current_holdings = [ - { "symbol" => "VFIAX", "cost_basis" => 1000.0, "shares" => 10 } - ] - @investment_account.update!( - holdings_snapshot_data: current_holdings, - holdings_snapshot_at: 1.day.ago - ) - - @detector.detect_and_mark_internal_activity(current_holdings, [ transaction ]) - - # Since there's no holdings change, no label gets set via holdings match - # But if we manually test the infer_from_description method... - label = @detector.send(:infer_from_description, entry) - assert_equal "Dividend", label - end -end diff --git a/test/models/rule/action_test.rb b/test/models/rule/action_test.rb index db3933ea0..bcb8846af 100644 --- a/test/models/rule/action_test.rb +++ b/test/models/rule/action_test.rb @@ -100,4 +100,36 @@ class Rule::ActionTest < ActiveSupport::TestCase assert_equal new_name, transaction.reload.entry.name end end + + test "set_investment_activity_label" do + # Does not modify transactions that are locked (user edited them) + @txn1.lock_attr!(:investment_activity_label) + + action = Rule::Action.new( + rule: @transaction_rule, + action_type: "set_investment_activity_label", + value: "Dividend" + ) + + action.apply(@rule_scope) + + assert_nil @txn1.reload.investment_activity_label + + [ @txn2, @txn3 ].each do |transaction| + assert_equal "Dividend", transaction.reload.investment_activity_label + end + end + + test "set_investment_activity_label ignores invalid values" do + action = Rule::Action.new( + rule: @transaction_rule, + action_type: "set_investment_activity_label", + value: "InvalidLabel" + ) + + result = action.apply(@rule_scope) + + assert_equal 0, result + assert_nil @txn1.reload.investment_activity_label + end end diff --git a/test/models/transaction_test.rb b/test/models/transaction_test.rb index 3d7b3236b..3f0f903b0 100644 --- a/test/models/transaction_test.rb +++ b/test/models/transaction_test.rb @@ -35,13 +35,6 @@ class TransactionTest < ActiveSupport::TestCase end end - test "INTERNAL_ACTIVITY_LABELS contains expected labels" do - assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Buy" - assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Sell" - assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Reinvestment" - assert_includes Transaction::INTERNAL_ACTIVITY_LABELS, "Exchange" - end - test "ACTIVITY_LABELS contains all valid labels" do assert_includes Transaction::ACTIVITY_LABELS, "Buy" assert_includes Transaction::ACTIVITY_LABELS, "Sell" From 582eda999ba93d94cc02e867fa55d639d941bac1 Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Mon, 12 Jan 2026 15:10:33 -0500 Subject: [PATCH 44/54] Remove `exclude_from_cashflow` flag and consolidate logic into `excluded` toggle - Removed `exclude_from_cashflow` attribute across models, controllers, and views. - Updated queries to rely solely on the `excluded` flag for filtering transactions and entries. - Simplified migration by consolidating `exclude_from_cashflow` functionality into the existing `excluded` toggle. - Refactored related tests to remove outdated logic and ensured compatibility with the updated implementation. --- app/controllers/accounts_controller.rb | 2 +- app/controllers/reports_controller.rb | 10 ++-- app/controllers/transactions_controller.rb | 2 +- app/models/income_statement/category_stats.rb | 1 - app/models/income_statement/family_stats.rb | 1 - app/models/income_statement/totals.rb | 9 ++-- app/models/transaction/search.rb | 4 +- app/views/transactions/_transaction.html.erb | 6 --- app/views/transactions/show.html.erb | 22 --------- config/locales/views/transactions/en.yml | 4 -- ...0120000_add_investment_cashflow_support.rb | 10 +--- test/models/income_statement_test.rb | 48 ------------------- 12 files changed, 13 insertions(+), 106 deletions(-) diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index b71318b62..1524b25d0 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -36,7 +36,7 @@ class AccountsController < ApplicationController @chart_view = params[:chart_view] || "balance" @tab = params[:tab] @q = params.fetch(:q, {}).permit(:search, status: []) - entries = @account.entries.where(excluded: false).search(@q).reverse_chronological + entries = @account.entries.search(@q).reverse_chronological @pagy, @entries = pagy(entries, limit: params[:per_page] || "10") diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb index 842ff50e8..91593f801 100644 --- a/app/controllers/reports_controller.rb +++ b/app/controllers/reports_controller.rb @@ -338,7 +338,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Transaction", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) + .where(entries: { entryable_type: "Transaction", excluded: false, date: @period.date_range }) .where.not(kind: [ "funds_movement", "one_time", "cc_payment" ]) .includes(entry: :account, category: []) @@ -350,7 +350,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Trade", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) + .where(entries: { entryable_type: "Trade", excluded: false, date: @period.date_range }) .includes(entry: :account, category: []) # Get sort parameters @@ -519,7 +519,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Transaction", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) + .where(entries: { entryable_type: "Transaction", excluded: false, date: @period.date_range }) .where.not(kind: [ "funds_movement", "one_time", "cc_payment" ]) .includes(entry: :account, category: []) @@ -556,7 +556,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Transaction", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) + .where(entries: { entryable_type: "Transaction", excluded: false, date: @period.date_range }) .where.not(kind: [ "funds_movement", "one_time", "cc_payment" ]) .includes(entry: :account, category: []) @@ -567,7 +567,7 @@ class ReportsController < ApplicationController .joins(:entry) .joins(entry: :account) .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Trade", excluded: false, exclude_from_cashflow: false, date: @period.date_range }) + .where(entries: { entryable_type: "Trade", excluded: false, date: @period.date_range }) .includes(entry: :account, category: []) # Group by category, type, and month diff --git a/app/controllers/transactions_controller.rb b/app/controllers/transactions_controller.rb index 03dcbc2db..753b77300 100644 --- a/app/controllers/transactions_controller.rb +++ b/app/controllers/transactions_controller.rb @@ -217,7 +217,7 @@ class TransactionsController < ApplicationController def entry_params entry_params = params.require(:entry).permit( - :name, :date, :amount, :currency, :excluded, :exclude_from_cashflow, :notes, :nature, :entryable_type, + :name, :date, :amount, :currency, :excluded, :notes, :nature, :entryable_type, entryable_attributes: [ :id, :category_id, :merchant_id, :kind, :investment_activity_label, { tag_ids: [] } ] ) diff --git a/app/models/income_statement/category_stats.rb b/app/models/income_statement/category_stats.rb index 31679a662..f4eb2815f 100644 --- a/app/models/income_statement/category_stats.rb +++ b/app/models/income_statement/category_stats.rb @@ -49,7 +49,6 @@ class IncomeStatement::CategoryStats WHERE a.family_id = :family_id AND t.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false - AND ae.exclude_from_cashflow = false AND (t.extra -> 'simplefin' ->> 'pending')::boolean IS DISTINCT FROM true AND (t.extra -> 'plaid' ->> 'pending')::boolean IS DISTINCT FROM true GROUP BY c.id, period, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END diff --git a/app/models/income_statement/family_stats.rb b/app/models/income_statement/family_stats.rb index c96f13b35..48b0d9507 100644 --- a/app/models/income_statement/family_stats.rb +++ b/app/models/income_statement/family_stats.rb @@ -46,7 +46,6 @@ class IncomeStatement::FamilyStats WHERE a.family_id = :family_id AND t.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false - AND ae.exclude_from_cashflow = false AND (t.extra -> 'simplefin' ->> 'pending')::boolean IS DISTINCT FROM true AND (t.extra -> 'plaid' ->> 'pending')::boolean IS DISTINCT FROM true GROUP BY period, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END diff --git a/app/models/income_statement/totals.rb b/app/models/income_statement/totals.rb index 2b5bd07b7..758ae6be3 100644 --- a/app/models/income_statement/totals.rb +++ b/app/models/income_statement/totals.rb @@ -71,8 +71,7 @@ class IncomeStatement::Totals ) WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false - AND ae.exclude_from_cashflow = false - AND a.family_id = :family_id + AND a.family_id = :family_id AND a.status IN ('draft', 'active') GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END; SQL @@ -98,8 +97,7 @@ class IncomeStatement::Totals ) WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment', 'investment_contribution') AND ae.excluded = false - AND ae.exclude_from_cashflow = false - AND a.family_id = :family_id + AND a.family_id = :family_id AND a.status IN ('draft', 'active') GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END SQL @@ -128,8 +126,7 @@ class IncomeStatement::Totals WHERE a.family_id = :family_id AND a.status IN ('draft', 'active') AND ae.excluded = false - AND ae.exclude_from_cashflow = false - AND ae.date BETWEEN :start_date AND :end_date + AND ae.date BETWEEN :start_date AND :end_date GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END, CASE WHEN t.category_id IS NULL THEN true ELSE false END SQL end diff --git a/app/models/transaction/search.rb b/app/models/transaction/search.rb index 3ece18e45..e46a66472 100644 --- a/app/models/transaction/search.rb +++ b/app/models/transaction/search.rb @@ -49,8 +49,8 @@ class Transaction::Search Rails.cache.fetch("transaction_search_totals/#{cache_key_base}") do result = transactions_scope .select( - "COALESCE(SUM(CASE WHEN entries.amount >= 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution') AND entries.exclude_from_cashflow = false THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as expense_total", - "COALESCE(SUM(CASE WHEN entries.amount < 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution') AND entries.exclude_from_cashflow = false THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as income_total", + "COALESCE(SUM(CASE WHEN entries.amount >= 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution') THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as expense_total", + "COALESCE(SUM(CASE WHEN entries.amount < 0 AND transactions.kind NOT IN ('funds_movement', 'cc_payment', 'investment_contribution') THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as income_total", "COUNT(entries.id) as transactions_count" ) .joins( diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index c645d515d..9c52d6acd 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -78,12 +78,6 @@ <% end %> - <% if entry.exclude_from_cashflow? %> - - <%= icon "eye-off", size: "sm", color: "current" %> - - <% end %> - <%# Investment activity label badge %> <% if transaction.investment_activity_label.present? %> "> diff --git a/app/views/transactions/show.html.erb b/app/views/transactions/show.html.erb index 4af991f1a..3d8910dcb 100644 --- a/app/views/transactions/show.html.erb +++ b/app/views/transactions/show.html.erb @@ -203,28 +203,6 @@ <% end %>
-
- <%= styled_form_with model: @entry, - url: transaction_path(@entry), - class: "p-3", - data: { controller: "auto-submit-form" } do |f| %> -
-
-

<%= t(".exclude_from_cashflow") %>

-

- <% if @entry.account.investment? || @entry.account.crypto? %> - <%= t(".exclude_from_cashflow_description_investment") %> - <% else %> - <%= t(".exclude_from_cashflow_description") %> - <% end %> -

-
- - <%= f.toggle :exclude_from_cashflow, { data: { auto_submit_form_target: "auto" } } %> -
- <% end %> -
- <% if @entry.account.investment? || @entry.account.crypto? %>
<%= styled_form_with model: @entry, diff --git a/config/locales/views/transactions/en.yml b/config/locales/views/transactions/en.yml index 267adce43..3b295b2f4 100644 --- a/config/locales/views/transactions/en.yml +++ b/config/locales/views/transactions/en.yml @@ -33,9 +33,6 @@ en: details: Details exclude: Exclude exclude_description: Excluded transactions will be removed from budgeting calculations and reports. - exclude_from_cashflow: Exclude from Cashflow - exclude_from_cashflow_description: Hide from income/expense reports and Sankey chart. Useful for transactions you don't want in cashflow analysis. - exclude_from_cashflow_description_investment: Hide from income/expense reports and Sankey chart. Use for internal investment activity like fund swaps, reinvestments, or money market sweeps. activity_type: Activity Type activity_type_description: Type of investment activity (Buy, Sell, Dividend, etc.). Auto-detected or set manually. one_time_title: One-time %{type} @@ -76,7 +73,6 @@ en: transaction: pending: Pending pending_tooltip: Pending transaction — may change when posted - excluded_from_cashflow_tooltip: Excluded from cashflow reports activity_type_tooltip: Investment activity type possible_duplicate: Duplicate? potential_duplicate_tooltip: This may be a duplicate of another transaction diff --git a/db/migrate/20260110120000_add_investment_cashflow_support.rb b/db/migrate/20260110120000_add_investment_cashflow_support.rb index 26cffbc55..39765f4f1 100644 --- a/db/migrate/20260110120000_add_investment_cashflow_support.rb +++ b/db/migrate/20260110120000_add_investment_cashflow_support.rb @@ -1,13 +1,5 @@ class AddInvestmentCashflowSupport < ActiveRecord::Migration[7.2] + # No-op: exclude_from_cashflow was consolidated into the existing 'excluded' toggle def change - # Flag for excluding from cashflow (user-controllable) - # Used for internal investment activity like fund swaps - add_column :entries, :exclude_from_cashflow, :boolean, default: false, null: false - add_index :entries, :exclude_from_cashflow - - # Holdings snapshot for comparison (provider-agnostic) - # Used to detect internal investment activity by comparing holdings between syncs - add_column :accounts, :holdings_snapshot_data, :jsonb - add_column :accounts, :holdings_snapshot_at, :datetime end end diff --git a/test/models/income_statement_test.rb b/test/models/income_statement_test.rb index ba5fa3e81..e4af61d66 100644 --- a/test/models/income_statement_test.rb +++ b/test/models/income_statement_test.rb @@ -286,35 +286,6 @@ class IncomeStatementTest < ActiveSupport::TestCase assert_equal Money.new(1050, @family.currency), totals.expense_money # 900 + 150 end - # NEW TESTS: exclude_from_cashflow Feature - test "excludes transactions with exclude_from_cashflow flag from totals" do - # Create an expense transaction and mark it as excluded from cashflow - excluded_entry = create_transaction(account: @checking_account, amount: 250, category: @groceries_category) - excluded_entry.update!(exclude_from_cashflow: true) - - income_statement = IncomeStatement.new(@family) - totals = income_statement.totals(date_range: Period.last_30_days.date_range) - - # Should NOT include the excluded transaction - assert_equal 4, totals.transactions_count # Only original 4 transactions - assert_equal Money.new(1000, @family.currency), totals.income_money - assert_equal Money.new(900, @family.currency), totals.expense_money - end - - test "excludes income transactions with exclude_from_cashflow flag" do - # Create income and mark as excluded from cashflow - excluded_income = create_transaction(account: @checking_account, amount: -500, category: @income_category) - excluded_income.update!(exclude_from_cashflow: true) - - income_statement = IncomeStatement.new(@family) - totals = income_statement.totals(date_range: Period.last_30_days.date_range) - - # Should NOT include the excluded income - assert_equal 4, totals.transactions_count - assert_equal Money.new(1000, @family.currency), totals.income_money # Original income only - assert_equal Money.new(900, @family.currency), totals.expense_money - end - test "excludes investment_contribution transactions from income statement" do # Create a transfer to investment account (marked as investment_contribution) investment_contribution = create_transaction( @@ -332,23 +303,4 @@ class IncomeStatementTest < ActiveSupport::TestCase assert_equal Money.new(1000, @family.currency), totals.income_money assert_equal Money.new(900, @family.currency), totals.expense_money end - - test "exclude_from_cashflow works with median calculations" do - # Clear existing transactions - Entry.joins(:account).where(accounts: { family_id: @family.id }).destroy_all - - # Create expenses: 100, 200, 300 - create_transaction(account: @checking_account, amount: 100, category: @groceries_category) - create_transaction(account: @checking_account, amount: 200, category: @groceries_category) - excluded_entry = create_transaction(account: @checking_account, amount: 300, category: @groceries_category) - - # Exclude the 300 transaction from cashflow - excluded_entry.update!(exclude_from_cashflow: true) - - income_statement = IncomeStatement.new(@family) - - # Median should only consider non-excluded transactions (100, 200) - # Monthly total = 300, so median = 300.0 - assert_equal 300.0, income_statement.median_expense(interval: "month") - end end From 2f20d715a492e4e18cec8084a79fc4a961c59e61 Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Mon, 12 Jan 2026 15:26:17 -0500 Subject: [PATCH 45/54] Remove exclude_from_cashflow column from schema This column was never meant to be added - the migration is a no-op. The schema.rb was incorrectly committed with this column during rebase. Co-Authored-By: Claude Opus 4.5 --- db/schema.rb | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 067d2c9f6..ad548839c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -342,14 +342,12 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.jsonb "locked_attributes", default: {} t.string "external_id" t.string "source" - t.boolean "exclude_from_cashflow", default: false, null: false t.index "lower((name)::text)", name: "index_entries_on_lower_name" t.index ["account_id", "date"], name: "index_entries_on_account_id_and_date" t.index ["account_id", "source", "external_id"], name: "index_entries_on_account_source_and_external_id", unique: true, where: "((external_id IS NOT NULL) AND (source IS NOT NULL))" t.index ["account_id"], name: "index_entries_on_account_id" t.index ["date"], name: "index_entries_on_date" t.index ["entryable_type"], name: "index_entries_on_entryable_type" - t.index ["exclude_from_cashflow"], name: "index_entries_on_exclude_from_cashflow" t.index ["import_id"], name: "index_entries_on_import_id" end @@ -476,6 +474,22 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.index ["merchant_id"], name: "index_family_merchant_associations_on_merchant_id" end + create_table "flipper_features", force: :cascade do |t| + t.string "key", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["key"], name: "index_flipper_features_on_key", unique: true + end + + create_table "flipper_gates", force: :cascade do |t| + t.string "feature_key", null: false + t.string "key", null: false + t.text "value" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["feature_key", "key", "value"], name: "index_flipper_gates_on_feature_key_and_key_and_value", unique: true + end + create_table "holdings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "account_id", null: false t.uuid "security_id", null: false @@ -800,6 +814,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.datetime "last_authenticated_at" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "issuer" + t.index ["issuer"], name: "index_oidc_identities_on_issuer" t.index ["provider", "uid"], name: "index_oidc_identities_on_provider_and_uid", unique: true t.index ["user_id"], name: "index_oidc_identities_on_user_id" end @@ -1055,6 +1071,38 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do t.index ["status"], name: "index_simplefin_items_on_status" end + create_table "sso_audit_logs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "user_id" + t.string "event_type", null: false + t.string "provider" + t.string "ip_address" + t.string "user_agent" + t.jsonb "metadata", default: {}, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["created_at"], name: "index_sso_audit_logs_on_created_at" + t.index ["event_type"], name: "index_sso_audit_logs_on_event_type" + t.index ["user_id", "created_at"], name: "index_sso_audit_logs_on_user_id_and_created_at" + t.index ["user_id"], name: "index_sso_audit_logs_on_user_id" + end + + create_table "sso_providers", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "strategy", null: false + t.string "name", null: false + t.string "label", null: false + t.string "icon" + t.boolean "enabled", default: true, null: false + t.string "issuer" + t.string "client_id" + t.string "client_secret" + t.string "redirect_uri" + t.jsonb "settings", default: {}, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["enabled"], name: "index_sso_providers_on_enabled" + t.index ["name"], name: "index_sso_providers_on_name", unique: true + end + create_table "subscriptions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "family_id", null: false t.string "status", null: false @@ -1289,6 +1337,7 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_12_065106) do add_foreign_key "sessions", "users" add_foreign_key "simplefin_accounts", "simplefin_items" add_foreign_key "simplefin_items", "families" + add_foreign_key "sso_audit_logs", "users" add_foreign_key "subscriptions", "families" add_foreign_key "syncs", "syncs", column: "parent_id" add_foreign_key "taggings", "tags" From 308a4ab048d036d70756251202cfce6cf118e8e1 Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Mon, 12 Jan 2026 16:04:53 -0500 Subject: [PATCH 46/54] Refactor Plaid transaction type mapping and improve label handling - Updated `PLAID_TYPE_TO_LABEL` in `TransactionsProcessor` to consolidate labels ("Cancel" and "Cash" now mapped to "Other"). - Adjusted `label_from_plaid_type` to return "Other" as the default fallback. - Enhanced tests to include additional valid activity labels and ensure label consistency. - Minor fixes to locale keys for transaction views. --- .../investments/transactions_processor.rb | 7 ++-- app/models/transaction.rb | 2 +- config/locales/views/transactions/en.yml | 35 +++++++++---------- test/models/transaction_test.rb | 6 ++++ 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/app/models/plaid_account/investments/transactions_processor.rb b/app/models/plaid_account/investments/transactions_processor.rb index ded2a473f..d90657ce6 100644 --- a/app/models/plaid_account/investments/transactions_processor.rb +++ b/app/models/plaid_account/investments/transactions_processor.rb @@ -2,11 +2,12 @@ class PlaidAccount::Investments::TransactionsProcessor SecurityNotFoundError = Class.new(StandardError) # Map Plaid investment transaction types to activity labels + # All values must be valid Transaction::ACTIVITY_LABELS PLAID_TYPE_TO_LABEL = { "buy" => "Buy", "sell" => "Sell", - "cancel" => "Cancelled", - "cash" => "Cash", + "cancel" => "Other", + "cash" => "Other", "fee" => "Fee", "transfer" => "Transfer", "dividend" => "Dividend", @@ -92,7 +93,7 @@ class PlaidAccount::Investments::TransactionsProcessor def label_from_plaid_type(transaction) plaid_type = transaction["type"]&.downcase - PLAID_TYPE_TO_LABEL[plaid_type] || plaid_type&.titleize + PLAID_TYPE_TO_LABEL[plaid_type] || "Other" end def transactions diff --git a/app/models/transaction.rb b/app/models/transaction.rb index d1f65a704..e1a86afb5 100644 --- a/app/models/transaction.rb +++ b/app/models/transaction.rb @@ -17,7 +17,7 @@ class Transaction < ApplicationRecord cc_payment: "cc_payment", # A CC payment, excluded from budget analytics (CC payments offset the sum of expense transactions) loan_payment: "loan_payment", # A payment to a Loan account, treated as an expense in budgets one_time: "one_time", # A one-time expense/income, excluded from budget analytics - investment_contribution: "investment_contribution" # Transfer to investment/crypto account, included in budget as investment expense + investment_contribution: "investment_contribution" # Transfer to investment/crypto account, excluded from budget analytics } # All valid investment activity labels (for UI dropdown) diff --git a/config/locales/views/transactions/en.yml b/config/locales/views/transactions/en.yml index 3b295b2f4..a53c28d85 100644 --- a/config/locales/views/transactions/en.yml +++ b/config/locales/views/transactions/en.yml @@ -51,25 +51,24 @@ en: withdrawal: Withdrawal exchange: Exchange other: Other - mark_recurring: Mark as Recurring - mark_recurring_subtitle: Track this as a recurring transaction. Amount variance is automatically calculated from past 6 months of similar transactions. - mark_recurring_title: Recurring Transaction - merchant_label: Merchant - name_label: Name - nature: Type - none: "(none)" - note_label: Notes - note_placeholder: Enter a note - overview: Overview - settings: Settings - tags_label: Tags - uncategorized: "(uncategorized)" - potential_duplicate_title: Possible duplicate detected - potential_duplicate_description: This pending transaction may be the same as the posted transaction below. If so, merge them to avoid double-counting. - merge_duplicate: Yes, merge them - keep_both: No, keep both loan_payment: Loan Payment - transfer: Transfer + mark_recurring: Mark as Recurring + mark_recurring_subtitle: Track this as a recurring transaction. Amount variance is automatically calculated from past 6 months of similar transactions. + mark_recurring_title: Recurring Transaction + merchant_label: Merchant + name_label: Name + nature: Type + none: "(none)" + note_label: Notes + note_placeholder: Enter a note + overview: Overview + settings: Settings + tags_label: Tags + uncategorized: "(uncategorized)" + potential_duplicate_title: Possible duplicate detected + potential_duplicate_description: This pending transaction may be the same as the posted transaction below. If so, merge them to avoid double-counting. + merge_duplicate: Yes, merge them + keep_both: No, keep both transaction: pending: Pending pending_tooltip: Pending transaction — may change when posted diff --git a/test/models/transaction_test.rb b/test/models/transaction_test.rb index 3f0f903b0..9064f6b97 100644 --- a/test/models/transaction_test.rb +++ b/test/models/transaction_test.rb @@ -41,7 +41,13 @@ class TransactionTest < ActiveSupport::TestCase assert_includes Transaction::ACTIVITY_LABELS, "Sweep In" assert_includes Transaction::ACTIVITY_LABELS, "Sweep Out" assert_includes Transaction::ACTIVITY_LABELS, "Dividend" + assert_includes Transaction::ACTIVITY_LABELS, "Reinvestment" assert_includes Transaction::ACTIVITY_LABELS, "Interest" assert_includes Transaction::ACTIVITY_LABELS, "Fee" + assert_includes Transaction::ACTIVITY_LABELS, "Transfer" + assert_includes Transaction::ACTIVITY_LABELS, "Contribution" + assert_includes Transaction::ACTIVITY_LABELS, "Withdrawal" + assert_includes Transaction::ACTIVITY_LABELS, "Exchange" + assert_includes Transaction::ACTIVITY_LABELS, "Other" end end From 8f56aa287713b39abfc8d04ee92b87b6046c5609 Mon Sep 17 00:00:00 2001 From: Pieter Date: Mon, 12 Jan 2026 22:08:59 +0100 Subject: [PATCH 47/54] Fix duplicate category menu IDs for mobile/desktop Mobile and desktop transaction category menus now use variant-specific DOM IDs and the turbo stream response updates both targets. This prevents duplicate IDs that caused the menu not to close on first click on desktop. --- .../transaction_categories_controller.rb | 11 ++++++++--- app/views/transactions/_transaction.html.erb | 4 ++-- .../transactions/_transaction_category.html.erb | 4 ++-- app/views/transfers/update.turbo_stream.erb | 16 ++++++++++++---- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/app/controllers/transaction_categories_controller.rb b/app/controllers/transaction_categories_controller.rb index 3fe8207dc..d8fc64e96 100644 --- a/app/controllers/transaction_categories_controller.rb +++ b/app/controllers/transaction_categories_controller.rb @@ -24,9 +24,14 @@ class TransactionCategoriesController < ApplicationController format.turbo_stream do render turbo_stream: [ turbo_stream.replace( - dom_id(transaction, :category_menu), - partial: "categories/menu", - locals: { transaction: transaction } + dom_id(transaction, "category_menu_mobile"), + partial: "transactions/transaction_category", + locals: { transaction: transaction, variant: "mobile" } + ), + turbo_stream.replace( + dom_id(transaction, "category_menu_desktop"), + partial: "transactions/transaction_category", + locals: { transaction: transaction, variant: "desktop" } ), turbo_stream.replace( "category_name_mobile_#{transaction.id}", diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index ec89bb308..467bec3db 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -36,7 +36,7 @@ <% end %>
- <%= render "transactions/transaction_category", transaction: transaction %> + <%= render "transactions/transaction_category", transaction: transaction, variant: "mobile" %> <% if transaction.merchant&.logo_url.present? %> <%= image_tag transaction.merchant.logo_url, class: "w-5 h-5 rounded-full absolute -bottom-1 -right-1 border border-secondary pointer-events-none", @@ -137,7 +137,7 @@
diff --git a/app/views/transactions/_transaction_category.html.erb b/app/views/transactions/_transaction_category.html.erb index e124c0213..3b5c8d1b9 100644 --- a/app/views/transactions/_transaction_category.html.erb +++ b/app/views/transactions/_transaction_category.html.erb @@ -1,6 +1,6 @@ -<%# locals: (transaction:) %> +<%# locals: (transaction:, variant:) %> -
"> +
"> <% if transaction.transfer&.categorizable? || transaction.transfer.nil? %> <%= render "categories/menu", transaction: transaction %> <% else %> diff --git a/app/views/transfers/update.turbo_stream.erb b/app/views/transfers/update.turbo_stream.erb index 82a503d0f..929a63ab5 100644 --- a/app/views/transfers/update.turbo_stream.erb +++ b/app/views/transfers/update.turbo_stream.erb @@ -2,13 +2,21 @@ <%= turbo_stream.replace @transfer.inflow_transaction.entry %> <%= turbo_stream.replace @transfer.outflow_transaction.entry %> - <%= turbo_stream.replace dom_id(@transfer.inflow_transaction, "category_menu"), + <%= turbo_stream.replace dom_id(@transfer.inflow_transaction, "category_menu_mobile"), partial: "transactions/transaction_category", - locals: { transaction: @transfer.inflow_transaction } %> + locals: { transaction: @transfer.inflow_transaction, variant: "mobile" } %> - <%= turbo_stream.replace dom_id(@transfer.outflow_transaction, "category_menu"), + <%= turbo_stream.replace dom_id(@transfer.inflow_transaction, "category_menu_desktop"), partial: "transactions/transaction_category", - locals: { transaction: @transfer.outflow_transaction } %> + locals: { transaction: @transfer.inflow_transaction, variant: "desktop" } %> + + <%= turbo_stream.replace dom_id(@transfer.outflow_transaction, "category_menu_mobile"), + partial: "transactions/transaction_category", + locals: { transaction: @transfer.outflow_transaction, variant: "mobile" } %> + + <%= turbo_stream.replace dom_id(@transfer.outflow_transaction, "category_menu_desktop"), + partial: "transactions/transaction_category", + locals: { transaction: @transfer.outflow_transaction, variant: "desktop" } %> <%= turbo_stream.replace dom_id(@transfer.inflow_transaction, "transfer_match"), partial: "transactions/transfer_match", From 5c300989a7142ad8c61fcc8e578c74f8854d9b2b Mon Sep 17 00:00:00 2001 From: Josh Waldrep Date: Mon, 12 Jan 2026 16:17:36 -0500 Subject: [PATCH 48/54] Remove unused `loan_payment` key from transaction locale --- config/locales/views/transactions/en.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/locales/views/transactions/en.yml b/config/locales/views/transactions/en.yml index a53c28d85..ae7bb1cb3 100644 --- a/config/locales/views/transactions/en.yml +++ b/config/locales/views/transactions/en.yml @@ -51,7 +51,6 @@ en: withdrawal: Withdrawal exchange: Exchange other: Other - loan_payment: Loan Payment mark_recurring: Mark as Recurring mark_recurring_subtitle: Track this as a recurring transaction. Amount variance is automatically calculated from past 6 months of similar transactions. mark_recurring_title: Recurring Transaction From 62dabb6971cec8bfb0fad48cfbc5010384a569d0 Mon Sep 17 00:00:00 2001 From: Lazy Bone <89256478+dwvwdv@users.noreply.github.com> Date: Tue, 13 Jan 2026 16:27:39 +0800 Subject: [PATCH 49/54] Fix: Transaction Sync Issues & Enhanced Debugging (#632) * Fix mobile app to fetch all transactions with pagination The mobile app was only fetching 25 transactions per account because: 1. TransactionsService didn't pass pagination parameters to the API 2. The backend defaults to 25 records per page when no per_page is specified 3. SyncService didn't implement pagination to fetch all pages Changes: - Updated TransactionsService.getTransactions() to accept page and perPage parameters - Modified the method to extract and return pagination metadata from API response - Updated SyncService.syncFromServer() to fetch all pages (up to 100 per page) - Added pagination loop to continue fetching until all pages are retrieved - Enhanced logging to show pagination progress This ensures users see all their transactions in the mobile app, not just the first 25. * Add clear local data feature and enhanced sync logging Added features: 1. Clear Local Data button in Settings - Allows users to clear all cached transactions and accounts - Shows confirmation dialog before clearing - Displays success/error feedback 2. Enhanced sync logging for debugging - Added detailed logs in syncFromServer to track pagination - Shows page-by-page progress with transaction counts - Logs pagination metadata (total pages, total count, etc.) - Tracks upsert progress every 50 transactions - Added clear section markers for easier log reading 3. Simplified upsertTransactionFromServer logging - Removed verbose debug logs to reduce noise - Keeps only essential error/warning logs This will help users troubleshoot sync issues by: - Clearing stale data and forcing a fresh sync - Providing detailed logs to identify where sync might fail * Fix transaction accountId parsing from API response The mobile app was only showing 25 transactions per account because: - The backend API returns account info in nested format: {"account": {"id": "xxx"}} - The mobile Transaction model expected flat format: {"account_id": "xxx"} - When parsing, accountId was always empty, so database queries by account_id returned incomplete results Changes: 1. Updated Transaction.fromJson to handle both formats: - New format: {"account": {"id": "xxx", "name": "..."}} - Old format: {"account_id": "xxx"} (for backward compatibility) 2. Fixed classification/nature field parsing: - Backend sends "classification" field (income/expense) - Mobile uses "nature" field - Now handles both fields correctly 3. Added debug logging to identify empty accountId issues: - Logs first transaction's accountId when syncing - Counts and warns about transactions with empty accountId - Shows critical errors when trying to save with empty accountId This ensures all transactions from the server are correctly associated with their accounts in the local database. --------- Co-authored-by: Claude --- mobile/lib/models/transaction.dart | 23 +++- mobile/lib/screens/settings_screen.dart | 86 ++++++++++++ .../lib/services/offline_storage_service.dart | 31 ++++- mobile/lib/services/sync_service.dart | 123 +++++++++++++----- mobile/lib/services/transactions_service.dart | 25 +++- 5 files changed, 243 insertions(+), 45 deletions(-) diff --git a/mobile/lib/models/transaction.dart b/mobile/lib/models/transaction.dart index 8d2e93b3a..291f571c0 100644 --- a/mobile/lib/models/transaction.dart +++ b/mobile/lib/models/transaction.dart @@ -20,14 +20,33 @@ class Transaction { }); factory Transaction.fromJson(Map json) { + // Handle both API formats: + // 1. New format: {"account": {"id": "xxx", "name": "..."}} + // 2. Old format: {"account_id": "xxx"} + String accountId = ''; + if (json['account'] != null && json['account'] is Map) { + accountId = json['account']['id']?.toString() ?? ''; + } else if (json['account_id'] != null) { + accountId = json['account_id']?.toString() ?? ''; + } + + // Handle classification (from backend) or nature (from mobile) + String nature = 'expense'; + if (json['classification'] != null) { + final classification = json['classification']?.toString().toLowerCase() ?? ''; + nature = classification == 'income' ? 'income' : 'expense'; + } else if (json['nature'] != null) { + nature = json['nature']?.toString() ?? 'expense'; + } + return Transaction( id: json['id']?.toString(), - accountId: json['account_id']?.toString() ?? '', + accountId: accountId, name: json['name']?.toString() ?? '', date: json['date']?.toString() ?? '', amount: json['amount']?.toString() ?? '0', currency: json['currency']?.toString() ?? '', - nature: json['nature']?.toString() ?? 'expense', + nature: nature, notes: json['notes']?.toString(), ); } diff --git a/mobile/lib/screens/settings_screen.dart b/mobile/lib/screens/settings_screen.dart index 5ff79ae69..0f5518c8c 100644 --- a/mobile/lib/screens/settings_screen.dart +++ b/mobile/lib/screens/settings_screen.dart @@ -1,10 +1,73 @@ import 'package:flutter/material.dart'; import 'package:provider/provider.dart'; import '../providers/auth_provider.dart'; +import '../services/offline_storage_service.dart'; +import '../services/log_service.dart'; class SettingsScreen extends StatelessWidget { const SettingsScreen({super.key}); + Future _handleClearLocalData(BuildContext context) async { + final confirmed = await showDialog( + context: context, + builder: (context) => AlertDialog( + title: const Text('Clear Local Data'), + content: const Text( + 'This will delete all locally cached transactions and accounts. ' + 'Your data on the server will not be affected. ' + 'Are you sure you want to continue?' + ), + actions: [ + TextButton( + onPressed: () => Navigator.pop(context, false), + child: const Text('Cancel'), + ), + TextButton( + onPressed: () => Navigator.pop(context, true), + style: TextButton.styleFrom( + foregroundColor: Theme.of(context).colorScheme.error, + ), + child: const Text('Clear Data'), + ), + ], + ), + ); + + if (confirmed == true && context.mounted) { + try { + final offlineStorage = OfflineStorageService(); + final log = LogService.instance; + + log.info('Settings', 'Clearing all local data...'); + await offlineStorage.clearAllData(); + log.info('Settings', 'Local data cleared successfully'); + + if (context.mounted) { + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('Local data cleared successfully. Pull to refresh to sync from server.'), + backgroundColor: Colors.green, + duration: Duration(seconds: 3), + ), + ); + } + } catch (e) { + final log = LogService.instance; + log.error('Settings', 'Failed to clear local data: $e'); + + if (context.mounted) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar( + content: Text('Failed to clear local data: $e'), + backgroundColor: Colors.red, + duration: const Duration(seconds: 3), + ), + ); + } + } + } + } + Future _handleLogout(BuildContext context) async { final confirmed = await showDialog( context: context, @@ -102,6 +165,29 @@ class SettingsScreen extends StatelessWidget { const Divider(), + // Data Management Section + const Padding( + padding: EdgeInsets.fromLTRB(16, 16, 16, 8), + child: Text( + 'Data Management', + style: TextStyle( + fontSize: 14, + fontWeight: FontWeight.bold, + color: Colors.grey, + ), + ), + ), + + // Clear local data button + ListTile( + leading: const Icon(Icons.delete_outline), + title: const Text('Clear Local Data'), + subtitle: const Text('Remove all cached transactions and accounts'), + onTap: () => _handleClearLocalData(context), + ), + + const Divider(), + // Sign out button Padding( padding: const EdgeInsets.all(16), diff --git a/mobile/lib/services/offline_storage_service.dart b/mobile/lib/services/offline_storage_service.dart index f6df78743..9f5567dc6 100644 --- a/mobile/lib/services/offline_storage_service.dart +++ b/mobile/lib/services/offline_storage_service.dart @@ -171,18 +171,31 @@ class OfflineStorageService { Future syncTransactionsFromServer(List serverTransactions) async { _log.info('OfflineStorage', 'syncTransactionsFromServer called with ${serverTransactions.length} transactions from server'); + // Log first transaction's accountId for debugging + if (serverTransactions.isNotEmpty) { + final firstTx = serverTransactions.first; + _log.info('OfflineStorage', 'First transaction: id=${firstTx.id}, accountId="${firstTx.accountId}", name="${firstTx.name}"'); + } + // Use upsert logic instead of clear + insert to preserve recently uploaded transactions _log.info('OfflineStorage', 'Upserting all transactions from server (preserving pending/failed)'); int upsertedCount = 0; + int emptyAccountIdCount = 0; for (final transaction in serverTransactions) { if (transaction.id != null) { + if (transaction.accountId.isEmpty) { + emptyAccountIdCount++; + } await upsertTransactionFromServer(transaction); upsertedCount++; } } _log.info('OfflineStorage', 'Upserted $upsertedCount transactions from server'); + if (emptyAccountIdCount > 0) { + _log.error('OfflineStorage', 'WARNING: $emptyAccountIdCount transactions had EMPTY accountId!'); + } } Future upsertTransactionFromServer( @@ -199,15 +212,22 @@ class OfflineStorageService { ? accountId : transaction.accountId; - _log.debug('OfflineStorage', 'Upserting transaction ${transaction.id}: accountId="${transaction.accountId}" -> effective="$effectiveAccountId"'); + // Log if transaction has empty accountId + if (transaction.accountId.isEmpty) { + _log.warning('OfflineStorage', 'Transaction ${transaction.id} has empty accountId from server! Provided accountId: $accountId, effective: $effectiveAccountId'); + } // Check if we already have this transaction final existing = await getTransactionByServerId(transaction.id!); if (existing != null) { - _log.debug('OfflineStorage', 'Updating existing transaction (localId: ${existing.localId}, was ${existing.syncStatus})'); // Update existing transaction, preserving its accountId if effectiveAccountId is empty final finalAccountId = effectiveAccountId.isEmpty ? existing.accountId : effectiveAccountId; + + if (finalAccountId.isEmpty) { + _log.error('OfflineStorage', 'CRITICAL: Updating transaction ${transaction.id} with EMPTY accountId!'); + } + final updated = OfflineTransaction( id: transaction.id, localId: existing.localId, @@ -221,10 +241,12 @@ class OfflineStorageService { syncStatus: SyncStatus.synced, ); await _dbHelper.updateTransaction(existing.localId, updated.toDatabaseMap()); - _log.debug('OfflineStorage', 'Transaction updated successfully with accountId="$finalAccountId"'); } else { - _log.debug('OfflineStorage', 'Inserting new transaction with accountId="$effectiveAccountId"'); // Insert new transaction + if (effectiveAccountId.isEmpty) { + _log.error('OfflineStorage', 'CRITICAL: Inserting transaction ${transaction.id} with EMPTY accountId!'); + } + final offlineTransaction = OfflineTransaction( id: transaction.id, localId: _uuid.v4(), @@ -238,7 +260,6 @@ class OfflineStorageService { syncStatus: SyncStatus.synced, ); await _dbHelper.insertTransaction(offlineTransaction.toDatabaseMap()); - _log.debug('OfflineStorage', 'Transaction inserted successfully'); } } diff --git a/mobile/lib/services/sync_service.dart b/mobile/lib/services/sync_service.dart index e7c8f328d..82abbf575 100644 --- a/mobile/lib/services/sync_service.dart +++ b/mobile/lib/services/sync_service.dart @@ -217,49 +217,100 @@ class SyncService with ChangeNotifier { String? accountId, }) async { try { - _log.debug('SyncService', 'Fetching transactions from server (accountId: $accountId)'); - final result = await _transactionsService.getTransactions( - accessToken: accessToken, - accountId: accountId, - ); + _log.info('SyncService', '========== SYNC FROM SERVER START =========='); + _log.info('SyncService', 'Fetching transactions from server (accountId: ${accountId ?? "ALL"})'); - if (result['success'] == true) { - final transactions = (result['transactions'] as List?) - ?.cast() ?? []; + List allTransactions = []; + int currentPage = 1; + int totalPages = 1; + const int perPage = 100; // Use maximum allowed by backend - _log.info('SyncService', 'Received ${transactions.length} transactions from server'); + // Fetch all pages + while (currentPage <= totalPages) { + _log.info('SyncService', '>>> Fetching page $currentPage of $totalPages (perPage: $perPage)'); - // Update local cache with server data - if (accountId == null) { - _log.debug('SyncService', 'Full sync - clearing and replacing all transactions'); - // Full sync - replace all transactions - await _offlineStorage.syncTransactionsFromServer(transactions); + final result = await _transactionsService.getTransactions( + accessToken: accessToken, + accountId: accountId, + page: currentPage, + perPage: perPage, + ); + + _log.debug('SyncService', 'API call completed for page $currentPage, success: ${result['success']}'); + + if (result['success'] == true) { + final pageTransactions = (result['transactions'] as List?) + ?.cast() ?? []; + + _log.info('SyncService', 'Page $currentPage returned ${pageTransactions.length} transactions'); + allTransactions.addAll(pageTransactions); + _log.info('SyncService', 'Total transactions accumulated: ${allTransactions.length}'); + + // Extract pagination info if available + final pagination = result['pagination'] as Map?; + if (pagination != null) { + final prevTotalPages = totalPages; + totalPages = pagination['total_pages'] as int? ?? 1; + final totalCount = pagination['total_count'] as int? ?? 0; + final currentPageFromApi = pagination['page'] as int? ?? currentPage; + final perPageFromApi = pagination['per_page'] as int? ?? perPage; + + _log.info('SyncService', 'Pagination info: page=$currentPageFromApi/$totalPages, per_page=$perPageFromApi, total_count=$totalCount'); + + if (prevTotalPages != totalPages) { + _log.info('SyncService', 'Total pages updated from $prevTotalPages to $totalPages'); + } + } else { + // No pagination info means this is the only page + _log.warning('SyncService', 'No pagination info in response - assuming single page'); + totalPages = currentPage; + } + + _log.info('SyncService', 'Moving to next page (current: $currentPage, total: $totalPages)'); + currentPage++; } else { - _log.debug('SyncService', 'Partial sync - upserting ${transactions.length} transactions for account $accountId'); - // Partial sync - upsert transactions - for (final transaction in transactions) { - _log.debug('SyncService', 'Upserting transaction ${transaction.id} (accountId from server: "${transaction.accountId}", provided: "$accountId")'); - await _offlineStorage.upsertTransactionFromServer( - transaction, - accountId: accountId, - ); + _log.error('SyncService', 'Server returned error on page $currentPage: ${result['error']}'); + return SyncResult( + success: false, + error: result['error'] as String? ?? 'Failed to sync from server', + ); + } + } + + _log.info('SyncService', '>>> Pagination loop completed. Fetched ${currentPage - 1} pages'); + _log.info('SyncService', '>>> Received total of ${allTransactions.length} transactions from server'); + + // Update local cache with server data + _log.info('SyncService', '========== UPDATING LOCAL CACHE =========='); + if (accountId == null) { + _log.info('SyncService', 'Full sync - clearing and replacing all transactions'); + // Full sync - replace all transactions + await _offlineStorage.syncTransactionsFromServer(allTransactions); + } else { + _log.info('SyncService', 'Partial sync - upserting ${allTransactions.length} transactions for account $accountId'); + // Partial sync - upsert transactions + int upsertCount = 0; + for (final transaction in allTransactions) { + await _offlineStorage.upsertTransactionFromServer( + transaction, + accountId: accountId, + ); + upsertCount++; + if (upsertCount % 50 == 0) { + _log.info('SyncService', 'Upserted $upsertCount/${allTransactions.length} transactions'); } } - - _lastSyncTime = DateTime.now(); - notifyListeners(); - - return SyncResult( - success: true, - syncedCount: transactions.length, - ); - } else { - _log.error('SyncService', 'Server returned error: ${result['error']}'); - return SyncResult( - success: false, - error: result['error'] as String? ?? 'Failed to sync from server', - ); + _log.info('SyncService', 'Completed upserting $upsertCount transactions'); } + + _log.info('SyncService', '========== SYNC FROM SERVER COMPLETE =========='); + _lastSyncTime = DateTime.now(); + notifyListeners(); + + return SyncResult( + success: true, + syncedCount: allTransactions.length, + ); } catch (e) { _log.error('SyncService', 'Exception in syncFromServer: $e'); return SyncResult( diff --git a/mobile/lib/services/transactions_service.dart b/mobile/lib/services/transactions_service.dart index 93c65c6c4..e22c6d964 100644 --- a/mobile/lib/services/transactions_service.dart +++ b/mobile/lib/services/transactions_service.dart @@ -75,10 +75,24 @@ class TransactionsService { Future> getTransactions({ required String accessToken, String? accountId, + int? page, + int? perPage, }) async { + final Map queryParams = {}; + + if (accountId != null) { + queryParams['account_id'] = accountId; + } + if (page != null) { + queryParams['page'] = page.toString(); + } + if (perPage != null) { + queryParams['per_page'] = perPage.toString(); + } + final baseUri = Uri.parse('${ApiConfig.baseUrl}/api/v1/transactions'); - final url = accountId != null - ? baseUri.replace(queryParameters: {'account_id': accountId}) + final url = queryParams.isNotEmpty + ? baseUri.replace(queryParameters: queryParams) : baseUri; try { @@ -96,10 +110,16 @@ class TransactionsService { // Handle both array and object responses List transactionsJson; + Map? pagination; + if (responseData is List) { transactionsJson = responseData; } else if (responseData is Map && responseData.containsKey('transactions')) { transactionsJson = responseData['transactions']; + // Extract pagination metadata if present + if (responseData.containsKey('pagination')) { + pagination = responseData['pagination']; + } } else { transactionsJson = []; } @@ -111,6 +131,7 @@ class TransactionsService { return { 'success': true, 'transactions': transactions, + if (pagination != null) 'pagination': pagination, }; } else if (response.statusCode == 401) { return { From 6e240a23324119062f6d71268b0066fc491eb9db Mon Sep 17 00:00:00 2001 From: LPW Date: Tue, 13 Jan 2026 03:32:05 -0500 Subject: [PATCH 50/54] Add test for dormant credit cards with zero balance and adjust processor logic (#630) - Added a new test to validate how dormant credit cards with zero balance and negative available balance are processed. - Updated processor logic to ensure `current_balance` takes precedence when explicitly set to zero, preventing incorrect usage of `available_balance`. Co-authored-by: Josh Waldrep --- app/models/simplefin_account/processor.rb | 4 +++- .../simplefin_account_processor_test.rb | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/app/models/simplefin_account/processor.rb b/app/models/simplefin_account/processor.rb index 569f515cd..37d211673 100644 --- a/app/models/simplefin_account/processor.rb +++ b/app/models/simplefin_account/processor.rb @@ -46,7 +46,9 @@ class SimplefinAccount::Processor avail = to_decimal(simplefin_account.available_balance) # Choose an observed value prioritizing posted balance first - observed = bal.nonzero? ? bal : avail + # Use available_balance only when current_balance is truly missing (nil), + # not when it's explicitly zero (e.g., dormant credit card with no debt) + observed = simplefin_account.current_balance.nil? ? avail : bal # Determine if this should be treated as a liability for normalization is_linked_liability = [ "CreditCard", "Loan" ].include?(account.accountable_type) diff --git a/test/models/simplefin_account_processor_test.rb b/test/models/simplefin_account_processor_test.rb index 0b13f0e97..11b9b4456 100644 --- a/test/models/simplefin_account_processor_test.rb +++ b/test/models/simplefin_account_processor_test.rb @@ -178,4 +178,24 @@ class SimplefinAccountProcessorTest < ActiveSupport::TestCase # Mapper should infer liability from name; final should be negative assert_equal BigDecimal("-100.00"), acct.reload.balance end + + test "dormant credit card with zero balance and negative available-balance shows zero debt" do + sfin_acct = SimplefinAccount.create!( + simplefin_item: @item, + name: "Discover Card", + account_id: "cc_dormant", + currency: "USD", + account_type: "credit", + current_balance: BigDecimal("0"), + available_balance: BigDecimal("-3800") # credit limit reported as negative + ) + + acct = accounts(:credit_card) + acct.update!(simplefin_account: sfin_acct) + + SimplefinAccount::Processor.new(sfin_acct).send(:process_account!) + + # Should use explicit zero balance, not negative available_balance + assert_equal BigDecimal("0"), acct.reload.balance + end end From 320e087a228fa91fbe7330ffc3f53391c5116e2e Mon Sep 17 00:00:00 2001 From: LPW Date: Tue, 13 Jan 2026 03:37:19 -0500 Subject: [PATCH 51/54] Add support for displaying and managing legacy SSO providers (#628) * feat: add support for displaying and managing legacy SSO providers - Introduced UI section for environment/YAML-configured SSO providers. - Added warnings and guidance on migrating legacy providers to database-backed configuration. - Enhanced localization with new keys for legacy provider management. - Updated form and toggle components for improved usability. * Expand SSO documentation: add SAML 2.0 support, JIT provisioning settings, super-admin setup steps, audit logging, and user administration details. * Update JIT provisioning docs: clarify role mapping behavior and add examples; note new `logout_idp` audit log event. --------- Co-authored-by: Josh Waldrep --- .../admin/sso_providers_controller.rb | 6 ++ app/views/admin/sso_providers/_form.html.erb | 10 ++- app/views/admin/sso_providers/index.html.erb | 38 +++++++++ .../locales/views/admin/sso_providers/en.yml | 4 + docs/hosting/oidc.md | 84 ++++++++++++++++++- 5 files changed, 137 insertions(+), 5 deletions(-) diff --git a/app/controllers/admin/sso_providers_controller.rb b/app/controllers/admin/sso_providers_controller.rb index d47864f43..97bc86fa3 100644 --- a/app/controllers/admin/sso_providers_controller.rb +++ b/app/controllers/admin/sso_providers_controller.rb @@ -7,6 +7,12 @@ module Admin def index authorize SsoProvider @sso_providers = policy_scope(SsoProvider).order(:name) + + # Load runtime providers (from YAML/env) that might not be in the database + # This helps show users that legacy providers are active but not manageable via UI + @runtime_providers = Rails.configuration.x.auth.sso_providers || [] + db_provider_names = @sso_providers.pluck(:name) + @legacy_providers = @runtime_providers.reject { |p| db_provider_names.include?(p[:name].to_s) } end def show diff --git a/app/views/admin/sso_providers/_form.html.erb b/app/views/admin/sso_providers/_form.html.erb index 61b8a6c17..590ba740e 100644 --- a/app/views/admin/sso_providers/_form.html.erb +++ b/app/views/admin/sso_providers/_form.html.erb @@ -55,9 +55,13 @@
- <%= form.check_box :enabled, - label: "Enable this provider", - checked: sso_provider.enabled? %> +
+
+

<%= t("admin.sso_providers.form.enabled_label") %>

+

<%= t("admin.sso_providers.form.enabled_help") %>

+
+ <%= form.toggle :enabled %> +
diff --git a/app/views/admin/sso_providers/index.html.erb b/app/views/admin/sso_providers/index.html.erb index b99272db6..006dc2647 100644 --- a/app/views/admin/sso_providers/index.html.erb +++ b/app/views/admin/sso_providers/index.html.erb @@ -62,6 +62,44 @@
<% end %> + <% if @legacy_providers.any? %> + <%= settings_section title: t("admin.sso_providers.index.legacy_providers_title"), collapsible: true, open: true do %> +
+
+ <%= icon "alert-triangle", class: "w-5 h-5 text-amber-600 shrink-0" %> +

+ <%= t("admin.sso_providers.index.legacy_providers_notice") %> +

+
+
+ +
+ <% @legacy_providers.each do |provider| %> +
+
+ <% provider_icon = provider[:icon].presence || "key" %> + <%= icon provider_icon, class: "w-5 h-5 text-secondary" %> +
+

<%= provider[:label].presence || provider[:name] %>

+

+ <%= provider[:strategy].to_s.titleize %> · <%= provider[:name] %> + <% if provider[:issuer].present? %> + · <%= provider[:issuer] %> + <% end %> +

+
+
+
+ + <%= t("admin.sso_providers.index.env_configured") %> + +
+
+ <% end %> +
+ <% end %> + <% end %> + <%= settings_section title: "Configuration Mode", collapsible: true, open: false do %>
diff --git a/config/locales/views/admin/sso_providers/en.yml b/config/locales/views/admin/sso_providers/en.yml index c59c26380..ff26989aa 100644 --- a/config/locales/views/admin/sso_providers/en.yml +++ b/config/locales/views/admin/sso_providers/en.yml @@ -18,6 +18,9 @@ en: actions: "Actions" enabled: "Enabled" disabled: "Disabled" + legacy_providers_title: "Environment-Configured Providers" + legacy_providers_notice: "These providers are configured via environment variables or YAML and cannot be managed through this interface. To manage them here, migrate them to database-backed providers by enabling AUTH_PROVIDERS_SOURCE=db and recreating them in the UI." + env_configured: "Env/YAML" new: title: "Add SSO Provider" description: "Configure a new single sign-on authentication provider" @@ -51,6 +54,7 @@ en: icon_placeholder: "e.g., key, google, github" icon_help: "Lucide icon name (optional)" enabled_label: "Enable this provider" + enabled_help: "Users can sign in with this provider when enabled" issuer_label: "Issuer" issuer_placeholder: "https://accounts.google.com" issuer_help: "OIDC issuer URL (will validate .well-known/openid-configuration endpoint)" diff --git a/docs/hosting/oidc.md b/docs/hosting/oidc.md index 04be10bed..3aa676b8a 100644 --- a/docs/hosting/oidc.md +++ b/docs/hosting/oidc.md @@ -1,6 +1,6 @@ -# Configuring OpenID Connect and SSO providers +# Configuring OpenID Connect, SAML, and SSO Providers -This guide shows how to enable OpenID Connect (OIDC) and other single sign-on (SSO) providers for Sure using Google, GitHub, or another OIDC‑compatible identity provider (e.g. Keycloak, Authentik). +This guide shows how to enable OpenID Connect (OIDC), SAML 2.0, and other single sign-on (SSO) providers for Sure using Google, GitHub, or another identity provider (e.g. Keycloak, Authentik, Okta, Azure AD). It also documents the new `config/auth.yml` and environment variables that control: @@ -174,6 +174,26 @@ To enable Google: - `http://localhost:3000/auth//callback` +### 3.5 Bootstrapping the first super‑admin + +The first `super_admin` must be set via Rails console. Access the console in your container/pod or directly on the server: + +```bash +bin/rails console +``` + +Then promote a user: + +```ruby +# Set super_admin role +User.find_by(email: "admin@example.com").update!(role: :super_admin) + +# Verify +User.find_by(email: "admin@example.com").role # => "super_admin" +``` + +Once set, super‑admins can promote other users via the web UI at `/admin/users`. + --- ## 4. Example configurations @@ -419,6 +439,20 @@ To switch back to YAML-based configuration: 2. Restart the application 3. Providers will be loaded from `config/auth.yml` +### 6.6 JIT provisioning settings + +Each provider has a **Default Role** field (defaults to `member`) that sets the role for JIT-created users. + +**Role mapping from IdP groups:** + +Expand **"Role Mapping"** in the admin UI to map IdP group names to Sure roles. Enter comma-separated group names for each role: + +- **Super Admin Groups**: `Platform-Admins, IdP-Superusers` +- **Admin Groups**: `Team-Leads, Managers` +- **Member Groups**: `Everyone` or leave blank + +Mapping is case-sensitive and matches exact group claim values from the IdP. When a user belongs to multiple mapped groups, the highest role wins (`super_admin` > `admin` > `member`). If no groups match, the Default Role is used. + --- ## 7. Troubleshooting @@ -484,4 +518,50 @@ Each provider requires a callback URL configured in your identity provider: --- +## 9. SAML 2.0 Support + +Sure supports SAML 2.0 via database-backed providers. Select **"SAML 2.0"** as the strategy when adding a provider at `/admin/sso_providers`. + +Configure with either: +- **IdP Metadata URL** (recommended) - auto-fetches configuration +- **Manual config** - IdP SSO URL + certificate + +In your IdP, set: +- **ACS URL**: `https://yourdomain.com/auth//callback` +- **Entity ID**: `https://yourdomain.com` (your `APP_URL`) +- **Name ID**: Email Address + +--- + +## 10. User Administration + +Super‑admins can manage user roles at `/admin/users`. + +Roles: `member` (standard), `admin` (family admin), `super_admin` (platform admin). + +Note: Super‑admins cannot change their own role. + +--- + +## 11. Audit Logging + +SSO events are logged to `sso_audit_logs`: `login`, `login_failed`, `logout`, `logout_idp` (federated logout), `link`, `unlink`, `jit_account_created`. + +Query via console: + +```ruby +SsoAuditLog.by_event("login").recent.limit(50) +SsoAuditLog.by_event("login_failed").where("created_at > ?", 24.hours.ago) +``` + +--- + +## 12. User SSO Identity Management + +Users manage linked SSO identities at **Settings > Security**. + +SSO-only users (no password) cannot unlink their last identity. + +--- + For additional help, see the main [hosting documentation](../README.md) or open an issue on GitHub. From 7c3af7d85e542cbea1669964e27b220a4d9252f2 Mon Sep 17 00:00:00 2001 From: LPW Date: Tue, 13 Jan 2026 03:39:26 -0500 Subject: [PATCH 52/54] refactor: streamline SimpleFIN connection updates for improved efficiency (#631) - Introduced `update_access_url!` method to reuse existing SimpleFIN items during reconnections, preserving account linkages. - Refactored `SimplefinConnectionUpdateJob` to update access URLs in place without creating new items or transferring accounts. - Adjusted sync logic to leverage `repair_stale_linkages` for seamless reconnections. - Enhanced `SimplefinItem::Importer` to auto-recover the `good` status if no auth errors are found during sync. - Updated tests to validate in-place updates and preserved account relationships. Co-authored-by: Josh Waldrep --- app/jobs/simplefin_connection_update_job.rb | 166 ++---------------- app/models/simplefin_item.rb | 14 ++ app/models/simplefin_item/importer.rb | 20 +++ .../simplefin_items_controller_test.rb | 115 +++++------- 4 files changed, 94 insertions(+), 221 deletions(-) diff --git a/app/jobs/simplefin_connection_update_job.rb b/app/jobs/simplefin_connection_update_job.rb index 850eaac8b..db4593762 100644 --- a/app/jobs/simplefin_connection_update_job.rb +++ b/app/jobs/simplefin_connection_update_job.rb @@ -2,166 +2,30 @@ class SimplefinConnectionUpdateJob < ApplicationJob queue_as :high_priority # Disable automatic retries for this job since the setup token is single-use. - # If the token claim succeeds but import fails, retrying would fail at claim. + # If the token claim succeeds but sync fails, retrying would fail at claim. discard_on Provider::Simplefin::SimplefinError do |job, error| Rails.logger.error( "SimplefinConnectionUpdateJob discarded: #{error.class} - #{error.message} " \ - "(family_id=#{job.arguments.first[:family_id]}, old_item_id=#{job.arguments.first[:old_simplefin_item_id]})" + "(family_id=#{job.arguments.first[:family_id]}, item_id=#{job.arguments.first[:old_simplefin_item_id]})" ) end def perform(family_id:, old_simplefin_item_id:, setup_token:) family = Family.find(family_id) - old_item = family.simplefin_items.find(old_simplefin_item_id) + simplefin_item = family.simplefin_items.find(old_simplefin_item_id) - # Step 1: Claim the token and create the new item. - # This is the critical step - if it fails, we can safely retry. - # If it succeeds, the token is consumed and we must not retry the claim. - updated_item = family.create_simplefin_item!( - setup_token: setup_token, - item_name: old_item.name + # Step 1: Claim the new token and update the existing item's access_url. + # This preserves all existing account linkages - no need to transfer anything. + simplefin_item.update_access_url!(setup_token: setup_token) + + # Step 2: Sync the item to import fresh data. + # The existing repair_stale_linkages logic handles cases where SimpleFIN + # account IDs changed (e.g., user re-added institution in SimpleFIN Bridge). + simplefin_item.sync_later + + Rails.logger.info( + "SimplefinConnectionUpdateJob: Successfully updated SimplefinItem #{simplefin_item.id} " \ + "with new access_url for family #{family_id}" ) - - # Step 2: Import accounts from SimpleFin. - # If this fails, we have an orphaned item but the token is already consumed. - # We handle this gracefully by marking the item and continuing. - begin - updated_item.import_latest_simplefin_data - rescue => e - Rails.logger.error( - "SimplefinConnectionUpdateJob: import failed for new item #{updated_item.id}: " \ - "#{e.class} - #{e.message}. Item created but may need manual sync." - ) - # Mark the item as needing attention but don't fail the job entirely. - # The item exists and can be synced manually later. - updated_item.update!(status: :requires_update) - # Still proceed to transfer accounts and schedule old item deletion - end - - # Step 3: Transfer account links from old to new item. - # This is idempotent and safe to retry. - # Check for linked accounts via BOTH legacy FK and AccountProvider. - ActiveRecord::Base.transaction do - old_item.simplefin_accounts.includes(:account, account_provider: :account).each do |old_account| - # Get the linked account via either system - linked_account = old_account.current_account - next unless linked_account.present? - - new_simplefin_account = find_matching_simplefin_account(old_account, updated_item.simplefin_accounts) - next unless new_simplefin_account - - # Update legacy FK - linked_account.update!(simplefin_account_id: new_simplefin_account.id) - - # Also migrate AccountProvider if it exists - if old_account.account_provider.present? - old_account.account_provider.update!( - provider_type: "SimplefinAccount", - provider_id: new_simplefin_account.id - ) - else - # Create AccountProvider for consistency - new_simplefin_account.ensure_account_provider! - end - end - end - - # Schedule deletion outside transaction to avoid race condition where - # the job is enqueued even if the transaction rolls back - old_item.destroy_later - - # Only mark as good if import succeeded (status wasn't set to requires_update above) - updated_item.update!(status: :good) unless updated_item.requires_update? end - - private - # Find a matching SimpleFin account in the new item's accounts. - # Uses a multi-tier matching strategy: - # 1. Exact account_id match (preferred) - # 2. Fingerprint match (name + institution + account_type) - # 3. Fuzzy name match with same institution (fallback) - def find_matching_simplefin_account(old_account, new_accounts) - exact_match = new_accounts.find_by(account_id: old_account.account_id) - return exact_match if exact_match - - old_fingerprint = account_fingerprint(old_account) - fingerprint_match = new_accounts.find { |new_account| account_fingerprint(new_account) == old_fingerprint } - return fingerprint_match if fingerprint_match - - old_institution = extract_institution_id(old_account) - old_name_normalized = normalize_account_name(old_account.name) - - new_accounts.find do |new_account| - new_institution = extract_institution_id(new_account) - new_name_normalized = normalize_account_name(new_account.name) - - next false unless old_institution.present? && old_institution == new_institution - - names_similar?(old_name_normalized, new_name_normalized) - end - end - - def account_fingerprint(simplefin_account) - institution_id = extract_institution_id(simplefin_account) - name_normalized = normalize_account_name(simplefin_account.name) - account_type = simplefin_account.account_type.to_s.downcase - - "#{institution_id}:#{name_normalized}:#{account_type}" - end - - def extract_institution_id(simplefin_account) - org_data = simplefin_account.org_data - return nil unless org_data.is_a?(Hash) - - org_data["id"] || org_data["domain"] || org_data["name"]&.downcase&.gsub(/\s+/, "_") - end - - def normalize_account_name(name) - return "" if name.blank? - - name.to_s - .downcase - .gsub(/[^a-z0-9]/, "") - end - - def names_similar?(name1, name2) - return false if name1.blank? || name2.blank? - - return true if name1 == name2 - return true if name1.include?(name2) || name2.include?(name1) - - longer = [ name1.length, name2.length ].max - return false if longer == 0 - - # Use Levenshtein distance for more accurate similarity - distance = levenshtein_distance(name1, name2) - similarity = 1.0 - (distance.to_f / longer) - similarity >= 0.8 - end - - # Compute Levenshtein edit distance between two strings - def levenshtein_distance(s1, s2) - m, n = s1.length, s2.length - return n if m.zero? - return m if n.zero? - - # Use a single array and update in place for memory efficiency - prev_row = (0..n).to_a - curr_row = [] - - (1..m).each do |i| - curr_row[0] = i - (1..n).each do |j| - cost = s1[i - 1] == s2[j - 1] ? 0 : 1 - curr_row[j] = [ - prev_row[j] + 1, # deletion - curr_row[j - 1] + 1, # insertion - prev_row[j - 1] + cost # substitution - ].min - end - prev_row, curr_row = curr_row, prev_row - end - - prev_row[n] - end end diff --git a/app/models/simplefin_item.rb b/app/models/simplefin_item.rb index baab600b5..daabf99b3 100644 --- a/app/models/simplefin_item.rb +++ b/app/models/simplefin_item.rb @@ -55,6 +55,20 @@ class SimplefinItem < ApplicationRecord SimplefinItem::Importer.new(self, simplefin_provider: simplefin_provider, sync: sync).import end + # Update the access_url by claiming a new setup token. + # This is used when reconnecting an existing SimpleFIN connection. + # Unlike create_simplefin_item!, this updates in-place, preserving all account linkages. + def update_access_url!(setup_token:) + new_access_url = simplefin_provider.claim_access_url(setup_token) + + update!( + access_url: new_access_url, + status: :good + ) + + self + end + def process_accounts # Process accounts linked via BOTH legacy FK and AccountProvider # Use direct query to ensure fresh data from DB, bypassing any association cache diff --git a/app/models/simplefin_item/importer.rb b/app/models/simplefin_item/importer.rb index 8d43993f8..c465bf951 100644 --- a/app/models/simplefin_item/importer.rb +++ b/app/models/simplefin_item/importer.rb @@ -45,6 +45,11 @@ class SimplefinItem::Importer Rails.logger.info "SimplefinItem::Importer - Using REGULAR SYNC (last_synced_at=#{simplefin_item.last_synced_at&.strftime('%Y-%m-%d %H:%M')})" import_regular_sync end + + # Reset status to good if no auth errors occurred in this sync. + # This allows the item to recover automatically when a bank's auth issue is resolved + # in SimpleFIN Bridge, without requiring the user to manually reconnect. + maybe_clear_requires_update_status rescue RateLimitedError => e stats["rate_limited"] = true stats["rate_limited_at"] = Time.current.iso8601 @@ -321,6 +326,21 @@ class SimplefinItem::Importer sync.update_columns(sync_stats: merged) # avoid callbacks/validations during tight loops end + # Reset status to good if no auth errors occurred in this sync. + # This allows automatic recovery when a bank's auth issue is resolved in SimpleFIN Bridge. + def maybe_clear_requires_update_status + return unless simplefin_item.requires_update? + + auth_errors = stats.dig("error_buckets", "auth").to_i + if auth_errors.zero? + simplefin_item.update!(status: :good) + Rails.logger.info( + "SimpleFIN: cleared requires_update status for item ##{simplefin_item.id} " \ + "(no auth errors in this sync)" + ) + end + end + def import_with_chunked_history # SimpleFin's actual limit is 60 days (not 365 as documented) # Use 60-day chunks to stay within limits diff --git a/test/controllers/simplefin_items_controller_test.rb b/test/controllers/simplefin_items_controller_test.rb index f702a0006..c7e039583 100644 --- a/test/controllers/simplefin_items_controller_test.rb +++ b/test/controllers/simplefin_items_controller_test.rb @@ -182,20 +182,21 @@ class SimplefinItemsControllerTest < ActionDispatch::IntegrationTest assert_includes response.body, I18n.t("simplefin_items.update.errors.blank_token", default: "Please enter a SimpleFIN setup token") end - test "should transfer accounts when updating simplefin item token" do + test "should update simplefin item access_url in place preserving account linkages" do @simplefin_item.update!(status: :requires_update) + original_item_id = @simplefin_item.id token = Base64.strict_encode64("https://example.com/claim") - # Create old SimpleFIN accounts linked to Maybe accounts - old_simplefin_account1 = @simplefin_item.simplefin_accounts.create!( + # Create SimpleFIN accounts linked to Maybe accounts + simplefin_account1 = @simplefin_item.simplefin_accounts.create!( name: "Test Checking", account_id: "sf_account_123", currency: "USD", current_balance: 1000, account_type: "depository" ) - old_simplefin_account2 = @simplefin_item.simplefin_accounts.create!( + simplefin_account2 = @simplefin_item.simplefin_accounts.create!( name: "Test Savings", account_id: "sf_account_456", currency: "USD", @@ -211,7 +212,7 @@ class SimplefinItemsControllerTest < ActionDispatch::IntegrationTest currency: "USD", accountable_type: "Depository", accountable: Depository.create!(subtype: "checking"), - simplefin_account_id: old_simplefin_account1.id + simplefin_account_id: simplefin_account1.id ) maybe_account2 = Account.create!( family: @family, @@ -220,40 +221,19 @@ class SimplefinItemsControllerTest < ActionDispatch::IntegrationTest currency: "USD", accountable_type: "Depository", accountable: Depository.create!(subtype: "savings"), - simplefin_account_id: old_simplefin_account2.id + simplefin_account_id: simplefin_account2.id ) - # Update old SimpleFIN accounts to reference the Maybe accounts - old_simplefin_account1.update!(account: maybe_account1) - old_simplefin_account2.update!(account: maybe_account2) + # Update SimpleFIN accounts to reference the Maybe accounts + simplefin_account1.update!(account: maybe_account1) + simplefin_account2.update!(account: maybe_account2) - # Mock only the external API calls, let business logic run + # Mock only the external API calls mock_provider = mock() mock_provider.expects(:claim_access_url).with(token).returns("https://example.com/new_access") - mock_provider.expects(:get_accounts).returns({ - accounts: [ - { - id: "sf_account_123", - name: "Test Checking", - type: "depository", - currency: "USD", - balance: 1000, - transactions: [] - }, - { - id: "sf_account_456", - name: "Test Savings", - type: "depository", - currency: "USD", - balance: 5000, - transactions: [] - } - ] - }).at_least_once Provider::Simplefin.expects(:new).returns(mock_provider).at_least_once - # Perform the update (async job), but execute enqueued jobs inline so we can - # assert the link transfers. + # Perform the update - job updates access_url and enqueues sync perform_enqueued_jobs(only: SimplefinConnectionUpdateJob) do patch simplefin_item_url(@simplefin_item), params: { simplefin_item: { setup_token: token } @@ -263,40 +243,33 @@ class SimplefinItemsControllerTest < ActionDispatch::IntegrationTest assert_redirected_to accounts_path assert_equal "SimpleFIN connection updated.", flash[:notice] - # Verify accounts were transferred to new SimpleFIN accounts - assert Account.exists?(maybe_account1.id), "maybe_account1 should still exist" - assert Account.exists?(maybe_account2.id), "maybe_account2 should still exist" + # Verify the same SimpleFIN item was updated (not a new one created) + @simplefin_item.reload + assert_equal original_item_id, @simplefin_item.id + assert_equal "https://example.com/new_access", @simplefin_item.access_url + assert_equal "good", @simplefin_item.status + # Verify no duplicate SimpleFIN items were created + assert_equal 1, @family.simplefin_items.count + + # Verify account linkages remain intact maybe_account1.reload maybe_account2.reload + assert_equal simplefin_account1.id, maybe_account1.simplefin_account_id + assert_equal simplefin_account2.id, maybe_account2.simplefin_account_id - # Find the new SimpleFIN item that was created - new_simplefin_item = @family.simplefin_items.where.not(id: @simplefin_item.id).first - assert_not_nil new_simplefin_item, "New SimpleFIN item should have been created" - - new_sf_account1 = new_simplefin_item.simplefin_accounts.find_by(account_id: "sf_account_123") - new_sf_account2 = new_simplefin_item.simplefin_accounts.find_by(account_id: "sf_account_456") - - assert_not_nil new_sf_account1, "New SimpleFIN account with ID sf_account_123 should exist" - assert_not_nil new_sf_account2, "New SimpleFIN account with ID sf_account_456 should exist" - - assert_equal new_sf_account1.id, maybe_account1.simplefin_account_id - assert_equal new_sf_account2.id, maybe_account2.simplefin_account_id - - # The old item will be deleted asynchronously; until then, legacy links should be moved. - - # Verify old SimpleFIN item is scheduled for deletion - @simplefin_item.reload - assert @simplefin_item.scheduled_for_deletion? + # Verify item is NOT scheduled for deletion (we updated it, not replaced it) + assert_not @simplefin_item.scheduled_for_deletion? end - test "should handle partial account matching during token update" do + test "should preserve account linkages when reconnecting even if accounts change" do @simplefin_item.update!(status: :requires_update) + original_item_id = @simplefin_item.id token = Base64.strict_encode64("https://example.com/claim") - # Create old SimpleFIN account - old_simplefin_account = @simplefin_item.simplefin_accounts.create!( + # Create SimpleFIN account linked to Maybe account + simplefin_account = @simplefin_item.simplefin_accounts.create!( name: "Test Checking", account_id: "sf_account_123", currency: "USD", @@ -312,18 +285,16 @@ class SimplefinItemsControllerTest < ActionDispatch::IntegrationTest currency: "USD", accountable_type: "Depository", accountable: Depository.create!(subtype: "checking"), - simplefin_account_id: old_simplefin_account.id + simplefin_account_id: simplefin_account.id ) - old_simplefin_account.update!(account: maybe_account) + simplefin_account.update!(account: maybe_account) - # Mock only the external API calls, let business logic run + # Mock only the external API calls mock_provider = mock() mock_provider.expects(:claim_access_url).with(token).returns("https://example.com/new_access") - # Return empty accounts list to simulate account was removed from bank - mock_provider.expects(:get_accounts).returns({ accounts: [] }).at_least_once Provider::Simplefin.expects(:new).returns(mock_provider).at_least_once - # Perform update + # Perform update - job updates access_url and enqueues sync perform_enqueued_jobs(only: SimplefinConnectionUpdateJob) do patch simplefin_item_url(@simplefin_item), params: { simplefin_item: { setup_token: token } @@ -332,15 +303,19 @@ class SimplefinItemsControllerTest < ActionDispatch::IntegrationTest assert_redirected_to accounts_path - # Verify Maybe account still linked to old SimpleFIN account (no transfer occurred) - maybe_account.reload - old_simplefin_account.reload - assert_equal old_simplefin_account.id, maybe_account.simplefin_account_id - assert_equal maybe_account, old_simplefin_account.current_account - - # Old item still scheduled for deletion + # Verify item was updated in place @simplefin_item.reload - assert @simplefin_item.scheduled_for_deletion? + assert_equal original_item_id, @simplefin_item.id + assert_equal "https://example.com/new_access", @simplefin_item.access_url + + # Verify account linkage remains intact (linkage preserved regardless of sync results) + maybe_account.reload + simplefin_account.reload + assert_equal simplefin_account.id, maybe_account.simplefin_account_id + assert_equal maybe_account, simplefin_account.current_account + + # Item is NOT scheduled for deletion (we updated it, not replaced it) + assert_not @simplefin_item.scheduled_for_deletion? end test "select_existing_account renders empty-state modal when no simplefin accounts exist" do From 39ba65df77381b3f522c08cd656cee45c8001b50 Mon Sep 17 00:00:00 2001 From: Jose <39016041+jospaquim@users.noreply.github.com> Date: Tue, 13 Jan 2026 04:10:15 -0500 Subject: [PATCH 53/54] feat: Add Merchants and Tags API v1 Endpoints (#620) * Add files via upload Signed-off-by: Jose <39016041+jospaquim@users.noreply.github.com> * Add merchants and tags resources to routes Signed-off-by: Jose <39016041+jospaquim@users.noreply.github.com> * update * update spaces * fix: Apply CodeRabbit suggestions and add YARD documentation * docs: Add API documentation for merchants and tags endpoints * fix: Address CodeRabbit feedback on documentation --------- Signed-off-by: Jose <39016041+jospaquim@users.noreply.github.com> --- .../api/v1/merchants_controller.rb | 85 +++++++++ app/controllers/api/v1/tags_controller.rb | 130 ++++++++++++++ config/routes.rb | 3 + docs/api/merchants.md | 117 +++++++++++++ docs/api/tags.md | 162 ++++++++++++++++++ 5 files changed, 497 insertions(+) create mode 100644 app/controllers/api/v1/merchants_controller.rb create mode 100644 app/controllers/api/v1/tags_controller.rb create mode 100644 docs/api/merchants.md create mode 100644 docs/api/tags.md diff --git a/app/controllers/api/v1/merchants_controller.rb b/app/controllers/api/v1/merchants_controller.rb new file mode 100644 index 000000000..53df0ac35 --- /dev/null +++ b/app/controllers/api/v1/merchants_controller.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Api + module V1 + # API v1 endpoint for merchants + # Provides read-only access to family and provider merchants + # + # @example List all merchants + # GET /api/v1/merchants + # + # @example Get a specific merchant + # GET /api/v1/merchants/:id + # + class MerchantsController < BaseController + before_action :ensure_read_scope + + # List all merchants available to the family + # + # Returns both family-owned merchants and provider merchants + # that are assigned to the family's transactions. + # + # @return [Array] JSON array of merchant objects + def index + family = current_resource_owner.family + + # Single query with OR conditions - more efficient than Ruby deduplication + family_merchant_ids = family.merchants.select(:id) + provider_merchant_ids = family.transactions.select(:merchant_id) + + @merchants = Merchant + .where(id: family_merchant_ids) + .or(Merchant.where(id: provider_merchant_ids, type: "ProviderMerchant")) + .distinct + .alphabetically + + render json: @merchants.map { |m| merchant_json(m) } + rescue StandardError => e + Rails.logger.error("API Merchants Error: #{e.message}") + render json: { error: "Failed to fetch merchants" }, status: :internal_server_error + end + + # Get a specific merchant by ID + # + # Returns a merchant if it belongs to the family or is assigned + # to any of the family's transactions. + # + # @param id [String] The merchant ID + # @return [Hash] JSON merchant object or error + def show + family = current_resource_owner.family + + @merchant = family.merchants.find_by(id: params[:id]) || + Merchant.joins(:transactions) + .where(transactions: { account_id: family.accounts.select(:id) }) + .distinct + .find_by(id: params[:id]) + + if @merchant + render json: merchant_json(@merchant) + else + render json: { error: "Merchant not found" }, status: :not_found + end + rescue StandardError => e + Rails.logger.error("API Merchant Show Error: #{e.message}") + render json: { error: "Failed to fetch merchant" }, status: :internal_server_error + end + + private + + # Serialize a merchant to JSON format + # + # @param merchant [Merchant] The merchant to serialize + # @return [Hash] JSON-serializable hash + def merchant_json(merchant) + { + id: merchant.id, + name: merchant.name, + type: merchant.type, + created_at: merchant.created_at, + updated_at: merchant.updated_at + } + end + end + end +end diff --git a/app/controllers/api/v1/tags_controller.rb b/app/controllers/api/v1/tags_controller.rb new file mode 100644 index 000000000..287642930 --- /dev/null +++ b/app/controllers/api/v1/tags_controller.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module Api + module V1 + # API v1 endpoint for tags + # Provides full CRUD operations for family tags + # + # @example List all tags + # GET /api/v1/tags + # + # @example Create a new tag + # POST /api/v1/tags + # { "tag": { "name": "WhiteHouse", "color": "#3b82f6" } } + # + class TagsController < BaseController + before_action :ensure_read_scope, only: %i[index show] + before_action :ensure_write_scope, only: %i[create update destroy] + before_action :set_tag, only: %i[show update destroy] + + # List all tags belonging to the family + # + # @return [Array] JSON array of tag objects sorted alphabetically + def index + family = current_resource_owner.family + @tags = family.tags.alphabetically + + render json: @tags.map { |t| tag_json(t) } + rescue StandardError => e + Rails.logger.error("API Tags Error: #{e.message}") + render json: { error: "Failed to fetch tags" }, status: :internal_server_error + end + + # Get a specific tag by ID + # + # @param id [String] The tag ID + # @return [Hash] JSON tag object + def show + render json: tag_json(@tag) + rescue StandardError => e + Rails.logger.error("API Tag Show Error: #{e.message}") + render json: { error: "Failed to fetch tag" }, status: :internal_server_error + end + + # Create a new tag for the family + # + # @param name [String] Tag name (required) + # @param color [String] Hex color code (optional, auto-assigned if not provided) + # @return [Hash] JSON tag object with status 201 + def create + family = current_resource_owner.family + @tag = family.tags.new(tag_params) + + # Assign random color if not provided + @tag.color ||= Tag::COLORS.sample + + if @tag.save + render json: tag_json(@tag), status: :created + else + render json: { error: @tag.errors.full_messages.join(", ") }, status: :unprocessable_entity + end + rescue StandardError => e + Rails.logger.error("API Tag Create Error: #{e.message}") + render json: { error: "Failed to create tag" }, status: :internal_server_error + end + + # Update an existing tag + # + # @param id [String] The tag ID + # @param name [String] New tag name (optional) + # @param color [String] New hex color code (optional) + # @return [Hash] JSON tag object + def update + if @tag.update(tag_params) + render json: tag_json(@tag) + else + render json: { error: @tag.errors.full_messages.join(", ") }, status: :unprocessable_entity + end + rescue StandardError => e + Rails.logger.error("API Tag Update Error: #{e.message}") + render json: { error: "Failed to update tag" }, status: :internal_server_error + end + + # Delete a tag + # + # @param id [String] The tag ID + # @return [nil] Empty response with status 204 + def destroy + @tag.destroy! + head :no_content + rescue StandardError => e + Rails.logger.error("API Tag Destroy Error: #{e.message}") + render json: { error: "Failed to delete tag" }, status: :internal_server_error + end + + private + + # Find and set the tag from params + # + # @raise [ActiveRecord::RecordNotFound] if tag not found + # @return [Tag] The found tag + def set_tag + family = current_resource_owner.family + @tag = family.tags.find(params[:id]) + rescue ActiveRecord::RecordNotFound + render json: { error: "Tag not found" }, status: :not_found + end + + # Strong parameters for tag creation/update + # + # @return [ActionController::Parameters] Permitted parameters + def tag_params + params.require(:tag).permit(:name, :color) + end + + # Serialize a tag to JSON format + # + # @param tag [Tag] The tag to serialize + # @return [Hash] JSON-serializable hash + def tag_json(tag) + { + id: tag.id, + name: tag.name, + color: tag.color, + created_at: tag.created_at, + updated_at: tag.updated_at + } + end + end + end +end diff --git a/config/routes.rb b/config/routes.rb index 5c06caf5b..07a70144e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -296,6 +296,9 @@ Rails.application.routes.draw do # Production API endpoints resources :accounts, only: [ :index, :show ] resources :categories, only: [ :index, :show ] + resources :merchants, only: %i[index show] + resources :tags, only: %i[index show create update destroy] + resources :transactions, only: [ :index, :show, :create, :update, :destroy ] resources :imports, only: [ :index, :show, :create ] resource :usage, only: [ :show ], controller: :usage diff --git a/docs/api/merchants.md b/docs/api/merchants.md new file mode 100644 index 000000000..5e00b3dda --- /dev/null +++ b/docs/api/merchants.md @@ -0,0 +1,117 @@ +# Merchants API + +The Merchants API allows external applications to retrieve merchants within Sure. Merchants represent payees or vendors associated with transactions. + +## Generated OpenAPI specification + +- The source of truth for the documentation lives in [`spec/requests/api/v1/merchants_spec.rb`](../../spec/requests/api/v1/merchants_spec.rb). These specs authenticate against the Rails stack, exercise every merchant endpoint, and capture real response shapes. +- Regenerate the OpenAPI document with: + + ```sh + SWAGGER_DRY_RUN=0 bundle exec rspec spec/requests --format Rswag::Specs::SwaggerFormatter + ``` + + The task compiles the request specs and writes the result to [`docs/api/openapi.yaml`](openapi.yaml). + +- Run just the documentation specs with: + + ```sh + bundle exec rspec spec/requests/api/v1/merchants_spec.rb + ``` + +## Authentication requirements + +All merchant endpoints require an OAuth2 access token or API key that grants the `read` scope. + +## Available endpoints + +| Endpoint | Scope | Description | +| --- | --- | --- | +| `GET /api/v1/merchants` | `read` | List all merchants available to the family. | +| `GET /api/v1/merchants/{id}` | `read` | Retrieve a single merchant by ID. | + +Refer to the generated [`openapi.yaml`](openapi.yaml) for request/response schemas, reusable components, and security definitions. + +## Merchant types + +Sure supports two types of merchants: + +| Type | Description | +| --- | --- | +| `FamilyMerchant` | Merchants created and owned by the family. | +| `ProviderMerchant` | Merchants from external providers (e.g., Plaid) assigned to transactions. | + +The `GET /api/v1/merchants` endpoint returns both types: all family merchants plus any provider merchants that are assigned to the family's transactions. + +## Merchant object + +A merchant response includes: + +```json +{ + "id": "uuid", + "name": "Whole Foods", + "type": "FamilyMerchant", + "created_at": "2024-01-15T10:30:00Z", + "updated_at": "2024-01-15T10:30:00Z" +} +``` + +## Listing merchants + +Example request: + +```http +GET /api/v1/merchants +Authorization: Bearer +``` + +Example response: + +```json +[ + { + "id": "550e8400-e29b-41d4-a716-446655440001", + "name": "Amazon", + "type": "FamilyMerchant", + "created_at": "2024-01-10T08:00:00Z", + "updated_at": "2024-01-10T08:00:00Z" + }, + { + "id": "550e8400-e29b-41d4-a716-446655440002", + "name": "Starbucks", + "type": "ProviderMerchant", + "created_at": "2024-01-12T14:30:00Z", + "updated_at": "2024-01-12T14:30:00Z" + } +] +``` + +## Using merchants with transactions + +When creating or updating transactions, you can assign a merchant using the `merchant_id` field: + +```json +{ + "transaction": { + "account_id": "uuid", + "date": "2024-01-15", + "amount": 75.50, + "name": "Coffee", + "nature": "expense", + "merchant_id": "550e8400-e29b-41d4-a716-446655440002" + } +} +``` + +## Error responses + +Errors conform to the shared `ErrorResponse` schema in the OpenAPI document: + +```json +{ + "error": "Human readable error message" +} +``` + +Common error codes include `unauthorized`, `not_found`, and `internal_server_error`. diff --git a/docs/api/tags.md b/docs/api/tags.md new file mode 100644 index 000000000..ab040c267 --- /dev/null +++ b/docs/api/tags.md @@ -0,0 +1,162 @@ +# Tags API + +The Tags API allows external applications to manage tags within Sure. Tags provide a flexible way to categorize and label transactions beyond the standard category system. + +## Generated OpenAPI specification + +- The source of truth for the documentation lives in [`spec/requests/api/v1/tags_spec.rb`](../../spec/requests/api/v1/tags_spec.rb). These specs authenticate against the Rails stack, exercise every tag endpoint, and capture real response shapes. +- Regenerate the OpenAPI document with: + + ```sh + SWAGGER_DRY_RUN=0 bundle exec rspec spec/requests --format Rswag::Specs::SwaggerFormatter + ``` + + The task compiles the request specs and writes the result to [`docs/api/openapi.yaml`](openapi.yaml). + +- Run just the documentation specs with: + + ```sh + bundle exec rspec spec/requests/api/v1/tags_spec.rb + ``` + +## Authentication requirements + +| Operation | Scope Required | +| --- | --- | +| List/View tags | `read` | +| Create/Update/Delete tags | `write` | + +## Available endpoints + +| Endpoint | Scope | Description | +| --- | --- | --- | +| `GET /api/v1/tags` | `read` | List all tags belonging to the family. | +| `GET /api/v1/tags/{id}` | `read` | Retrieve a single tag by ID. | +| `POST /api/v1/tags` | `write` | Create a new tag. | +| `PATCH /api/v1/tags/{id}` | `write` | Update an existing tag. | +| `DELETE /api/v1/tags/{id}` | `write` | Permanently delete a tag. | + +Refer to the generated [`openapi.yaml`](openapi.yaml) for request/response schemas, reusable components, and security definitions. + +## Tag object + +A tag response includes: + +```json +{ + "id": "uuid", + "name": "Essential", + "color": "#3b82f6", + "created_at": "2024-01-15T10:30:00Z", + "updated_at": "2024-01-15T10:30:00Z" +} +``` + +## Available colors + +Sure provides a predefined set of colors for tags. If no color is specified when creating a tag, one will be randomly assigned from this palette: + +```text +#e99537, #4da568, #6471eb, #db5a54, #df4e92, +#c44fe9, #eb5429, #61c9ea, #805dee, #6b7c93 +``` + +## Creating tags + +Example request: + +```http +POST /api/v1/tags +Authorization: Bearer +Content-Type: application/json + +{ + "tag": { + "name": "Business", + "color": "#6471eb" + } +} +``` + +The `color` field is optional. If omitted, a random color from the predefined palette will be assigned. + +Example response (201 Created): + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440001", + "name": "Business", + "color": "#6471eb", + "created_at": "2024-01-15T10:30:00Z", + "updated_at": "2024-01-15T10:30:00Z" +} +``` + +## Updating tags + +Example request: + +```http +PATCH /api/v1/tags/550e8400-e29b-41d4-a716-446655440001 +Authorization: Bearer +Content-Type: application/json + +{ + "tag": { + "name": "Work Expenses", + "color": "#4da568" + } +} +``` + +Both `name` and `color` are optional in update requests. + +## Deleting tags + +Example request: + +```http +DELETE /api/v1/tags/550e8400-e29b-41d4-a716-446655440001 +Authorization: Bearer +``` + +Returns `204 No Content` on success. + +## Using tags with transactions + +Tags can be assigned to transactions using the `tag_ids` array field. A transaction can have multiple tags: + +```json +{ + "transaction": { + "account_id": "uuid", + "date": "2024-01-15", + "amount": 150.00, + "name": "Team lunch", + "nature": "expense", + "tag_ids": [ + "550e8400-e29b-41d4-a716-446655440001", + "550e8400-e29b-41d4-a716-446655440002" + ] + } +} +``` + +## Error responses + +Errors conform to the shared `ErrorResponse` schema in the OpenAPI document: + +```json +{ + "error": "Human readable error message" +} +``` + +Common error codes include: + +| Status | Error | Description | +| --- | --- | --- | +| 401 | `unauthorized` | Invalid or missing access token. | +| 404 | `not_found` | Tag not found or does not belong to the family. | +| 422 | `validation_failed` | Invalid input (e.g., duplicate name, missing required field). | +| 500 | `internal_server_error` | Unexpected server error. | From 425570cd1f5d3fa7289e61bbdf6f0d68f4c5eada Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 13 Jan 2026 09:26:03 +0000 Subject: [PATCH 54/54] Bump version to next alpha after v0.6.7-alpha.8 release --- config/initializers/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/version.rb b/config/initializers/version.rb index 611fca378..7e427c761 100644 --- a/config/initializers/version.rb +++ b/config/initializers/version.rb @@ -14,7 +14,7 @@ module Sure private def semver - "0.6.7-alpha.8" + "0.6.7-alpha.9" end end end