Add SnapTrade brokerage integration with full trade history support (#737)

* Introduce SnapTrade integration with models, migrations, views, and activity processing logic.

* Refactor SnapTrade activities processing: improve activity fetching flow, handle pending states, and update UI elements for enhanced user feedback.

* Update Brakeman ignore file to include intentional redirect for SnapTrade OAuth portal.

* Refactor SnapTrade models, views, and processing logic: add currency extraction helper, improve pending state handling, optimize migration checks, and enhance user feedback in UI.

* Remove encryption for SnapTrade `snaptrade_user_id`, as it is an identifier, not a secret.

* Introduce `SnaptradeConnectionCleanupJob` to asynchronously handle SnapTrade connection cleanup and improve i18n for SnapTrade item status messages.

* Update SnapTrade encryption: make `snaptrade_user_secret` non-deterministic to enhance security.

---------

Signed-off-by: Juan José Mata <juanjo.mata@gmail.com>
Co-authored-by: luckyPipewrench <luckypipewrench@proton.me>
Co-authored-by: Juan José Mata <juanjo.mata@gmail.com>

config/brakeman.ignore

@@ -1,5 +1,28 @@
 {
   "ignored_warnings": [
+    {
+      "warning_type": "Redirect",
+      "warning_code": 18,
+      "fingerprint": "556f2fdd1f091ed50811cb2cce28dd2b987cd0a2eed4d19bea138c8c083a3a5d",
+      "check_name": "Redirect",
+      "message": "Possible unprotected redirect",
+      "file": "app/controllers/snaptrade_items_controller.rb",
+      "line": 125,
+      "link": "https://brakemanscanner.org/docs/warning_types/redirect/",
+      "code": "redirect_to(Current.family.snaptrade_items.find(params[:id]).connection_portal_url(:redirect_url => callback_snaptrade_items_url(:item_id => Current.family.snaptrade_items.find(params[:id]).id)), :allow_other_host => true)",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "SnaptradeItemsController",
+        "method": "connect"
+      },
+      "user_input": "Current.family.snaptrade_items.find(params[:id]).connection_portal_url(:redirect_url => callback_snaptrade_items_url(:item_id => Current.family.snaptrade_items.find(params[:id]).id))",
+      "confidence": "Weak",
+      "cwe_id": [
+        601
+      ],
+      "note": "Intentional redirect to SnapTrade's external OAuth portal for brokerage connection"
+    },
     {
       "warning_type": "Redirect",
       "warning_code": 18,

config/initializers/filter_parameter_logging.rb

@@ -4,5 +4,6 @@
 # Use this to limit dissemination of sensitive information.
 # See the ActiveSupport::ParameterFilter documentation for supported notations and behaviors.
 Rails.application.config.filter_parameters += [
-  :passw, :email, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn, :openai_access_token
+  :passw, :email, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn, :openai_access_token,
+  :client_id, :consumer_key, :snaptrade_user_id, :snaptrade_user_secret
 ]

config/locales/views/components/en.yml

@@ -15,6 +15,7 @@ en:
       imported: "Imported: %{count}"
       updated: "Updated: %{count}"
       skipped: "Skipped: %{count}"
+      fetching: "Fetching from brokerage..."
       protected:
         one: "%{count} entry protected (not overwritten)"
         other: "%{count} entries protected (not overwritten)"
@@ -28,6 +29,11 @@ en:
       title: Holdings
       found: "Found: %{count}"
       processed: "Processed: %{count}"
+    trades:
+      title: Trades
+      imported: "Imported: %{count}"
+      skipped: "Skipped: %{count}"
+      fetching: "Fetching activities from brokerage..."
     health:
       title: Health
       view_error_details: View error details

config/locales/views/snaptrade_items/en.yml

@@ -0,0 +1,143 @@
+en:
+  snaptrade_items:
+    create:
+      success: "Successfully configured SnapTrade."
+    update:
+      success: "Successfully updated SnapTrade configuration."
+    destroy:
+      success: "Scheduled SnapTrade connection for deletion."
+    connect:
+      registration_failed: "Failed to register with SnapTrade: %{message}"
+      portal_error: "Failed to connect to SnapTrade: %{message}"
+    callback:
+      success: "Brokerage connected! Please select which accounts to link."
+      no_item: "SnapTrade configuration not found."
+    complete_account_setup:
+      success:
+        one: "Successfully linked %{count} account."
+        other: "Successfully linked %{count} accounts."
+      no_accounts: "No accounts were selected for linking."
+    preload_accounts:
+      not_configured: "SnapTrade is not configured."
+    select_accounts:
+      not_configured: "SnapTrade is not configured."
+    select_existing_account:
+      not_found: "Account or SnapTrade configuration not found."
+      title: "Link to SnapTrade Account"
+      header: "Link Existing Account"
+      subtitle: "Select a SnapTrade account to link to"
+      no_accounts: "No unlinked SnapTrade accounts available."
+      connect_hint: "You may need to connect a brokerage first."
+      settings_link: "Go to Provider Settings"
+      linking_to: "Linking to account:"
+      balance_label: "Balance:"
+      link_button: "Link"
+      cancel_button: "Cancel"
+    link_existing_account:
+      success: "Successfully linked to SnapTrade account."
+      failed: "Failed to link account: %{message}"
+      not_found: "Account not found."
+    setup_accounts:
+      title: "Set Up SnapTrade Accounts"
+      header: "Set Up Your SnapTrade Accounts"
+      subtitle: "Select which brokerage accounts to link"
+      syncing: "Fetching your accounts..."
+      loading: "Fetching accounts from SnapTrade..."
+      loading_hint: "This page will auto-refresh while loading."
+      refresh: "Refresh"
+      info_title: "SnapTrade Investment Data"
+      info_holdings: "Holdings with current prices and quantities"
+      info_cost_basis: "Cost basis per position (when available)"
+      info_activities: "Trade history with activity labels (Buy, Sell, Dividend, etc.)"
+      info_history: "Up to 3 years of transaction history"
+      free_tier_note: "SnapTrade free tier allows 5 brokerage connections. Check your SnapTrade dashboard for current usage."
+      no_accounts_title: "No Accounts Found"
+      no_accounts_message: "No brokerage accounts were found. This can happen if you cancelled the connection or if your brokerage isn't supported."
+      try_again: "Connect Brokerage"
+      back_to_settings: "Back to Settings"
+      available_accounts: "Available Accounts"
+      balance_label: "Balance:"
+      account_number: "Account:"
+      create_button: "Create Selected Accounts"
+      cancel_button: "Cancel"
+      creating: "Creating Accounts..."
+      done_button: "Done"
+      linked_accounts: "Already Linked"
+      linked_to: "Linked to:"
+    snaptrade_item:
+      accounts_need_setup:
+        one: "%{count} account needs setup"
+        other: "%{count} accounts need setup"
+      deletion_in_progress: "Deletion in progress..."
+      syncing: "Syncing..."
+      requires_update: "Connection needs update"
+      error: "Sync error"
+      status: "Last synced %{timestamp} ago - %{summary}"
+      status_never: "Never synced"
+      reconnect: "Reconnect"
+      connect_brokerage: "Connect Brokerage"
+      add_another_brokerage: "Connect another brokerage"
+      delete: "Delete"
+      setup_needed: "Accounts need setup"
+      setup_description: "Some accounts from SnapTrade need to be linked to Sure accounts."
+      setup_action: "Setup Accounts"
+      no_accounts_title: "No accounts discovered"
+      no_accounts_description: "Connect a brokerage to import your investment accounts."
+
+  providers:
+    snaptrade:
+      name: "SnapTrade"
+      connection_description: "Connect to your brokerage via SnapTrade (25+ brokers supported)"
+      description: "SnapTrade connects to 25+ major brokerages (Fidelity, Vanguard, Schwab, Robinhood, etc.) and provides full trade history with activity labels and cost basis."
+      setup_title: "Setup instructions:"
+      step_1_html: "Create an account at <a href=\"https://dashboard.snaptrade.com\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"text-primary underline\">dashboard.snaptrade.com</a>"
+      step_2: "Copy your Client ID and Consumer Key from the dashboard"
+      step_3: "Enter your credentials below and click Save"
+      step_4: "Go to the Accounts page and use 'Connect another brokerage' to link your investment accounts"
+      free_tier_warning: "Free tier includes 5 brokerage connections. Additional connections require a paid SnapTrade plan."
+      client_id_label: "Client ID"
+      client_id_placeholder: "Enter your SnapTrade Client ID"
+      client_id_update_placeholder: "Enter new Client ID to update"
+      consumer_key_label: "Consumer Key"
+      consumer_key_placeholder: "Enter your SnapTrade Consumer Key"
+      consumer_key_update_placeholder: "Enter new Consumer Key to update"
+      save_button: "Save Configuration"
+      update_button: "Update Configuration"
+      status_connected:
+        one: "%{count} account from SnapTrade"
+        other: "%{count} accounts from SnapTrade"
+      needs_setup:
+        one: "%{count} needs setup"
+        other: "%{count} need setup"
+      status_ready: "Ready to connect brokerages"
+      status_needs_registration: "Credentials saved. Go to Accounts page to connect brokerages."
+      status_not_configured: "Not configured"
+      setup_accounts_button: "Setup Accounts"
+      connect_button: "Connect Brokerage"
+      connected_brokerages: "Connected:"
+
+  snaptrade_item:
+    sync_status:
+      no_accounts: "No accounts found"
+      synced:
+        one: "%{count} account synced"
+        other: "%{count} accounts synced"
+      synced_with_setup: "%{linked} synced, %{unlinked} need setup"
+    institution_summary:
+      none: "No institutions connected"
+      count:
+        one: "%{count} institution"
+        other: "%{count} institutions"
+    brokerage_summary:
+      none: "No brokerages connected"
+      count:
+        one: "%{count} brokerage"
+        other: "%{count} brokerages"
+    syncer:
+      discovering: "Discovering accounts..."
+      importing: "Importing accounts from SnapTrade..."
+      processing: "Processing holdings and activities..."
+      calculating: "Calculating balances..."
+      checking_config: "Checking account configuration..."
+      needs_setup: "%{count} accounts need setup..."
+      activities_fetching_async: "Activities are being fetched in the background. This may take up to a minute for fresh brokerage connections."

config/routes.rb

@@ -34,6 +34,24 @@ Rails.application.routes.draw do
     end
   end
 
+  resources :snaptrade_items, only: [ :index, :new, :create, :show, :edit, :update, :destroy ] do
+    collection do
+      get :preload_accounts
+      get :select_accounts
+      post :link_accounts
+      get :select_existing_account
+      post :link_existing_account
+      get :callback
+    end
+
+    member do
+      post :sync
+      get :connect
+      get :setup_accounts
+      post :complete_account_setup
+    end
+  end
+
   # CoinStats routes
   resources :coinstats_items, only: [ :index, :new, :create, :update, :destroy ] do
     collection do