namespace :data_migration do desc "Migrate EU Plaid webhooks" # 2025-02-07: EU Plaid items need to be moved over to a new webhook URL so that we can # instantiate the correct Plaid client for verification based on which Plaid instance it comes from task eu_plaid_webhooks: :environment do Provider::PlaidEuAdapter.ensure_configuration_loaded provider = Provider::Plaid.new(Rails.application.config.plaid_eu, region: :eu) eu_items = PlaidItem.where(plaid_region: "eu") eu_items.find_each do |item| request = Plaid::ItemWebhookUpdateRequest.new( access_token: item.access_token, webhook: "https://app.sure.am/webhooks/plaid_eu" ) provider.client.item_webhook_update(request) puts "Updated webhook for Plaid item #{item.plaid_id}" rescue => error puts "Error updating webhook for Plaid item #{item.plaid_id}: #{error.message}" end end desc "Migrate duplicate securities" # 2025-05-22: older data allowed multiple rows with the same # ticker / exchange_operating_mic (case-insensitive, NULLs collapsed). # This task: # 1. Finds each duplicate group # 2. Chooses the earliest-created row as the keeper # 3. Re-points holdings and trades to the keeper # 4. Destroys the duplicate (which also removes its prices) task migrate_duplicate_securities: :environment do puts "==> Scanning for duplicate securities…" duplicate_sets = Security .select("UPPER(ticker) AS up_ticker, COALESCE(UPPER(exchange_operating_mic), '') AS up_mic, COUNT(*) AS dup_count") .group("up_ticker, up_mic") .having("COUNT(*) > 1") .to_a puts "Found #{duplicate_sets.size} duplicate groups." duplicate_sets.each_with_index do |set, idx| # Fetch duplicates ordered by creation; the first row becomes keeper duplicates_scope = Security .where("UPPER(ticker) = ? AND COALESCE(UPPER(exchange_operating_mic), '') = ?", set.up_ticker, set.up_mic) .order(:created_at) keeper = duplicates_scope.first next unless keeper duplicates = duplicates_scope.offset(1) dup_ids = duplicates.ids # Skip if nothing to merge (defensive; shouldn't occur) next if dup_ids.empty? begin ActiveRecord::Base.transaction do # -------------------------------------------------------------- # HOLDINGS # -------------------------------------------------------------- holdings_moved = 0 holdings_deleted = 0 dup_ids.each do |dup_id| Holding.where(security_id: dup_id).find_each(batch_size: 1_000) do |holding| # Will this holding collide with an existing keeper row? conflict_exists = Holding.where( account_id: holding.account_id, security_id: keeper.id, date: holding.date, currency: holding.currency ).exists? if conflict_exists holding.destroy! holdings_deleted += 1 else holding.update!(security_id: keeper.id) holdings_moved += 1 end end end # -------------------------------------------------------------- # TRADES — no uniqueness constraints -> bulk update is fine # -------------------------------------------------------------- trades_moved = Trade.where(security_id: dup_ids).update_all(security_id: keeper.id) # Ensure no rows remain pointing at duplicates before deletion raise "Leftover holdings detected" if Holding.where(security_id: dup_ids).exists? raise "Leftover trades detected" if Trade.where(security_id: dup_ids).exists? duplicates.each(&:destroy!) # destroys its security_prices via dependent: :destroy # Log inside the transaction so counters are in-scope total_holdings = holdings_moved + holdings_deleted puts "[#{idx + 1}/#{duplicate_sets.size}] Merged #{dup_ids.join(', ')} → #{keeper.id} " \ "(#{total_holdings} holdings → #{holdings_moved} moved, #{holdings_deleted} removed, " \ "#{trades_moved} trades)" end rescue => e puts "ERROR migrating #{dup_ids.join(', ')}: #{e.message}" end end puts "✅ Duplicate security migration complete." end desc "Migrate account valuation anchors" # 2025-07-10: Set opening_anchor kinds for valuations to support event-sourced ledger model. # Manual accounts get their oldest valuation marked as opening_anchor, which acts as the # starting balance for the account. Current anchors are only used for Plaid accounts. task migrate_account_valuation_anchors: :environment do puts "==> Migrating account valuation anchors..." manual_accounts = Account.manual.includes(valuations: :entry) total_accounts = manual_accounts.count accounts_processed = 0 opening_anchors_set = 0 manual_accounts.find_each do |account| accounts_processed += 1 # Find oldest account entry oldest_entry = account.entries .order("date ASC, created_at ASC") .first # Check if it's a valuation that isn't already an anchor if oldest_entry && oldest_entry.valuation? derived_valuation_name = Valuation.build_opening_anchor_name(account.accountable_type) Account.transaction do oldest_entry.valuation.update!(kind: "opening_anchor") oldest_entry.update!(name: derived_valuation_name) end opening_anchors_set += 1 end if accounts_processed % 100 == 0 puts "[#{accounts_processed}/#{total_accounts}] Processed #{accounts_processed} accounts..." end rescue => e puts "ERROR processing account #{account.id}: #{e.message}" end puts "✅ Account valuation anchor migration complete." puts " Processed: #{accounts_processed} accounts" puts " Opening anchors set: #{opening_anchors_set}" end desc "Migrate balance components" # 2025-07-20: Migrate balance components to support event-sourced ledger model. # This task: # 1. Sets the flows_factor for each account based on the account's classification # 2. Sets the start_cash_balance, start_non_cash_balance, and start_balance for each balance # 3. Sets the cash_inflows, cash_outflows, non_cash_inflows, non_cash_outflows, net_market_flows, cash_adjustments, and non_cash_adjustments for each balance # 4. Sets the end_cash_balance, end_non_cash_balance, and end_balance for each balance task migrate_balance_components: :environment do puts "==> Migrating balance components..." BalanceComponentMigrator.run puts "✅ Balance component migration complete." end desc "Migrate global provider settings to family-specific" # 2025-11-21: Move global Lunchflow API credentials to family-specific lunchflow_items # Global settings are NO LONGER SUPPORTED as of this migration. # This improves security and enables proper multi-tenant isolation where each family # can have their own Lunchflow credentials instead of sharing global ones. task migrate_provider_settings_to_family: :environment do puts "==> Migrating global provider settings to family-specific..." puts "NOTE: Global Lunch flow/SimpleFIN credentials are NO LONGER SUPPORTED after this migration." puts # Check if global Lunchflow API key exists global_api_key = Setting[:lunchflow_api_key] global_base_url = Setting[:lunchflow_base_url] if global_api_key.blank? puts "No global Lunchflow API key found. Nothing to migrate." puts puts "ℹ️ If you need to configure Lunchflow:" puts " 1. Go to /settings/providers" puts " 2. Configure Lunchflow credentials per-family" puts puts "✅ Migration complete." return end puts "Found global Lunchflow API key. Migrating to family-specific settings..." families_updated = 0 families_with_existing = 0 families_with_items = 0 Family.find_each do |family| # Check if this family has any lunchflow_items has_lunchflow_items = family.lunchflow_items.exists? if has_lunchflow_items families_with_items += 1 # Check if any of the family's lunchflow_items already have credentials has_credentials = family.lunchflow_items.where.not(api_key: nil).exists? if has_credentials families_with_existing += 1 puts " Family #{family.id} (#{family.name}): Already has credentials, skipping" else # Assign global credentials to the first lunchflow_item lunchflow_item = family.lunchflow_items.first lunchflow_item.update!( api_key: global_api_key, base_url: global_base_url ) families_updated += 1 puts " Family #{family.id} (#{family.name}): Migrated credentials to lunchflow_item #{lunchflow_item.id}" end end end puts puts "Migration Summary:" puts " Families with Lunchflow items: #{families_with_items}" puts " Families with existing credentials: #{families_with_existing}" puts " Families updated with global credentials: #{families_updated}" puts if families_updated > 0 puts "✅ Global credentials have been copied to #{families_updated} families." puts puts "⚠️ IMPORTANT: You should now remove the global settings:" puts " rails runner \"Setting[:lunchflow_api_key] = nil; Setting[:lunchflow_base_url] = nil\"" puts puts " Global credentials are NO LONGER USED by the application." end puts "✅ Provider settings migration complete." end end