Merge remote-tracking branch 'upstream/main' into sso-upgrades

# Conflicts:
#	app/views/simplefin_items/_simplefin_item.html.erb
#	db/schema.rb

db/migrate/20251125141213_add_category_to_trades.rb

@@ -0,0 +1,7 @@
+class AddCategoryToTrades < ActiveRecord::Migration[7.2]
+  def change
+    unless column_exists?(:trades, :category_id)
+      add_reference :trades, :category, null: true, foreign_key: true, type: :uuid
+    end
+  end
+end

db/migrate/20251221060111_create_coinstats_items_and_accounts.rb

@@ -0,0 +1,60 @@
+class CreateCoinstatsItemsAndAccounts < ActiveRecord::Migration[7.2]
+  def change
+    # Create provider items table (stores per-family connection credentials)
+    create_table :coinstats_items, id: :uuid do |t|
+      t.references :family, null: false, foreign_key: true, type: :uuid
+      t.string :name
+
+      # Institution metadata
+      t.string :institution_id
+      t.string :institution_name
+      t.string :institution_domain
+      t.string :institution_url
+      t.string :institution_color
+
+      # Status and lifecycle
+      t.string :status, default: "good"
+      t.boolean :scheduled_for_deletion, default: false
+      t.boolean :pending_account_setup, default: false
+
+      # Sync settings
+      t.datetime :sync_start_date
+
+      # Raw data storage
+      t.jsonb :raw_payload
+      t.jsonb :raw_institution_payload
+
+      # Provider-specific credential fields
+      t.string :api_key, null: false
+
+      t.timestamps
+    end
+
+    add_index :coinstats_items, :status
+
+    # Create provider accounts table (stores individual account data from provider)
+    create_table :coinstats_accounts, id: :uuid do |t|
+      t.references :coinstats_item, null: false, foreign_key: true, type: :uuid
+
+      # Account identification
+      t.string :name
+      t.string :account_id
+
+      # Account details
+      t.string :currency
+      t.decimal :current_balance, precision: 19, scale: 4
+      t.string :account_status
+      t.string :account_type
+      t.string :provider
+
+      # Metadata and raw data
+      t.jsonb :institution_metadata
+      t.jsonb :raw_payload
+      t.jsonb :raw_transactions_payload
+
+      t.timestamps
+    end
+
+    add_index :coinstats_accounts, :account_id
+  end
+end

db/migrate/20251224215026_add_rows_count_to_imports.rb

@@ -0,0 +1,16 @@
+class AddRowsCountToImports < ActiveRecord::Migration[7.2]
+  def up
+    add_column :imports, :rows_count, :integer, default: 0, null: false
+
+    say_with_time "Backfilling rows_count for imports" do
+      Import.reset_column_information
+      Import.find_each do |import|
+        Import.reset_counters(import.id, :rows)
+      end
+    end
+  end
+
+  def down
+    remove_column :imports, :rows_count
+  end
+end

db/migrate/20260106152346_add_provisional_to_security_prices.rb

@@ -0,0 +1,5 @@
+class AddProvisionalToSecurityPrices < ActiveRecord::Migration[7.2]
+  def change
+    add_column :security_prices, :provisional, :boolean, default: false, null: false
+  end
+end

db/migrate/20260108000000_cleanup_orphaned_currency_balances.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# This migration cleans up orphaned balance records that have a currency different
+# from their account's current currency. This can happen when linked accounts
+# (SimpleFIN, Lunchflow, Enable Banking, Plaid) were created with an initial sync
+# before the correct currency was known from the provider.
+#
+# The fix in Account.create_and_sync with skip_initial_sync: true prevents this
+# going forward, but existing data needs to be cleaned up.
+class CleanupOrphanedCurrencyBalances < ActiveRecord::Migration[7.2]
+  def up
+    # Skip in test environment with empty database (CI)
+    return say "Skipping in test environment - no data to clean" if Rails.env.test? && account_count.zero?
+
+    # First, identify affected accounts for logging
+    affected_accounts = execute(<<~SQL).to_a
+      SELECT DISTINCT
+        a.id,
+        a.name,
+        a.currency as account_currency,
+        b.currency as orphaned_currency,
+        COUNT(b.id) as orphaned_balance_count
+      FROM accounts a
+      JOIN balances b ON a.id = b.account_id
+      WHERE b.currency != a.currency
+        AND (
+          a.simplefin_account_id IS NOT NULL
+          OR a.plaid_account_id IS NOT NULL
+          OR EXISTS (SELECT 1 FROM account_providers WHERE account_id = a.id)
+        )
+      GROUP BY a.id, a.name, a.currency, b.currency
+      ORDER BY a.name
+    SQL
+
+    if affected_accounts.any?
+      say "Found #{affected_accounts.size} account-currency combinations with orphaned balances:"
+      affected_accounts.each do |row|
+        say "  - #{row['name']}: #{row['orphaned_balance_count']} balances in #{row['orphaned_currency']} (account is #{row['account_currency']})"
+      end
+
+      # Delete orphaned balances where currency doesn't match account currency
+      # Only for linked accounts (provider-connected accounts)
+      execute(<<~SQL)
+        DELETE FROM balances
+        WHERE id IN (
+          SELECT b.id
+          FROM balances b
+          JOIN accounts a ON b.account_id = a.id
+          WHERE b.currency != a.currency
+            AND (
+              a.simplefin_account_id IS NOT NULL
+              OR a.plaid_account_id IS NOT NULL
+              OR EXISTS (SELECT 1 FROM account_providers WHERE account_id = a.id)
+            )
+        )
+      SQL
+
+      say "Deleted orphaned balances from linked accounts"
+
+      # Get unique account IDs that need re-sync
+      account_ids = affected_accounts.map { |row| row["id"] }.uniq
+
+      # Schedule re-sync for affected accounts to regenerate correct balances
+      # Only if Account model is available and responds to sync_later
+      if defined?(Account) && Account.respond_to?(:where)
+        say "Scheduling re-sync for #{account_ids.size} affected accounts..."
+        Account.where(id: account_ids).find_each do |account|
+          account.sync_later if account.respond_to?(:sync_later)
+        end
+        say "Scheduled re-sync for #{account_ids.size} affected accounts"
+      else
+        say "Skipping re-sync scheduling (Account model not available)"
+        say "Please manually sync affected accounts: #{account_ids.join(', ')}"
+      end
+    else
+      say "No orphaned currency balances found - database is clean"
+    end
+  end
+
+  def down
+    say "This migration cannot be fully reversed."
+    say "The deleted balances will be regenerated by the scheduled syncs."
+    say "If syncs haven't run yet, you may need to manually trigger them."
+  end
+
+  private
+
+    def account_count
+      execute("SELECT COUNT(*) FROM accounts").first["count"].to_i
+    rescue
+      0
+    end
+end

db/migrate/20260109100000_add_holdings_columns_to_lunchflow_accounts.rb

@@ -0,0 +1,6 @@
+class AddHoldingsColumnsToLunchflowAccounts < ActiveRecord::Migration[7.2]
+  def change
+    add_column :lunchflow_accounts, :holdings_supported, :boolean, default: true, null: false
+    add_column :lunchflow_accounts, :raw_holdings_payload, :jsonb
+  end
+end

db/migrate/20260109135841_add_website_url_to_securities.rb

@@ -0,0 +1,5 @@
+class AddWebsiteUrlToSecurities < ActiveRecord::Migration[7.2]
+  def change
+    add_column :securities, :website_url, :string
+  end
+end

db/migrate/20260109144012_create_family_merchant_associations.rb

@@ -0,0 +1,13 @@
+class CreateFamilyMerchantAssociations < ActiveRecord::Migration[7.2]
+  def change
+    create_table :family_merchant_associations, id: :uuid do |t|
+      t.references :family, null: false, foreign_key: true, type: :uuid
+      t.references :merchant, null: false, foreign_key: true, type: :uuid
+      t.datetime :unlinked_at
+
+      t.timestamps
+    end
+
+    add_index :family_merchant_associations, [ :family_id, :merchant_id ], unique: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
+ActiveRecord::Schema[7.2].define(version: 2026_01_10_122603) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -197,6 +197,45 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
     t.index ["user_id"], name: "index_chats_on_user_id"
   end
 
+  create_table "coinstats_accounts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+    t.uuid "coinstats_item_id", null: false
+    t.string "name"
+    t.string "account_id"
+    t.string "currency"
+    t.decimal "current_balance", precision: 19, scale: 4
+    t.string "account_status"
+    t.string "account_type"
+    t.string "provider"
+    t.jsonb "institution_metadata"
+    t.jsonb "raw_payload"
+    t.jsonb "raw_transactions_payload"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_coinstats_accounts_on_account_id"
+    t.index ["coinstats_item_id"], name: "index_coinstats_accounts_on_coinstats_item_id"
+  end
+
+  create_table "coinstats_items", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+    t.uuid "family_id", null: false
+    t.string "name"
+    t.string "institution_id"
+    t.string "institution_name"
+    t.string "institution_domain"
+    t.string "institution_url"
+    t.string "institution_color"
+    t.string "status", default: "good"
+    t.boolean "scheduled_for_deletion", default: false
+    t.boolean "pending_account_setup", default: false
+    t.datetime "sync_start_date"
+    t.jsonb "raw_payload"
+    t.jsonb "raw_institution_payload"
+    t.string "api_key", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["family_id"], name: "index_coinstats_items_on_family_id"
+    t.index ["status"], name: "index_coinstats_items_on_status"
+  end
+
   create_table "credit_cards", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -422,6 +461,17 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
     t.index ["family_id"], name: "index_family_exports_on_family_id"
   end
 
+  create_table "family_merchant_associations", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+    t.uuid "family_id", null: false
+    t.uuid "merchant_id", null: false
+    t.datetime "unlinked_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["family_id", "merchant_id"], name: "idx_on_family_id_merchant_id_23e883e08f", unique: true
+    t.index ["family_id"], name: "index_family_merchant_associations_on_family_id"
+    t.index ["merchant_id"], name: "index_family_merchant_associations_on_merchant_id"
+  end
+
   create_table "flipper_features", force: :cascade do |t|
     t.string "key", null: false
     t.datetime "created_at", null: false
@@ -554,6 +604,7 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
     t.string "exchange_operating_mic_col_label"
     t.string "amount_type_strategy", default: "signed_amount"
     t.string "amount_type_inflow_value"
+    t.integer "rows_count", default: 0, null: false
     t.index ["family_id"], name: "index_imports_on_family_id"
   end
 
@@ -630,6 +681,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
     t.jsonb "raw_transactions_payload"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.boolean "holdings_supported", default: true, null: false
+    t.jsonb "raw_holdings_payload"
     t.index ["account_id"], name: "index_lunchflow_accounts_on_account_id"
     t.index ["lunchflow_item_id"], name: "index_lunchflow_accounts_on_lunchflow_item_id"
   end
@@ -928,6 +981,7 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
     t.datetime "failed_fetch_at"
     t.integer "failed_fetch_count", default: 0, null: false
     t.datetime "last_health_check_at"
+    t.string "website_url"
     t.index "upper((ticker)::text), COALESCE(upper((exchange_operating_mic)::text), ''::text)", name: "index_securities_on_ticker_and_exchange_operating_mic_unique", unique: true
     t.index ["country_code"], name: "index_securities_on_country_code"
     t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic"
@@ -940,6 +994,7 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.uuid "security_id"
+    t.boolean "provisional", default: false, null: false
     t.index ["security_id", "date", "currency"], name: "index_security_prices_on_security_id_and_date_and_currency", unique: true
     t.index ["security_id"], name: "index_security_prices_on_security_id"
   end
@@ -1221,6 +1276,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
   add_foreign_key "budgets", "families"
   add_foreign_key "categories", "families"
   add_foreign_key "chats", "users"
+  add_foreign_key "coinstats_accounts", "coinstats_items"
+  add_foreign_key "coinstats_items", "families"
   add_foreign_key "enable_banking_accounts", "enable_banking_items"
   add_foreign_key "enable_banking_items", "families"
   add_foreign_key "entries", "accounts", on_delete: :cascade
@@ -1230,6 +1287,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_01_03_170412) do
   add_foreign_key "eval_runs", "eval_datasets"
   add_foreign_key "eval_samples", "eval_datasets"
   add_foreign_key "family_exports", "families"
+  add_foreign_key "family_merchant_associations", "families"
+  add_foreign_key "family_merchant_associations", "merchants"
   add_foreign_key "holdings", "account_providers"
   add_foreign_key "holdings", "accounts", on_delete: :cascade
   add_foreign_key "holdings", "securities"