From 140ea78b0eeccfa865d55a40b8cfc42179f23ac9 Mon Sep 17 00:00:00 2001 From: LPW Date: Fri, 9 Jan 2026 13:26:37 -0500 Subject: [PATCH] Add global sync summary component for all providers (#588) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add shared sync statistics collection and provider sync summary UI - Introduced `SyncStats::Collector` concern to centralize sync statistics logic, including account, transaction, holdings, and health stats collection. - Added collapsible `ProviderSyncSummary` component for displaying sync summaries across providers. - Updated syncers (e.g., `LunchflowItem::Syncer`) to use the shared collector methods for consistent stats calculation. - Added rake tasks under `dev:sync_stats` for testing and development purposes, including fake stats generation with optional issues. - Enhanced provider-specific views to include sync summaries using the new shared component. * Refactor `ProviderSyncSummary` to improve maintainability - Extracted `severity_color_class` to simplify severity-to-CSS mapping. - Replaced `holdings_label` with `holdings_label_key` for streamlined localization. - Updated locale file to separate `found` and `processed` translations for clarity. --------- Signed-off-by: Juan José Mata Co-authored-by: Josh Waldrep Co-authored-by: Juan José Mata --- app/components/provider_sync_summary.html.erb | 105 ++++++ app/components/provider_sync_summary.rb | 157 +++++++++ app/controllers/accounts_controller.rb | 111 +++--- app/models/concerns/sync_stats/collector.rb | 198 +++++++++++ app/models/lunchflow_item/syncer.rb | 31 +- app/models/plaid_item/syncer.rb | 59 +++- .../coinstats_items/_coinstats_item.html.erb | 15 +- .../_enable_banking_item.html.erb | 11 + .../lunchflow_items/_lunchflow_item.html.erb | 12 + app/views/plaid_items/_plaid_item.html.erb | 15 +- .../simplefin_items/_simplefin_item.html.erb | 70 +--- config/locales/views/components/en.yml | 29 ++ lib/tasks/dev_sync_stats.rake | 315 ++++++++++++++++++ 13 files changed, 998 insertions(+), 130 deletions(-) create mode 100644 app/components/provider_sync_summary.html.erb create mode 100644 app/components/provider_sync_summary.rb create mode 100644 app/models/concerns/sync_stats/collector.rb create mode 100644 config/locales/views/components/en.yml create mode 100644 lib/tasks/dev_sync_stats.rake diff --git a/app/components/provider_sync_summary.html.erb b/app/components/provider_sync_summary.html.erb new file mode 100644 index 000000000..488a86004 --- /dev/null +++ b/app/components/provider_sync_summary.html.erb @@ -0,0 +1,105 @@ +
+ +
+ <%= helpers.icon "chevron-right", class: "group-open:transform group-open:rotate-90" %> + <%= t("provider_sync_summary.title") %> +
+
+ <% if last_synced_at %> + <%= t("provider_sync_summary.last_sync", time_ago: last_synced_ago) %> + <% end %> +
+ + +
+ <%# Accounts section - always shown if we have account stats %> + <% if total_accounts > 0 || stats.key?("total_accounts") %> +
+

<%= t("provider_sync_summary.accounts.title") %>

+
+ <%= t("provider_sync_summary.accounts.total", count: total_accounts) %> + <%= t("provider_sync_summary.accounts.linked", count: linked_accounts) %> + <%= t("provider_sync_summary.accounts.unlinked", count: unlinked_accounts) %> + <% if institutions_count.present? %> + <%= t("provider_sync_summary.accounts.institutions", count: institutions_count) %> + <% end %> +
+
+ <% end %> + + <%# Transactions section - shown if provider collects transaction stats %> + <% if has_transaction_stats? %> +
+

<%= t("provider_sync_summary.transactions.title") %>

+
+ <%= t("provider_sync_summary.transactions.seen", count: tx_seen) %> + <%= t("provider_sync_summary.transactions.imported", count: tx_imported) %> + <%= t("provider_sync_summary.transactions.updated", count: tx_updated) %> + <%= t("provider_sync_summary.transactions.skipped", count: tx_skipped) %> +
+
+ <% end %> + + <%# Holdings section - shown if provider collects holdings stats %> + <% if has_holdings_stats? %> +
+

<%= t("provider_sync_summary.holdings.title") %>

+
+ <%= t("provider_sync_summary.holdings.#{holdings_label_key}", count: holdings_count) %> +
+
+ <% end %> + + <%# Health section - always shown %> +
+

<%= t("provider_sync_summary.health.title") %>

+
+
+ <% if rate_limited? %> + + <%= t("provider_sync_summary.health.rate_limited", time_ago: rate_limited_ago || t("provider_sync_summary.health.recently")) %> + + <% end %> + <% if has_errors? %> + <%= t("provider_sync_summary.health.errors", count: total_errors) %> + <% elsif import_started? %> + <%= t("provider_sync_summary.health.errors", count: 0) %> + <% else %> + <%= t("provider_sync_summary.health.errors", count: 0) %> + <% end %> +
+ + <%# Data quality warnings %> + <% if has_data_quality_issues? %> +
+ <% if data_warnings > 0 %> +
+ <%= helpers.icon "alert-triangle", size: "sm", color: "warning" %> + <%= t("provider_sync_summary.health.data_warnings", count: data_warnings) %> +
+ <% end %> + <% if notices > 0 %> +
+ <%= helpers.icon "info", size: "sm" %> + <%= t("provider_sync_summary.health.notices", count: notices) %> +
+ <% end %> +
+ + <% if data_quality_details.any? %> +
+ + <%= t("provider_sync_summary.health.view_data_quality") %> + +
+ <% data_quality_details.each do |detail| %> +

"><%= detail["message"] %>

+ <% end %> +
+
+ <% end %> + <% end %> +
+
+
+
diff --git a/app/components/provider_sync_summary.rb b/app/components/provider_sync_summary.rb new file mode 100644 index 000000000..4d00f2343 --- /dev/null +++ b/app/components/provider_sync_summary.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +# Reusable sync summary component for provider items. +# +# This component displays sync statistics in a collapsible panel that can be used +# by any provider (SimpleFIN, Plaid, Lunchflow, etc.) to show their sync results. +# +# @example Basic usage +# <%= render ProviderSyncSummary.new( +# stats: @sync_stats, +# provider_item: @plaid_item +# ) %> +# +# @example With custom institution count +# <%= render ProviderSyncSummary.new( +# stats: @sync_stats, +# provider_item: @simplefin_item, +# institutions_count: @simplefin_item.connected_institutions.size +# ) %> +# +class ProviderSyncSummary < ViewComponent::Base + attr_reader :stats, :provider_item, :institutions_count + + # @param stats [Hash] The sync statistics hash from sync.sync_stats + # @param provider_item [Object] The provider item (must respond to last_synced_at) + # @param institutions_count [Integer, nil] Optional count of connected institutions + def initialize(stats:, provider_item:, institutions_count: nil) + @stats = stats || {} + @provider_item = provider_item + @institutions_count = institutions_count + end + + def render? + stats.present? + end + + # Account statistics + def total_accounts + stats["total_accounts"].to_i + end + + def linked_accounts + stats["linked_accounts"].to_i + end + + def unlinked_accounts + stats["unlinked_accounts"].to_i + end + + # Transaction statistics + def tx_seen + stats["tx_seen"].to_i + end + + def tx_imported + stats["tx_imported"].to_i + end + + def tx_updated + stats["tx_updated"].to_i + end + + def tx_skipped + stats["tx_skipped"].to_i + end + + def has_transaction_stats? + stats.key?("tx_seen") || stats.key?("tx_imported") || stats.key?("tx_updated") + end + + # Holdings statistics + def holdings_found + stats["holdings_found"].to_i + end + + def holdings_processed + stats["holdings_processed"].to_i + end + + def has_holdings_stats? + stats.key?("holdings_found") || stats.key?("holdings_processed") + end + + def holdings_label_key + stats.key?("holdings_processed") ? "processed" : "found" + end + + def holdings_count + stats.key?("holdings_processed") ? holdings_processed : holdings_found + end + + # Returns the CSS color class for a data quality detail severity + # @param severity [String] The severity level ("warning", "error", or other) + # @return [String] The Tailwind CSS class for the color + def severity_color_class(severity) + case severity + when "warning" then "text-warning" + when "error" then "text-destructive" + else "text-secondary" + end + end + + # Health statistics + def rate_limited? + stats["rate_limited"].present? || stats["rate_limited_at"].present? + end + + def rate_limited_ago + return nil unless stats["rate_limited_at"].present? + + begin + helpers.time_ago_in_words(Time.parse(stats["rate_limited_at"])) + rescue StandardError + nil + end + end + + def total_errors + stats["total_errors"].to_i + end + + def import_started? + stats["import_started"].present? + end + + def has_errors? + total_errors > 0 + end + + # Data quality / warnings + def data_warnings + stats["data_warnings"].to_i + end + + def notices + stats["notices"].to_i + end + + def data_quality_details + stats["data_quality_details"] || [] + end + + def has_data_quality_issues? + data_warnings > 0 || notices > 0 || data_quality_details.any? + end + + # Last sync time + def last_synced_at + provider_item.last_synced_at + end + + def last_synced_ago + return nil unless last_synced_at + + helpers.time_ago_in_words(last_synced_at) + end +end diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 1e33156d2..48e484f40 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -6,51 +6,14 @@ class AccountsController < ApplicationController @manual_accounts = family.accounts .listable_manual .order(:name) - @plaid_items = family.plaid_items.ordered + @plaid_items = family.plaid_items.ordered.includes(:syncs, :plaid_accounts) @simplefin_items = family.simplefin_items.ordered.includes(:syncs) - @lunchflow_items = family.lunchflow_items.ordered + @lunchflow_items = family.lunchflow_items.ordered.includes(:syncs, :lunchflow_accounts) @enable_banking_items = family.enable_banking_items.ordered.includes(:syncs) @coinstats_items = family.coinstats_items.ordered.includes(:coinstats_accounts, :accounts, :syncs) - # Precompute per-item maps to avoid queries in the view - @simplefin_sync_stats_map = {} - @simplefin_has_unlinked_map = {} - - @simplefin_items.each do |item| - latest_sync = item.syncs.ordered.first - @simplefin_sync_stats_map[item.id] = (latest_sync&.sync_stats || {}) - @simplefin_has_unlinked_map[item.id] = item.family.accounts - .listable_manual - .exists? - end - - # Count of SimpleFin accounts that are not linked (no legacy account and no AccountProvider) - @simplefin_unlinked_count_map = {} - @simplefin_items.each do |item| - count = item.simplefin_accounts - .left_joins(:account, :account_provider) - .where(accounts: { id: nil }, account_providers: { id: nil }) - .count - @simplefin_unlinked_count_map[item.id] = count - end - - # Compute CTA visibility map used by the simplefin_item partial - @simplefin_show_relink_map = {} - @simplefin_items.each do |item| - begin - unlinked_count = @simplefin_unlinked_count_map[item.id] || 0 - manuals_exist = @simplefin_has_unlinked_map[item.id] - sfa_any = if item.simplefin_accounts.loaded? - item.simplefin_accounts.any? - else - item.simplefin_accounts.exists? - end - @simplefin_show_relink_map[item.id] = (unlinked_count.to_i == 0 && manuals_exist && sfa_any) - rescue => e - Rails.logger.warn("SimpleFin card: CTA computation failed for item #{item.id}: #{e.class} - #{e.message}") - @simplefin_show_relink_map[item.id] = false - end - end + # Build sync stats maps for all providers + build_sync_stats_maps # Prevent Turbo Drive from caching this page to ensure fresh account lists expires_now @@ -210,4 +173,70 @@ class AccountsController < ApplicationController def set_account @account = family.accounts.find(params[:id]) end + + # Builds sync stats maps for all provider types to avoid N+1 queries in views + def build_sync_stats_maps + # SimpleFIN sync stats + @simplefin_sync_stats_map = {} + @simplefin_has_unlinked_map = {} + @simplefin_unlinked_count_map = {} + @simplefin_show_relink_map = {} + @simplefin_duplicate_only_map = {} + + @simplefin_items.each do |item| + latest_sync = item.syncs.ordered.first + stats = latest_sync&.sync_stats || {} + @simplefin_sync_stats_map[item.id] = stats + @simplefin_has_unlinked_map[item.id] = item.family.accounts.listable_manual.exists? + + # Count unlinked accounts + count = item.simplefin_accounts + .left_joins(:account, :account_provider) + .where(accounts: { id: nil }, account_providers: { id: nil }) + .count + @simplefin_unlinked_count_map[item.id] = count + + # CTA visibility + manuals_exist = @simplefin_has_unlinked_map[item.id] + sfa_any = item.simplefin_accounts.loaded? ? item.simplefin_accounts.any? : item.simplefin_accounts.exists? + @simplefin_show_relink_map[item.id] = (count.to_i == 0 && manuals_exist && sfa_any) + + # Check if all errors are duplicate-skips + errors = Array(stats["errors"]).map { |e| e.is_a?(Hash) ? e["message"] || e[:message] : e.to_s } + @simplefin_duplicate_only_map[item.id] = errors.present? && errors.all? { |m| m.to_s.downcase.include?("duplicate upstream account detected") } + rescue => e + Rails.logger.warn("SimpleFin stats map build failed for item #{item.id}: #{e.class} - #{e.message}") + @simplefin_sync_stats_map[item.id] = {} + @simplefin_show_relink_map[item.id] = false + @simplefin_duplicate_only_map[item.id] = false + end + + # Plaid sync stats + @plaid_sync_stats_map = {} + @plaid_items.each do |item| + latest_sync = item.syncs.ordered.first + @plaid_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + + # Lunchflow sync stats + @lunchflow_sync_stats_map = {} + @lunchflow_items.each do |item| + latest_sync = item.syncs.ordered.first + @lunchflow_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + + # Enable Banking sync stats + @enable_banking_sync_stats_map = {} + @enable_banking_items.each do |item| + latest_sync = item.syncs.ordered.first + @enable_banking_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + + # CoinStats sync stats + @coinstats_sync_stats_map = {} + @coinstats_items.each do |item| + latest_sync = item.syncs.ordered.first + @coinstats_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + end end diff --git a/app/models/concerns/sync_stats/collector.rb b/app/models/concerns/sync_stats/collector.rb new file mode 100644 index 000000000..8333a0d8d --- /dev/null +++ b/app/models/concerns/sync_stats/collector.rb @@ -0,0 +1,198 @@ +# frozen_string_literal: true + +# SyncStats::Collector provides shared methods for collecting sync statistics +# across different provider syncers. +# +# This concern standardizes the stat collection interface so all providers +# can report consistent sync summaries. +# +# @example Include in a syncer class +# class PlaidItem::Syncer +# include SyncStats::Collector +# +# def perform_sync(sync) +# # ... sync logic ... +# collect_setup_stats(sync, provider_accounts: plaid_item.plaid_accounts) +# collect_transaction_stats(sync, account_ids: account_ids, source: "plaid") +# # ... +# end +# end +# +module SyncStats + module Collector + extend ActiveSupport::Concern + + # Collects account setup statistics (total, linked, unlinked counts). + # + # @param sync [Sync] The sync record to update + # @param provider_accounts [ActiveRecord::Relation] The provider accounts (e.g., SimplefinAccount, PlaidAccount) + # @param linked_check [Proc, nil] Optional proc to check if an account is linked. If nil, uses default logic. + # @return [Hash] The setup stats that were collected + def collect_setup_stats(sync, provider_accounts:, linked_check: nil) + return {} unless sync.respond_to?(:sync_stats) + + total_accounts = provider_accounts.count + + # Count linked accounts - either via custom check or default association check + linked_count = if linked_check + provider_accounts.count { |pa| linked_check.call(pa) } + else + # Default: check for current_account method or account association + provider_accounts.count do |pa| + (pa.respond_to?(:current_account) && pa.current_account.present?) || + (pa.respond_to?(:account) && pa.account.present?) + end + end + + unlinked_count = total_accounts - linked_count + + setup_stats = { + "total_accounts" => total_accounts, + "linked_accounts" => linked_count, + "unlinked_accounts" => unlinked_count + } + + merge_sync_stats(sync, setup_stats) + setup_stats + end + + # Collects transaction statistics (imported, updated, seen, skipped). + # + # @param sync [Sync] The sync record to update + # @param account_ids [Array] The account IDs to count transactions for + # @param source [String] The transaction source (e.g., "simplefin", "plaid") + # @param window_start [Time, nil] Start of the sync window (defaults to sync.created_at or 30 minutes ago) + # @param window_end [Time, nil] End of the sync window (defaults to Time.current) + # @return [Hash] The transaction stats that were collected + def collect_transaction_stats(sync, account_ids:, source:, window_start: nil, window_end: nil) + return {} unless sync.respond_to?(:sync_stats) + return {} if account_ids.empty? + + window_start ||= sync.created_at || 30.minutes.ago + window_end ||= Time.current + + tx_scope = Entry.where(account_id: account_ids, source: source, entryable_type: "Transaction") + tx_imported = tx_scope.where(created_at: window_start..window_end).count + tx_updated = tx_scope.where(updated_at: window_start..window_end) + .where.not(created_at: window_start..window_end).count + tx_seen = tx_imported + tx_updated + + tx_stats = { + "tx_imported" => tx_imported, + "tx_updated" => tx_updated, + "tx_seen" => tx_seen, + "window_start" => window_start.iso8601, + "window_end" => window_end.iso8601 + } + + merge_sync_stats(sync, tx_stats) + tx_stats + end + + # Collects holdings statistics. + # + # @param sync [Sync] The sync record to update + # @param holdings_count [Integer] The number of holdings found/processed + # @param label [String] The label for the stat ("found" or "processed") + # @return [Hash] The holdings stats that were collected + def collect_holdings_stats(sync, holdings_count:, label: "found") + return {} unless sync.respond_to?(:sync_stats) + + key = label == "processed" ? "holdings_processed" : "holdings_found" + holdings_stats = { key => holdings_count } + + merge_sync_stats(sync, holdings_stats) + holdings_stats + end + + # Collects health/error statistics. + # + # @param sync [Sync] The sync record to update + # @param errors [Array, nil] Array of error objects with :message and optional :category + # @param rate_limited [Boolean] Whether the sync was rate limited + # @param rate_limited_at [Time, nil] When rate limiting occurred + # @return [Hash] The health stats that were collected + def collect_health_stats(sync, errors: nil, rate_limited: false, rate_limited_at: nil) + return {} unless sync.respond_to?(:sync_stats) + + health_stats = { + "import_started" => true + } + + if errors.present? + health_stats["errors"] = errors.map do |e| + e.is_a?(Hash) ? e.stringify_keys : { "message" => e.to_s } + end + health_stats["total_errors"] = errors.size + else + health_stats["total_errors"] = 0 + end + + if rate_limited + health_stats["rate_limited"] = true + health_stats["rate_limited_at"] = rate_limited_at&.iso8601 + end + + merge_sync_stats(sync, health_stats) + health_stats + end + + # Collects data quality warnings and notices. + # + # @param sync [Sync] The sync record to update + # @param warnings [Integer] Number of data warnings + # @param notices [Integer] Number of notices + # @param details [Array] Array of detail objects with :message and optional :severity + # @return [Hash] The data quality stats that were collected + def collect_data_quality_stats(sync, warnings: 0, notices: 0, details: []) + return {} unless sync.respond_to?(:sync_stats) + + quality_stats = { + "data_warnings" => warnings, + "notices" => notices + } + + if details.present? + quality_stats["data_quality_details"] = details.map do |d| + d.is_a?(Hash) ? d.stringify_keys : { "message" => d.to_s, "severity" => "info" } + end + end + + merge_sync_stats(sync, quality_stats) + quality_stats + end + + # Marks the sync as having started import (used for health indicator). + # + # @param sync [Sync] The sync record to update + def mark_import_started(sync) + return unless sync.respond_to?(:sync_stats) + + merge_sync_stats(sync, { "import_started" => true }) + end + + # Clears previous sync stats (useful at start of sync). + # + # @param sync [Sync] The sync record to update + def clear_sync_stats(sync) + return unless sync.respond_to?(:sync_stats) + + sync.update!(sync_stats: { "cleared_at" => Time.current.iso8601 }) + end + + private + + # Merges new stats into the existing sync_stats hash. + # + # @param sync [Sync] The sync record to update + # @param new_stats [Hash] The new stats to merge + def merge_sync_stats(sync, new_stats) + return unless sync.respond_to?(:sync_stats) + + existing = sync.sync_stats || {} + sync.update!(sync_stats: existing.merge(new_stats)) + rescue => e + Rails.logger.warn("SyncStats::Collector#merge_sync_stats failed: #{e.class} - #{e.message}") + end + end +end diff --git a/app/models/lunchflow_item/syncer.rb b/app/models/lunchflow_item/syncer.rb index b39b5586e..73147c4ef 100644 --- a/app/models/lunchflow_item/syncer.rb +++ b/app/models/lunchflow_item/syncer.rb @@ -1,4 +1,6 @@ class LunchflowItem::Syncer + include SyncStats::Collector + attr_reader :lunchflow_item def initialize(lunchflow_item) @@ -10,18 +12,13 @@ class LunchflowItem::Syncer sync.update!(status_text: "Importing accounts from Lunchflow...") if sync.respond_to?(:status_text) lunchflow_item.import_latest_lunchflow_data - # Phase 2: Check account setup status and collect sync statistics + # Phase 2: Collect setup statistics using shared concern sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text) - total_accounts = lunchflow_item.lunchflow_accounts.count - linked_accounts = lunchflow_item.lunchflow_accounts.joins(:account).merge(Account.visible) - unlinked_accounts = lunchflow_item.lunchflow_accounts.includes(:account).where(accounts: { id: nil }) + collect_setup_stats(sync, provider_accounts: lunchflow_item.lunchflow_accounts) - # Store sync statistics for display - sync_stats = { - total_accounts: total_accounts, - linked_accounts: linked_accounts.count, - unlinked_accounts: unlinked_accounts.count - } + # Check for unlinked accounts + linked_accounts = lunchflow_item.lunchflow_accounts.joins(:account_provider) + unlinked_accounts = lunchflow_item.lunchflow_accounts.left_joins(:account_provider).where(account_providers: { id: nil }) # Set pending_account_setup if there are unlinked accounts if unlinked_accounts.any? @@ -34,6 +31,7 @@ class LunchflowItem::Syncer # Phase 3: Process transactions and holdings for linked accounts only if linked_accounts.any? sync.update!(status_text: "Processing transactions and holdings...") if sync.respond_to?(:status_text) + mark_import_started(sync) Rails.logger.info "LunchflowItem::Syncer - Processing #{linked_accounts.count} linked accounts" lunchflow_item.process_accounts Rails.logger.info "LunchflowItem::Syncer - Finished processing accounts" @@ -45,14 +43,19 @@ class LunchflowItem::Syncer window_start_date: sync.window_start_date, window_end_date: sync.window_end_date ) + + # Phase 5: Collect transaction statistics + account_ids = linked_accounts.includes(:account_provider).filter_map { |la| la.current_account&.id } + collect_transaction_stats(sync, account_ids: account_ids, source: "lunchflow") else Rails.logger.info "LunchflowItem::Syncer - No linked accounts to process" end - # Store sync statistics in the sync record for status display - if sync.respond_to?(:sync_stats) - sync.update!(sync_stats: sync_stats) - end + # Mark sync health + collect_health_stats(sync, errors: nil) + rescue => e + collect_health_stats(sync, errors: [ { message: e.message, category: "sync_error" } ]) + raise end def perform_post_sync diff --git a/app/models/plaid_item/syncer.rb b/app/models/plaid_item/syncer.rb index b76c37b67..58c2baccd 100644 --- a/app/models/plaid_item/syncer.rb +++ b/app/models/plaid_item/syncer.rb @@ -1,4 +1,6 @@ class PlaidItem::Syncer + include SyncStats::Collector + attr_reader :plaid_item def initialize(plaid_item) @@ -6,21 +8,60 @@ class PlaidItem::Syncer end def perform_sync(sync) - # Loads item metadata, accounts, transactions, and other data to our DB + # Phase 1: Import data from Plaid API + sync.update!(status_text: "Importing accounts from Plaid...") if sync.respond_to?(:status_text) plaid_item.import_latest_plaid_data - # Processes the raw Plaid data and updates internal domain objects - plaid_item.process_accounts + # Phase 2: Collect setup statistics + sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text) + collect_setup_stats(sync, provider_accounts: plaid_item.plaid_accounts) - # All data is synced, so we can now run an account sync to calculate historical balances and more - plaid_item.schedule_account_syncs( - parent_sync: sync, - window_start_date: sync.window_start_date, - window_end_date: sync.window_end_date - ) + # Check for unlinked accounts and update pending_account_setup flag + unlinked_count = plaid_item.plaid_accounts.count { |pa| pa.current_account.nil? } + if unlinked_count > 0 + plaid_item.update!(pending_account_setup: true) if plaid_item.respond_to?(:pending_account_setup=) + sync.update!(status_text: "#{unlinked_count} accounts need setup...") if sync.respond_to?(:status_text) + else + plaid_item.update!(pending_account_setup: false) if plaid_item.respond_to?(:pending_account_setup=) + end + + # Phase 3: Process the raw Plaid data and updates internal domain objects + linked_accounts = plaid_item.plaid_accounts.select { |pa| pa.current_account.present? } + if linked_accounts.any? + sync.update!(status_text: "Processing transactions...") if sync.respond_to?(:status_text) + mark_import_started(sync) + plaid_item.process_accounts + + # Phase 4: Schedule balance calculations + sync.update!(status_text: "Calculating balances...") if sync.respond_to?(:status_text) + plaid_item.schedule_account_syncs( + parent_sync: sync, + window_start_date: sync.window_start_date, + window_end_date: sync.window_end_date + ) + + # Phase 5: Collect transaction and holdings statistics + account_ids = linked_accounts.filter_map { |pa| pa.current_account&.id } + collect_transaction_stats(sync, account_ids: account_ids, source: "plaid") + collect_holdings_stats(sync, holdings_count: count_holdings(linked_accounts), label: "processed") + end + + # Mark sync health + collect_health_stats(sync, errors: nil) + rescue => e + collect_health_stats(sync, errors: [ { message: e.message, category: "sync_error" } ]) + raise end def perform_post_sync # no-op end + + private + + def count_holdings(plaid_accounts) + plaid_accounts.sum do |pa| + Array(pa.raw_investments_payload).size + end + end end diff --git a/app/views/coinstats_items/_coinstats_item.html.erb b/app/views/coinstats_items/_coinstats_item.html.erb index 4d6875281..55736702e 100644 --- a/app/views/coinstats_items/_coinstats_item.html.erb +++ b/app/views/coinstats_items/_coinstats_item.html.erb @@ -80,7 +80,20 @@
<% if coinstats_item.accounts.any? %> <%= render "accounts/index/account_groups", accounts: coinstats_item.accounts %> - <% else %> + <% end %> + + <%# Sync summary (collapsible) - using shared ProviderSyncSummary component %> + <% stats = if defined?(@coinstats_sync_stats_map) && @coinstats_sync_stats_map + @coinstats_sync_stats_map[coinstats_item.id] || {} + else + coinstats_item.syncs.ordered.first&.sync_stats || {} + end %> + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: coinstats_item + ) %> + + <% if coinstats_item.accounts.empty? %>

<%= t(".no_wallets_title") %>

<%= t(".no_wallets_message") %>

diff --git a/app/views/enable_banking_items/_enable_banking_item.html.erb b/app/views/enable_banking_items/_enable_banking_item.html.erb index 4c13e8860..f82ea7b27 100644 --- a/app/views/enable_banking_items/_enable_banking_item.html.erb +++ b/app/views/enable_banking_items/_enable_banking_item.html.erb @@ -85,6 +85,17 @@ <%= render "accounts/index/account_groups", accounts: enable_banking_item.accounts %> <% end %> + <%# Sync summary (collapsible) - using shared ProviderSyncSummary component %> + <% stats = if defined?(@enable_banking_sync_stats_map) && @enable_banking_sync_stats_map + @enable_banking_sync_stats_map[enable_banking_item.id] || {} + else + enable_banking_item.syncs.ordered.first&.sync_stats || {} + end %> + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: enable_banking_item + ) %> + <% if enable_banking_item.unlinked_accounts_count > 0 %>

Setup needed

diff --git a/app/views/lunchflow_items/_lunchflow_item.html.erb b/app/views/lunchflow_items/_lunchflow_item.html.erb index fefb2f910..f03b8755c 100644 --- a/app/views/lunchflow_items/_lunchflow_item.html.erb +++ b/app/views/lunchflow_items/_lunchflow_item.html.erb @@ -82,6 +82,18 @@ <%= render "accounts/index/account_groups", accounts: lunchflow_item.accounts %> <% end %> + <%# Sync summary (collapsible) - using shared ProviderSyncSummary component %> + <% stats = if defined?(@lunchflow_sync_stats_map) && @lunchflow_sync_stats_map + @lunchflow_sync_stats_map[lunchflow_item.id] || {} + else + lunchflow_item.syncs.ordered.first&.sync_stats || {} + end %> + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: lunchflow_item, + institutions_count: lunchflow_item.connected_institutions.size + ) %> + <%# Use model methods for consistent counts %> <% unlinked_count = lunchflow_item.unlinked_accounts_count %> <% linked_count = lunchflow_item.linked_accounts_count %> diff --git a/app/views/plaid_items/_plaid_item.html.erb b/app/views/plaid_items/_plaid_item.html.erb index 738f7facf..2c8e26c15 100644 --- a/app/views/plaid_items/_plaid_item.html.erb +++ b/app/views/plaid_items/_plaid_item.html.erb @@ -80,7 +80,20 @@
<% if plaid_item.accounts.any? %> <%= render "accounts/index/account_groups", accounts: plaid_item.accounts %> - <% else %> + <% end %> + + <%# Sync summary (collapsible) - using shared ProviderSyncSummary component %> + <% stats = if defined?(@plaid_sync_stats_map) && @plaid_sync_stats_map + @plaid_sync_stats_map[plaid_item.id] || {} + else + plaid_item.syncs.ordered.first&.sync_stats || {} + end %> + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: plaid_item + ) %> + + <% if plaid_item.accounts.empty? %>

<%= t(".no_accounts_title") %>

<%= t(".no_accounts_description") %>

diff --git a/app/views/simplefin_items/_simplefin_item.html.erb b/app/views/simplefin_items/_simplefin_item.html.erb index c4cee3b88..db3a0b731 100644 --- a/app/views/simplefin_items/_simplefin_item.html.erb +++ b/app/views/simplefin_items/_simplefin_item.html.erb @@ -171,7 +171,7 @@ <%= render "accounts/index/account_groups", accounts: simplefin_item.accounts %> <% end %> - <%# Sync summary (collapsible) + <%# Sync summary (collapsible) - using shared ProviderSyncSummary component Prefer controller-provided map; fallback to latest sync stats so Turbo broadcasts can render the summary without requiring a full page refresh. %> <% stats = if defined?(@simplefin_sync_stats_map) && @simplefin_sync_stats_map @@ -180,69 +180,11 @@ # `latest_sync` is private on Syncable; access via association for broadcast renders. simplefin_item.syncs.ordered.first&.sync_stats || {} end %> - <% if stats.present? %> -
- -
- <%= icon "chevron-right", class: "group-open:transform group-open:rotate-90" %> - Sync summary -
-
- <% if simplefin_item.last_synced_at %> - Last sync: <%= time_ago_in_words(simplefin_item.last_synced_at) %> ago - <% end %> -
- -
-
-

Accounts

-
- Total: <%= stats["total_accounts"].to_i %> - Linked: <%= stats["linked_accounts"].to_i %> - Unlinked: <%= stats["unlinked_accounts"].to_i %> - <% institutions = simplefin_item.connected_institutions %> - Institutions: <%= institutions.size %> -
-
-
-

Transactions

-
- Seen: <%= stats["tx_seen"].to_i %> - Imported: <%= stats["tx_imported"].to_i %> - Updated: <%= stats["tx_updated"].to_i %> - Skipped: <%= stats["tx_skipped"].to_i %> -
-
-
-

Holdings

-
- Found: <%= stats["holdings_found"].to_i %> -
-
-
-

Health

-
-
- <% if stats["rate_limited"].present? || stats["rate_limited_at"].present? %> - <% ts = stats["rate_limited_at"] %> - <% ago = (ts.present? ? (begin; time_ago_in_words(Time.parse(ts)); rescue StandardError; nil; end) : nil) %> - Rate limited <%= ago ? "(#{ago} ago)" : "recently" %> - <% end %> - <% total_errors = stats["total_errors"].to_i %> - <% import_started = stats["import_started"].present? %> - <% if total_errors > 0 %> - Errors: <%= total_errors %> - <% elsif import_started %> - Errors: 0 - <% else %> - Errors: 0 - <% end %> -
-
-
-
-
- <% end %> + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: simplefin_item, + institutions_count: simplefin_item.connected_institutions.size + ) %> <%# Compute unlinked SimpleFin accounts (no legacy account and no AccountProvider link) # Prefer controller-provided map; fallback to a local query so the card stays accurate after Turbo broadcasts %> diff --git a/config/locales/views/components/en.yml b/config/locales/views/components/en.yml new file mode 100644 index 000000000..296d151d6 --- /dev/null +++ b/config/locales/views/components/en.yml @@ -0,0 +1,29 @@ +--- +en: + provider_sync_summary: + title: Sync summary + last_sync: "Last sync: %{time_ago} ago" + accounts: + title: Accounts + total: "Total: %{count}" + linked: "Linked: %{count}" + unlinked: "Unlinked: %{count}" + institutions: "Institutions: %{count}" + transactions: + title: Transactions + seen: "Seen: %{count}" + imported: "Imported: %{count}" + updated: "Updated: %{count}" + skipped: "Skipped: %{count}" + holdings: + title: Holdings + found: "Found: %{count}" + processed: "Processed: %{count}" + health: + title: Health + rate_limited: "Rate limited %{time_ago}" + recently: recently + errors: "Errors: %{count}" + data_warnings: "Data warnings: %{count}" + notices: "Notices: %{count}" + view_data_quality: View data quality details diff --git a/lib/tasks/dev_sync_stats.rake b/lib/tasks/dev_sync_stats.rake new file mode 100644 index 000000000..f7c353a7b --- /dev/null +++ b/lib/tasks/dev_sync_stats.rake @@ -0,0 +1,315 @@ +# frozen_string_literal: true + +# Helper module for sync stats rake tasks +module DevSyncStatsHelpers + extend self + + def generate_fake_stats_for_items(item_class, provider_name, include_issues: false) + items = item_class.all + if items.empty? + puts " No #{item_class.name} items found, skipping..." + return + end + + items.each do |item| + # Create or find a sync record + sync = item.syncs.ordered.first + if sync.nil? + sync = item.syncs.create!(status: :completed, completed_at: Time.current) + end + + stats = generate_fake_stats(provider_name, include_issues: include_issues) + sync.update!(sync_stats: stats, status: :completed, completed_at: Time.current) + + item_name = item.respond_to?(:name) ? item.name : item.try(:institution_name) || item.id + puts " Generated stats for #{item_class.name} ##{item.id} (#{item_name})" + end + end + + def generate_fake_stats(provider_name, include_issues: false) + # Base stats that all providers have + stats = { + "total_accounts" => rand(3..15), + "linked_accounts" => rand(2..10), + "unlinked_accounts" => rand(0..3), + "import_started" => true, + "window_start" => 1.hour.ago.iso8601, + "window_end" => Time.current.iso8601 + } + + # Ensure linked + unlinked <= total + stats["linked_accounts"] = [ stats["linked_accounts"], stats["total_accounts"] ].min + stats["unlinked_accounts"] = stats["total_accounts"] - stats["linked_accounts"] + + # Add transaction stats for most providers + unless provider_name == "coinstats" + stats.merge!( + "tx_seen" => rand(50..500), + "tx_imported" => rand(10..100), + "tx_updated" => rand(0..50), + "tx_skipped" => rand(0..5) + ) + # Ensure seen = imported + updated + stats["tx_seen"] = stats["tx_imported"] + stats["tx_updated"] + end + + # Add holdings stats for investment-capable providers + if %w[simplefin plaid coinstats].include?(provider_name) + stats["holdings_found"] = rand(5..50) + end + + # Add issues if requested + if include_issues + # Random chance of rate limiting + if rand < 0.3 + stats["rate_limited"] = true + stats["rate_limited_at"] = rand(1..24).hours.ago.iso8601 + end + + # Random errors + if rand < 0.4 + error_count = rand(1..3) + stats["errors"] = error_count.times.map do + { + "message" => [ + "Connection timeout", + "Invalid credentials", + "Rate limit exceeded", + "Temporary API error" + ].sample, + "category" => %w[api_error connection_error auth_error].sample + } + end + stats["total_errors"] = error_count + else + stats["total_errors"] = 0 + end + + # Data quality warnings + if rand < 0.5 + stats["data_warnings"] = rand(1..8) + stats["notices"] = rand(0..3) + stats["data_quality_details"] = stats["data_warnings"].times.map do |i| + start_date = rand(30..180).days.ago.to_date + end_date = start_date + rand(14..60).days + gap_days = (end_date - start_date).to_i + + { + "message" => "No transactions between #{start_date} and #{end_date} (#{gap_days} days)", + "severity" => gap_days > 30 ? "warning" : "info" + } + end + end + else + stats["total_errors"] = 0 + end + + stats + end +end + +namespace :dev do + namespace :sync_stats do + desc "Generate fake sync stats for testing the sync summary UI" + task generate: :environment do + unless Rails.env.development? + puts "This task is only available in development mode" + exit 1 + end + + puts "Generating fake sync stats for testing..." + + DevSyncStatsHelpers.generate_fake_stats_for_items(PlaidItem, "plaid") + DevSyncStatsHelpers.generate_fake_stats_for_items(SimplefinItem, "simplefin") + DevSyncStatsHelpers.generate_fake_stats_for_items(LunchflowItem, "lunchflow") + DevSyncStatsHelpers.generate_fake_stats_for_items(EnableBankingItem, "enable_banking") + DevSyncStatsHelpers.generate_fake_stats_for_items(CoinstatsItem, "coinstats") + + puts "Done! Refresh your browser to see the sync summaries." + end + + desc "Clear all sync stats from syncs" + task clear: :environment do + unless Rails.env.development? + puts "This task is only available in development mode" + exit 1 + end + + puts "Clearing all sync stats..." + Sync.where.not(sync_stats: nil).update_all(sync_stats: nil) + puts "Done!" + end + + desc "Generate fake sync stats with errors and warnings for testing" + task generate_with_issues: :environment do + unless Rails.env.development? + puts "This task is only available in development mode" + exit 1 + end + + puts "Generating fake sync stats with errors and warnings..." + + DevSyncStatsHelpers.generate_fake_stats_for_items(PlaidItem, "plaid", include_issues: true) + DevSyncStatsHelpers.generate_fake_stats_for_items(SimplefinItem, "simplefin", include_issues: true) + DevSyncStatsHelpers.generate_fake_stats_for_items(LunchflowItem, "lunchflow", include_issues: true) + DevSyncStatsHelpers.generate_fake_stats_for_items(EnableBankingItem, "enable_banking", include_issues: true) + DevSyncStatsHelpers.generate_fake_stats_for_items(CoinstatsItem, "coinstats", include_issues: true) + + puts "Done! Refresh your browser to see the sync summaries with issues." + end + + desc "Create fake provider items with sync stats for testing (use when you have no provider connections)" + task create_test_providers: :environment do + unless Rails.env.development? + puts "This task is only available in development mode" + exit 1 + end + + family = Family.first + unless family + puts "No family found. Please create a user account first." + exit 1 + end + + puts "Creating fake provider items for family: #{family.name || family.id}..." + + # Create a fake SimpleFIN item + simplefin_item = family.simplefin_items.create!( + name: "Test SimpleFIN Connection", + access_url: "https://test.simplefin.org/fake" + ) + puts " Created SimplefinItem: #{simplefin_item.name}" + + # Create fake SimpleFIN accounts + 3.times do |i| + simplefin_item.simplefin_accounts.create!( + name: "Test Account #{i + 1}", + account_id: "test-account-#{SecureRandom.hex(8)}", + currency: "USD", + current_balance: rand(1000..50000), + account_type: %w[checking savings credit_card].sample + ) + end + puts " Created 3 SimplefinAccounts" + + # Create a fake Plaid item (requires access_token) + plaid_item = family.plaid_items.create!( + name: "Test Plaid Connection", + access_token: "test-access-token-#{SecureRandom.hex(16)}", + plaid_id: "test-plaid-id-#{SecureRandom.hex(8)}" + ) + puts " Created PlaidItem: #{plaid_item.name}" + + # Create fake Plaid accounts + 2.times do |i| + plaid_item.plaid_accounts.create!( + name: "Test Plaid Account #{i + 1}", + plaid_id: "test-plaid-account-#{SecureRandom.hex(8)}", + currency: "USD", + current_balance: rand(1000..50000), + plaid_type: %w[depository credit investment].sample, + plaid_subtype: "checking" + ) + end + puts " Created 2 PlaidAccounts" + + # Create a fake Lunchflow item + lunchflow_item = family.lunchflow_items.create!( + name: "Test Lunchflow Connection", + api_key: "test-api-key-#{SecureRandom.hex(16)}" + ) + puts " Created LunchflowItem: #{lunchflow_item.name}" + + # Create fake Lunchflow accounts + 2.times do |i| + lunchflow_item.lunchflow_accounts.create!( + name: "Test Lunchflow Account #{i + 1}", + account_id: "test-lunchflow-#{SecureRandom.hex(8)}", + currency: "USD", + current_balance: rand(1000..50000) + ) + end + puts " Created 2 LunchflowAccounts" + + # Create a fake CoinStats item + coinstats_item = family.coinstats_items.create!( + name: "Test CoinStats Connection", + api_key: "test-coinstats-key-#{SecureRandom.hex(16)}", + institution_name: "CoinStats" + ) + puts " Created CoinstatsItem: #{coinstats_item.name}" + + # Create fake CoinStats accounts (wallets) + 3.times do |i| + coinstats_item.coinstats_accounts.create!( + name: "Test Wallet #{i + 1}", + account_id: "test-wallet-#{SecureRandom.hex(8)}", + currency: "USD", + current_balance: rand(100..10000), + account_type: %w[wallet exchange defi].sample + ) + end + puts " Created 3 CoinstatsAccounts" + + # Create a fake EnableBanking item + begin + enable_banking_item = family.enable_banking_items.create!( + name: "Test EnableBanking Connection", + institution_name: "Test Bank EU", + institution_id: "test-bank-#{SecureRandom.hex(8)}", + country_code: "DE", + aspsp_name: "Test Bank", + aspsp_id: "test-aspsp-#{SecureRandom.hex(8)}", + application_id: "test-app-#{SecureRandom.hex(8)}", + client_certificate: "-----BEGIN CERTIFICATE-----\nTEST_CERTIFICATE\n-----END CERTIFICATE-----" + ) + puts " Created EnableBankingItem: #{enable_banking_item.institution_name}" + + # Create fake EnableBanking accounts + 2.times do |i| + uid = "test-eb-uid-#{SecureRandom.hex(8)}" + enable_banking_item.enable_banking_accounts.create!( + name: "Test EU Account #{i + 1}", + uid: uid, + account_id: "test-eb-account-#{SecureRandom.hex(8)}", + currency: "EUR", + current_balance: rand(1000..50000), + iban: "DE#{rand(10..99)}#{SecureRandom.hex(10).upcase[0..17]}" + ) + end + puts " Created 2 EnableBankingAccounts" + rescue => e + puts " Failed to create EnableBankingItem: #{e.message}" + end + + puts "\nNow generating sync stats for the test providers..." + DevSyncStatsHelpers.generate_fake_stats_for_items(SimplefinItem, "simplefin", include_issues: true) + DevSyncStatsHelpers.generate_fake_stats_for_items(PlaidItem, "plaid", include_issues: false) + DevSyncStatsHelpers.generate_fake_stats_for_items(LunchflowItem, "lunchflow", include_issues: false) + DevSyncStatsHelpers.generate_fake_stats_for_items(CoinstatsItem, "coinstats", include_issues: true) + DevSyncStatsHelpers.generate_fake_stats_for_items(EnableBankingItem, "enable_banking", include_issues: false) + + puts "\nDone! Visit /accounts to see the sync summaries." + end + + desc "Remove all test provider items created by create_test_providers" + task remove_test_providers: :environment do + unless Rails.env.development? + puts "This task is only available in development mode" + exit 1 + end + + puts "Removing test provider items..." + + # Remove items that start with "Test " + count = 0 + count += SimplefinItem.where("name LIKE ?", "Test %").destroy_all.count + count += PlaidItem.where("name LIKE ?", "Test %").destroy_all.count + count += LunchflowItem.where("name LIKE ?", "Test %").destroy_all.count + count += CoinstatsItem.where("name LIKE ?", "Test %").destroy_all.count + count += EnableBankingItem.where("name LIKE ? OR institution_name LIKE ?", "Test %", "Test %").destroy_all.count + + puts "Removed #{count} test provider items. Done!" + end + end +end