diff --git a/.github/workflows/flutter-build.yml b/.github/workflows/flutter-build.yml new file mode 100644 index 000000000..886ae376f --- /dev/null +++ b/.github/workflows/flutter-build.yml @@ -0,0 +1,182 @@ +name: Flutter Mobile Build + +on: + push: + branches: + - main + tags: + - 'v*' + paths: + - 'mobile/lib/**' + - 'mobile/android/**' + - 'mobile/ios/**' + - 'mobile/pubspec.yaml' + - '.github/workflows/flutter-build.yml' + pull_request: + paths: + - 'mobile/lib/**' + - 'mobile/android/**' + - 'mobile/ios/**' + - 'mobile/pubspec.yaml' + - '.github/workflows/flutter-build.yml' + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-android: + name: Build Android APK + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Set up Flutter + uses: subosito/flutter-action@v2 + with: + flutter-version: '3.32.4' + channel: 'stable' + cache: true + + - name: Get dependencies + working-directory: mobile + run: flutter pub get + + - name: Generate app icons + working-directory: mobile + run: flutter pub run flutter_launcher_icons + + - name: Analyze code + working-directory: mobile + run: flutter analyze --no-fatal-infos + + - name: Run tests + working-directory: mobile + run: flutter test + + - name: Check if keystore secrets exist + id: check_secrets + run: | + if [ -n "${{ secrets.KEYSTORE_BASE64 }}" ]; then + echo "has_keystore=true" >> $GITHUB_OUTPUT + echo "✓ Keystore secrets found, will build signed APK" + else + echo "has_keystore=false" >> $GITHUB_OUTPUT + echo "⚠ No keystore secrets, will build unsigned debug APK" + fi + + - name: Decode and setup keystore + if: steps.check_secrets.outputs.has_keystore == 'true' + env: + KEYSTORE_BASE64: ${{ secrets.KEYSTORE_BASE64 }} + KEY_STORE_PASSWORD: ${{ secrets.KEY_STORE_PASSWORD }} + KEY_PASSWORD: ${{ secrets.KEY_PASSWORD }} + KEY_ALIAS: ${{ secrets.KEY_ALIAS }} + run: | + echo "$KEYSTORE_BASE64" | base64 -d > mobile/android/app/upload-keystore.jks + echo "storePassword=$KEY_STORE_PASSWORD" > mobile/android/key.properties + echo "keyPassword=$KEY_PASSWORD" >> mobile/android/key.properties + echo "keyAlias=$KEY_ALIAS" >> mobile/android/key.properties + echo "storeFile=upload-keystore.jks" >> mobile/android/key.properties + echo "✓ Keystore configured successfully" + + - name: Build APK (Release) + working-directory: mobile + run: | + if [ "${{ steps.check_secrets.outputs.has_keystore }}" == "true" ]; then + echo "Building signed release APK..." + flutter build apk --release + else + echo "Building debug-signed APK..." + flutter build apk --debug + fi + + - name: Upload APK artifact + uses: actions/upload-artifact@v4 + with: + name: app-release-apk + path: | + mobile/build/app/outputs/flutter-apk/app-release.apk + mobile/build/app/outputs/flutter-apk/app-debug.apk + retention-days: 30 + if-no-files-found: ignore + + - name: Build App Bundle (Release) + if: steps.check_secrets.outputs.has_keystore == 'true' + working-directory: mobile + run: flutter build appbundle --release + + - name: Upload AAB artifact + if: steps.check_secrets.outputs.has_keystore == 'true' + uses: actions/upload-artifact@v4 + with: + name: app-release-aab + path: mobile/build/app/outputs/bundle/release/app-release.aab + retention-days: 30 + + build-ios: + name: Build iOS IPA + runs-on: macos-latest + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Flutter + uses: subosito/flutter-action@v2 + with: + flutter-version: '3.32.4' + channel: 'stable' + cache: true + + - name: Get dependencies + working-directory: mobile + run: flutter pub get + + - name: Generate app icons + working-directory: mobile + run: flutter pub run flutter_launcher_icons + + - name: Install CocoaPods dependencies + working-directory: mobile/ios + run: pod install + + - name: Analyze code + working-directory: mobile + run: flutter analyze --no-fatal-infos + + - name: Run tests + working-directory: mobile + run: flutter test + + - name: Build iOS (No Code Signing) + working-directory: mobile + run: flutter build ios --release --no-codesign + + - name: Create IPA archive info + working-directory: mobile + run: | + echo "iOS build completed successfully" > build/ios-build-info.txt + echo "Build date: $(date)" >> build/ios-build-info.txt + echo "Note: This build is not code-signed and cannot be installed on physical devices" >> build/ios-build-info.txt + echo "For distribution, you need to configure code signing with Apple certificates" >> build/ios-build-info.txt + + - name: Upload iOS build artifact + uses: actions/upload-artifact@v4 + with: + name: ios-build-unsigned + path: | + mobile/build/ios/iphoneos/Runner.app + mobile/build/ios-build-info.txt + retention-days: 30 diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml index cb87ec6de..8c28a4928 100644 --- a/.github/workflows/helm-release.yaml +++ b/.github/workflows/helm-release.yaml @@ -22,9 +22,12 @@ jobs: fetch-depth: 0 - name: Configure Git + env: + GIT_USER_NAME: ${{ github.actor }} + GIT_USER_EMAIL: ${{ github.actor }}@users.noreply.github.com run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config user.name "$GIT_USER_NAME" + git config user.email "$GIT_USER_EMAIL" - name: Install Helm uses: azure/setup-helm@v3 @@ -64,18 +67,21 @@ jobs: path: gh-pages - name: Update index and push + env: + GIT_USER_NAME: ${{ github.actor }} + GIT_USER_EMAIL: ${{ github.actor }}@users.noreply.github.com run: | # Copy packaged chart cp .cr-release-packages/*.tgz gh-pages/ - + # Update index helm repo index gh-pages --url https://we-promise.github.io/sure --merge gh-pages/index.yaml # Push to gh-pages - git config --global user.email "sure-admin@sure.am" - git config --global user.name "sure-admin" git config --global credential.helper cache cd gh-pages + git config user.name "$GIT_USER_NAME" + git config user.email "$GIT_USER_EMAIL" git add . git commit -m "Release nightly: ${{ steps.version.outputs.version }}" git push diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 292296305..66ceff881 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,7 +39,8 @@ env: IMAGE_NAME: ${{ github.repository }} permissions: - contents: read + contents: write + packages: write jobs: ci: @@ -96,6 +97,7 @@ jobs: BASE_CONFIG+=$'\n'"type=raw,value=latest" else BASE_CONFIG+=$'\n'"type=raw,value=stable" + BASE_CONFIG+=$'\n'"type=raw,value=latest" fi fi fi @@ -238,3 +240,177 @@ jobs: echo "Push failed (attempt $attempts). Retrying in ${delay} seconds..." sleep ${delay} done + + mobile: + name: Build Mobile Apps + if: startsWith(github.ref, 'refs/tags/v') + uses: ./.github/workflows/flutter-build.yml + secrets: inherit + + release: + name: Create GitHub Release + if: startsWith(github.ref, 'refs/tags/v') + needs: [merge, mobile] + runs-on: ubuntu-latest + timeout-minutes: 10 + + permissions: + contents: write + + steps: + - name: Download Android APK artifact + uses: actions/download-artifact@v4.3.0 + with: + name: app-release-apk + path: ${{ runner.temp }}/mobile-artifacts + + - name: Download iOS build artifact + uses: actions/download-artifact@v4.3.0 + with: + name: ios-build-unsigned + path: ${{ runner.temp }}/ios-build + + - name: Prepare release assets + run: | + mkdir -p ${{ runner.temp }}/release-assets + + echo "=== Debugging: List downloaded artifacts ===" + echo "Mobile artifacts:" + ls -laR "${{ runner.temp }}/mobile-artifacts" || echo "No mobile-artifacts directory" + echo "iOS build:" + ls -laR "${{ runner.temp }}/ios-build" || echo "No ios-build directory" + echo "===========================================" + + # Copy debug APK if it exists + if [ -f "${{ runner.temp }}/mobile-artifacts/app-debug.apk" ]; then + cp "${{ runner.temp }}/mobile-artifacts/app-debug.apk" "${{ runner.temp }}/release-assets/sure-${{ github.ref_name }}-debug.apk" + echo "✓ Debug APK prepared" + fi + + # Copy release APK if it exists + if [ -f "${{ runner.temp }}/mobile-artifacts/app-release.apk" ]; then + cp "${{ runner.temp }}/mobile-artifacts/app-release.apk" "${{ runner.temp }}/release-assets/sure-${{ github.ref_name }}.apk" + echo "✓ Release APK prepared" + fi + + # Create iOS app archive (zip the .app bundle) + # Path preserves directory structure from artifact upload + if [ -d "${{ runner.temp }}/ios-build/ios/iphoneos/Runner.app" ]; then + cd "${{ runner.temp }}/ios-build/ios/iphoneos" + zip -r "${{ runner.temp }}/release-assets/sure-${{ github.ref_name }}-ios-unsigned.zip" Runner.app + echo "✓ iOS build archive prepared" + fi + + # Copy iOS build info + if [ -f "${{ runner.temp }}/ios-build/ios-build-info.txt" ]; then + cp "${{ runner.temp }}/ios-build/ios-build-info.txt" "${{ runner.temp }}/release-assets/" + fi + + echo "Release assets:" + ls -la "${{ runner.temp }}/release-assets/" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + name: ${{ github.ref_name }} + draft: false + prerelease: ${{ contains(github.ref_name, 'alpha') || contains(github.ref_name, 'beta') || contains(github.ref_name, 'rc') }} + generate_release_notes: true + files: | + ${{ runner.temp }}/release-assets/* + body: | + ## Mobile Debug Builds + + This release includes debug builds of the mobile applications: + + - **Android APK**: Debug build for testing on Android devices + - **iOS Build**: Unsigned iOS build (requires code signing for installation) + + > **Note**: These are debug builds intended for testing purposes. For production use, please build from source with proper signing credentials. + + bump-alpha-version: + name: Bump Alpha Version + if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, 'alpha') + needs: [merge] + runs-on: ubuntu-latest + timeout-minutes: 10 + + permissions: + contents: write + + steps: + - name: Check out main branch + uses: actions/checkout@v4.2.0 + with: + ref: main + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Bump alpha version + run: | + VERSION_FILE="config/initializers/version.rb" + + # Ensure version file exists + if [ ! -f "$VERSION_FILE" ]; then + echo "ERROR: Version file not found: $VERSION_FILE" + exit 1 + fi + + # Extract current version + CURRENT_VERSION=$(grep -oP '"\K[0-9]+\.[0-9]+\.[0-9]+-alpha\.[0-9]+' "$VERSION_FILE") + if [ -z "$CURRENT_VERSION" ]; then + echo "ERROR: Could not extract version from $VERSION_FILE" + exit 1 + fi + echo "Current version: $CURRENT_VERSION" + + # Extract the alpha number and increment it + ALPHA_NUM=$(echo "$CURRENT_VERSION" | grep -oP 'alpha\.\K[0-9]+') + if [ -z "$ALPHA_NUM" ]; then + echo "ERROR: Could not extract alpha number from $CURRENT_VERSION" + exit 1 + fi + NEW_ALPHA_NUM=$((ALPHA_NUM + 1)) + + # Create new version string + BASE_VERSION=$(echo "$CURRENT_VERSION" | grep -oP '^[0-9]+\.[0-9]+\.[0-9]+') + if [ -z "$BASE_VERSION" ]; then + echo "ERROR: Could not extract base version from $CURRENT_VERSION" + exit 1 + fi + NEW_VERSION="${BASE_VERSION}-alpha.${NEW_ALPHA_NUM}" + echo "New version: $NEW_VERSION" + + # Update the version file + sed -i "s/\"$CURRENT_VERSION\"/\"$NEW_VERSION\"/" "$VERSION_FILE" + + # Verify the change + echo "Updated version.rb:" + + - name: Commit and push version bump + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + git add config/initializers/version.rb + + # Check if there are changes to commit + if git diff --cached --quiet; then + echo "No changes to commit - version may have already been bumped" + exit 0 + + git commit -m "Bump version to next alpha after ${{ github.ref_name }} release" + + # Push with retry logic + attempts=0 + until git push origin main; do + attempts=$((attempts + 1)) + if [[ $attempts -ge 4 ]]; then + echo "ERROR: Failed to push after 4 attempts." >&2 + exit 1 + fi + delay=$((2 ** attempts)) + echo "Push failed (attempt $attempts). Retrying in ${delay} seconds..." + sleep ${delay} + git pull --rebase origin main + done diff --git a/Gemfile b/Gemfile index 0658791a6..957a56449 100644 --- a/Gemfile +++ b/Gemfile @@ -17,7 +17,7 @@ gem "bootsnap", require: false gem "importmap-rails" gem "propshaft" gem "tailwindcss-rails" -gem "lucide-rails", github: "maybe-finance/lucide-rails" +gem "lucide-rails" # Hotwire + UI gem "stimulus-rails" diff --git a/Gemfile.lock b/Gemfile.lock index d251bef9a..190ef7382 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,10 +1,3 @@ -GIT - remote: https://github.com/maybe-finance/lucide-rails.git - revision: 272e5fb8418ea458da3995d6abe0ba0ceee9c9f0 - specs: - lucide-rails (0.2.0) - railties (>= 4.1.0) - GEM remote: https://rubygems.org/ specs: @@ -121,7 +114,7 @@ GEM erubi (~> 1.4) parser (>= 2.4) smart_properties - bigdecimal (3.2.2) + bigdecimal (3.3.1) bindata (2.5.1) bindex (0.8.1) bootsnap (1.18.6) @@ -254,7 +247,7 @@ GEM turbo-rails (>= 1.2) htmlbeautifier (1.4.3) htmlentities (4.3.4) - httparty (0.23.1) + httparty (0.24.0) csv mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) @@ -346,6 +339,8 @@ GEM view_component (>= 2.0) yard (~> 0.9) zeitwerk (~> 2.5) + lucide-rails (0.7.3) + railties (>= 4.1.0) mail (2.8.1) mini_mime (>= 0.1.1) net-imap @@ -364,8 +359,8 @@ GEM mocha (2.7.1) ruby2_keywords (>= 0.0.5) msgpack (1.8.0) - multi_xml (0.7.2) - bigdecimal (~> 3.1) + multi_xml (0.8.0) + bigdecimal (>= 3.1, < 5) multipart-post (2.4.1) mutex_m (0.3.0) net-http (0.6.0) @@ -719,7 +714,7 @@ GEM unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) - uri (1.0.3) + uri (1.0.4) useragent (0.16.11) validate_url (1.0.15) activemodel (>= 3.0.0) @@ -803,7 +798,7 @@ DEPENDENCIES letter_opener logtail-rails lookbook (= 2.3.11) - lucide-rails! + lucide-rails mocha octokit omniauth (~> 2.1) diff --git a/app/components/DS/menu_controller.js b/app/components/DS/menu_controller.js index 1dc01db3d..512358f6c 100644 --- a/app/components/DS/menu_controller.js +++ b/app/components/DS/menu_controller.js @@ -80,7 +80,7 @@ export default class extends Controller { const firstFocusableElement = this.contentTarget.querySelectorAll(focusableElements)[0]; if (firstFocusableElement) { - firstFocusableElement.focus(); + firstFocusableElement.focus({ preventScroll: true }); } } diff --git a/app/components/provider_sync_summary.html.erb b/app/components/provider_sync_summary.html.erb new file mode 100644 index 000000000..488a86004 --- /dev/null +++ b/app/components/provider_sync_summary.html.erb @@ -0,0 +1,105 @@ +
+ +
+ <%= helpers.icon "chevron-right", class: "group-open:transform group-open:rotate-90" %> + <%= t("provider_sync_summary.title") %> +
+
+ <% if last_synced_at %> + <%= t("provider_sync_summary.last_sync", time_ago: last_synced_ago) %> + <% end %> +
+
+ +
+ <%# Accounts section - always shown if we have account stats %> + <% if total_accounts > 0 || stats.key?("total_accounts") %> +
+

<%= t("provider_sync_summary.accounts.title") %>

+
+ <%= t("provider_sync_summary.accounts.total", count: total_accounts) %> + <%= t("provider_sync_summary.accounts.linked", count: linked_accounts) %> + <%= t("provider_sync_summary.accounts.unlinked", count: unlinked_accounts) %> + <% if institutions_count.present? %> + <%= t("provider_sync_summary.accounts.institutions", count: institutions_count) %> + <% end %> +
+
+ <% end %> + + <%# Transactions section - shown if provider collects transaction stats %> + <% if has_transaction_stats? %> +
+

<%= t("provider_sync_summary.transactions.title") %>

+
+ <%= t("provider_sync_summary.transactions.seen", count: tx_seen) %> + <%= t("provider_sync_summary.transactions.imported", count: tx_imported) %> + <%= t("provider_sync_summary.transactions.updated", count: tx_updated) %> + <%= t("provider_sync_summary.transactions.skipped", count: tx_skipped) %> +
+
+ <% end %> + + <%# Holdings section - shown if provider collects holdings stats %> + <% if has_holdings_stats? %> +
+

<%= t("provider_sync_summary.holdings.title") %>

+
+ <%= t("provider_sync_summary.holdings.#{holdings_label_key}", count: holdings_count) %> +
+
+ <% end %> + + <%# Health section - always shown %> +
+

<%= t("provider_sync_summary.health.title") %>

+
+
+ <% if rate_limited? %> + + <%= t("provider_sync_summary.health.rate_limited", time_ago: rate_limited_ago || t("provider_sync_summary.health.recently")) %> + + <% end %> + <% if has_errors? %> + <%= t("provider_sync_summary.health.errors", count: total_errors) %> + <% elsif import_started? %> + <%= t("provider_sync_summary.health.errors", count: 0) %> + <% else %> + <%= t("provider_sync_summary.health.errors", count: 0) %> + <% end %> +
+ + <%# Data quality warnings %> + <% if has_data_quality_issues? %> +
+ <% if data_warnings > 0 %> +
+ <%= helpers.icon "alert-triangle", size: "sm", color: "warning" %> + <%= t("provider_sync_summary.health.data_warnings", count: data_warnings) %> +
+ <% end %> + <% if notices > 0 %> +
+ <%= helpers.icon "info", size: "sm" %> + <%= t("provider_sync_summary.health.notices", count: notices) %> +
+ <% end %> +
+ + <% if data_quality_details.any? %> +
+ + <%= t("provider_sync_summary.health.view_data_quality") %> + +
+ <% data_quality_details.each do |detail| %> +

"><%= detail["message"] %>

+ <% end %> +
+
+ <% end %> + <% end %> +
+
+
+
diff --git a/app/components/provider_sync_summary.rb b/app/components/provider_sync_summary.rb new file mode 100644 index 000000000..4d00f2343 --- /dev/null +++ b/app/components/provider_sync_summary.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +# Reusable sync summary component for provider items. +# +# This component displays sync statistics in a collapsible panel that can be used +# by any provider (SimpleFIN, Plaid, Lunchflow, etc.) to show their sync results. +# +# @example Basic usage +# <%= render ProviderSyncSummary.new( +# stats: @sync_stats, +# provider_item: @plaid_item +# ) %> +# +# @example With custom institution count +# <%= render ProviderSyncSummary.new( +# stats: @sync_stats, +# provider_item: @simplefin_item, +# institutions_count: @simplefin_item.connected_institutions.size +# ) %> +# +class ProviderSyncSummary < ViewComponent::Base + attr_reader :stats, :provider_item, :institutions_count + + # @param stats [Hash] The sync statistics hash from sync.sync_stats + # @param provider_item [Object] The provider item (must respond to last_synced_at) + # @param institutions_count [Integer, nil] Optional count of connected institutions + def initialize(stats:, provider_item:, institutions_count: nil) + @stats = stats || {} + @provider_item = provider_item + @institutions_count = institutions_count + end + + def render? + stats.present? + end + + # Account statistics + def total_accounts + stats["total_accounts"].to_i + end + + def linked_accounts + stats["linked_accounts"].to_i + end + + def unlinked_accounts + stats["unlinked_accounts"].to_i + end + + # Transaction statistics + def tx_seen + stats["tx_seen"].to_i + end + + def tx_imported + stats["tx_imported"].to_i + end + + def tx_updated + stats["tx_updated"].to_i + end + + def tx_skipped + stats["tx_skipped"].to_i + end + + def has_transaction_stats? + stats.key?("tx_seen") || stats.key?("tx_imported") || stats.key?("tx_updated") + end + + # Holdings statistics + def holdings_found + stats["holdings_found"].to_i + end + + def holdings_processed + stats["holdings_processed"].to_i + end + + def has_holdings_stats? + stats.key?("holdings_found") || stats.key?("holdings_processed") + end + + def holdings_label_key + stats.key?("holdings_processed") ? "processed" : "found" + end + + def holdings_count + stats.key?("holdings_processed") ? holdings_processed : holdings_found + end + + # Returns the CSS color class for a data quality detail severity + # @param severity [String] The severity level ("warning", "error", or other) + # @return [String] The Tailwind CSS class for the color + def severity_color_class(severity) + case severity + when "warning" then "text-warning" + when "error" then "text-destructive" + else "text-secondary" + end + end + + # Health statistics + def rate_limited? + stats["rate_limited"].present? || stats["rate_limited_at"].present? + end + + def rate_limited_ago + return nil unless stats["rate_limited_at"].present? + + begin + helpers.time_ago_in_words(Time.parse(stats["rate_limited_at"])) + rescue StandardError + nil + end + end + + def total_errors + stats["total_errors"].to_i + end + + def import_started? + stats["import_started"].present? + end + + def has_errors? + total_errors > 0 + end + + # Data quality / warnings + def data_warnings + stats["data_warnings"].to_i + end + + def notices + stats["notices"].to_i + end + + def data_quality_details + stats["data_quality_details"] || [] + end + + def has_data_quality_issues? + data_warnings > 0 || notices > 0 || data_quality_details.any? + end + + # Last sync time + def last_synced_at + provider_item.last_synced_at + end + + def last_synced_ago + return nil unless last_synced_at + + helpers.time_ago_in_words(last_synced_at) + end +end diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 4940ea69c..48e484f40 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -6,50 +6,14 @@ class AccountsController < ApplicationController @manual_accounts = family.accounts .listable_manual .order(:name) - @plaid_items = family.plaid_items.ordered + @plaid_items = family.plaid_items.ordered.includes(:syncs, :plaid_accounts) @simplefin_items = family.simplefin_items.ordered.includes(:syncs) - @lunchflow_items = family.lunchflow_items.ordered + @lunchflow_items = family.lunchflow_items.ordered.includes(:syncs, :lunchflow_accounts) @enable_banking_items = family.enable_banking_items.ordered.includes(:syncs) + @coinstats_items = family.coinstats_items.ordered.includes(:coinstats_accounts, :accounts, :syncs) - # Precompute per-item maps to avoid queries in the view - @simplefin_sync_stats_map = {} - @simplefin_has_unlinked_map = {} - - @simplefin_items.each do |item| - latest_sync = item.syncs.ordered.first - @simplefin_sync_stats_map[item.id] = (latest_sync&.sync_stats || {}) - @simplefin_has_unlinked_map[item.id] = item.family.accounts - .listable_manual - .exists? - end - - # Count of SimpleFin accounts that are not linked (no legacy account and no AccountProvider) - @simplefin_unlinked_count_map = {} - @simplefin_items.each do |item| - count = item.simplefin_accounts - .left_joins(:account, :account_provider) - .where(accounts: { id: nil }, account_providers: { id: nil }) - .count - @simplefin_unlinked_count_map[item.id] = count - end - - # Compute CTA visibility map used by the simplefin_item partial - @simplefin_show_relink_map = {} - @simplefin_items.each do |item| - begin - unlinked_count = @simplefin_unlinked_count_map[item.id] || 0 - manuals_exist = @simplefin_has_unlinked_map[item.id] - sfa_any = if item.simplefin_accounts.loaded? - item.simplefin_accounts.any? - else - item.simplefin_accounts.exists? - end - @simplefin_show_relink_map[item.id] = (unlinked_count.to_i == 0 && manuals_exist && sfa_any) - rescue => e - Rails.logger.warn("SimpleFin card: CTA computation failed for item #{item.id}: #{e.class} - #{e.message}") - @simplefin_show_relink_map[item.id] = false - end - end + # Build sync stats maps for all providers + build_sync_stats_maps # Prevent Turbo Drive from caching this page to ensure fresh account lists expires_now @@ -140,11 +104,27 @@ class AccountsController < ApplicationController begin Account.transaction do + # Detach holdings from provider links before destroying them + provider_link_ids = @account.account_providers.pluck(:id) + if provider_link_ids.any? + Holding.where(account_provider_id: provider_link_ids).update_all(account_provider_id: nil) + end + + # Capture SimplefinAccount before clearing FK (so we can destroy it) + simplefin_account_to_destroy = @account.simplefin_account + # Remove new system links (account_providers join table) @account.account_providers.destroy_all # Remove legacy system links (foreign keys) @account.update!(plaid_account_id: nil, simplefin_account_id: nil) + + # Destroy the SimplefinAccount record so it doesn't cause stale account issues + # This is safe because: + # - Account data (transactions, holdings, balances) lives on the Account, not SimplefinAccount + # - SimplefinAccount only caches API data which is regenerated on reconnect + # - If user reconnects SimpleFin later, a new SimplefinAccount will be created + simplefin_account_to_destroy&.destroy! end redirect_to accounts_path, notice: t("accounts.unlink.success") @@ -193,4 +173,70 @@ class AccountsController < ApplicationController def set_account @account = family.accounts.find(params[:id]) end + + # Builds sync stats maps for all provider types to avoid N+1 queries in views + def build_sync_stats_maps + # SimpleFIN sync stats + @simplefin_sync_stats_map = {} + @simplefin_has_unlinked_map = {} + @simplefin_unlinked_count_map = {} + @simplefin_show_relink_map = {} + @simplefin_duplicate_only_map = {} + + @simplefin_items.each do |item| + latest_sync = item.syncs.ordered.first + stats = latest_sync&.sync_stats || {} + @simplefin_sync_stats_map[item.id] = stats + @simplefin_has_unlinked_map[item.id] = item.family.accounts.listable_manual.exists? + + # Count unlinked accounts + count = item.simplefin_accounts + .left_joins(:account, :account_provider) + .where(accounts: { id: nil }, account_providers: { id: nil }) + .count + @simplefin_unlinked_count_map[item.id] = count + + # CTA visibility + manuals_exist = @simplefin_has_unlinked_map[item.id] + sfa_any = item.simplefin_accounts.loaded? ? item.simplefin_accounts.any? : item.simplefin_accounts.exists? + @simplefin_show_relink_map[item.id] = (count.to_i == 0 && manuals_exist && sfa_any) + + # Check if all errors are duplicate-skips + errors = Array(stats["errors"]).map { |e| e.is_a?(Hash) ? e["message"] || e[:message] : e.to_s } + @simplefin_duplicate_only_map[item.id] = errors.present? && errors.all? { |m| m.to_s.downcase.include?("duplicate upstream account detected") } + rescue => e + Rails.logger.warn("SimpleFin stats map build failed for item #{item.id}: #{e.class} - #{e.message}") + @simplefin_sync_stats_map[item.id] = {} + @simplefin_show_relink_map[item.id] = false + @simplefin_duplicate_only_map[item.id] = false + end + + # Plaid sync stats + @plaid_sync_stats_map = {} + @plaid_items.each do |item| + latest_sync = item.syncs.ordered.first + @plaid_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + + # Lunchflow sync stats + @lunchflow_sync_stats_map = {} + @lunchflow_items.each do |item| + latest_sync = item.syncs.ordered.first + @lunchflow_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + + # Enable Banking sync stats + @enable_banking_sync_stats_map = {} + @enable_banking_items.each do |item| + latest_sync = item.syncs.ordered.first + @enable_banking_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + + # CoinStats sync stats + @coinstats_sync_stats_map = {} + @coinstats_items.each do |item| + latest_sync = item.syncs.ordered.first + @coinstats_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + end + end end diff --git a/app/controllers/api/v1/imports_controller.rb b/app/controllers/api/v1/imports_controller.rb new file mode 100644 index 000000000..2b6a5a5af --- /dev/null +++ b/app/controllers/api/v1/imports_controller.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +class Api::V1::ImportsController < Api::V1::BaseController + include Pagy::Backend + + # Ensure proper scope authorization + before_action :ensure_read_scope, only: [ :index, :show ] + before_action :ensure_write_scope, only: [ :create ] + before_action :set_import, only: [ :show ] + + def index + family = current_resource_owner.family + imports_query = family.imports.ordered + + # Apply filters + if params[:status].present? + imports_query = imports_query.where(status: params[:status]) + end + + if params[:type].present? + imports_query = imports_query.where(type: params[:type]) + end + + # Pagination + @pagy, @imports = pagy( + imports_query, + page: safe_page_param, + limit: safe_per_page_param + ) + + @per_page = safe_per_page_param + + render :index + + rescue StandardError => e + Rails.logger.error "ImportsController#index error: #{e.message}" + render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error + end + + def show + render :show + rescue StandardError => e + Rails.logger.error "ImportsController#show error: #{e.message}" + render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error + end + + def create + family = current_resource_owner.family + + # 1. Determine type and validate + type = params[:type].to_s + type = "TransactionImport" unless Import::TYPES.include?(type) + + # 2. Build the import object with permitted config attributes + @import = family.imports.build(import_config_params) + @import.type = type + @import.account_id = params[:account_id] if params[:account_id].present? + + # 3. Attach the uploaded file if present (with validation) + if params[:file].present? + file = params[:file] + + if file.size > Import::MAX_CSV_SIZE + return render json: { + error: "file_too_large", + message: "File is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + }, status: :unprocessable_entity + end + + unless Import::ALLOWED_MIME_TYPES.include?(file.content_type) + return render json: { + error: "invalid_file_type", + message: "Invalid file type. Please upload a CSV file." + }, status: :unprocessable_entity + end + + @import.raw_file_str = file.read + elsif params[:raw_file_content].present? + if params[:raw_file_content].bytesize > Import::MAX_CSV_SIZE + return render json: { + error: "content_too_large", + message: "Content is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + }, status: :unprocessable_entity + end + + @import.raw_file_str = params[:raw_file_content] + end + + # 4. Save and Process + if @import.save + # Generate rows if file content was provided + if @import.uploaded? + begin + @import.generate_rows_from_csv + @import.reload + rescue StandardError => e + Rails.logger.error "Row generation failed for import #{@import.id}: #{e.message}" + end + end + + # If the import is configured (has rows), we can try to auto-publish or just leave it as pending + # For API simplicity, if enough info is provided, we might want to trigger processing + + if @import.configured? && params[:publish] == "true" + @import.publish_later + end + + render :show, status: :created + else + render json: { + error: "validation_failed", + message: "Import could not be created", + errors: @import.errors.full_messages + }, status: :unprocessable_entity + end + + rescue StandardError => e + Rails.logger.error "ImportsController#create error: #{e.message}" + render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error + end + + private + + def set_import + @import = current_resource_owner.family.imports.includes(:rows).find(params[:id]) + rescue ActiveRecord::RecordNotFound + render json: { error: "not_found", message: "Import not found" }, status: :not_found + end + + def ensure_read_scope + authorize_scope!(:read) + end + + def ensure_write_scope + authorize_scope!(:write) + end + + def import_config_params + params.permit( + :date_col_label, + :amount_col_label, + :name_col_label, + :category_col_label, + :tags_col_label, + :notes_col_label, + :account_col_label, + :qty_col_label, + :ticker_col_label, + :price_col_label, + :entity_type_col_label, + :currency_col_label, + :exchange_operating_mic_col_label, + :date_format, + :number_format, + :signage_convention, + :col_sep, + :amount_type_strategy, + :amount_type_inflow_value + ) + end + + def safe_page_param + page = params[:page].to_i + page > 0 ? page : 1 + end + + def safe_per_page_param + per_page = params[:per_page].to_i + (1..100).include?(per_page) ? per_page : 25 + end +end diff --git a/app/controllers/coinstats_items_controller.rb b/app/controllers/coinstats_items_controller.rb new file mode 100644 index 000000000..9e47d8f1a --- /dev/null +++ b/app/controllers/coinstats_items_controller.rb @@ -0,0 +1,169 @@ +class CoinstatsItemsController < ApplicationController + before_action :set_coinstats_item, only: [ :show, :edit, :update, :destroy, :sync ] + + def index + @coinstats_items = Current.family.coinstats_items.ordered + end + + def show + end + + def new + @coinstats_item = Current.family.coinstats_items.build + @coinstats_items = Current.family.coinstats_items.where.not(api_key: nil) + @blockchains = fetch_blockchain_options(@coinstats_items.first) + end + + def create + @coinstats_item = Current.family.coinstats_items.build(coinstats_item_params) + @coinstats_item.name ||= t(".default_name") + + # Validate API key before saving + unless validate_api_key(@coinstats_item.api_key) + return render_error_response(@coinstats_item.errors.full_messages.join(", ")) + end + + if @coinstats_item.save + render_success_response(".success") + else + render_error_response(@coinstats_item.errors.full_messages.join(", ")) + end + end + + def edit + end + + def update + # Validate API key if it's being changed + unless validate_api_key(coinstats_item_params[:api_key]) + return render_error_response(@coinstats_item.errors.full_messages.join(", ")) + end + + if @coinstats_item.update(coinstats_item_params) + render_success_response(".success") + else + render_error_response(@coinstats_item.errors.full_messages.join(", ")) + end + end + + def destroy + @coinstats_item.destroy_later + redirect_to settings_providers_path, notice: t(".success"), status: :see_other + end + + def sync + unless @coinstats_item.syncing? + @coinstats_item.sync_later + end + + respond_to do |format| + format.html { redirect_back_or_to accounts_path } + format.json { head :ok } + end + end + + def link_wallet + coinstats_item_id = params[:coinstats_item_id].presence + @address = params[:address]&.to_s&.strip.presence + @blockchain = params[:blockchain]&.to_s&.strip.presence + + unless coinstats_item_id && @address && @blockchain + return render_link_wallet_error(t(".missing_params")) + end + + @coinstats_item = Current.family.coinstats_items.find(coinstats_item_id) + + result = CoinstatsItem::WalletLinker.new(@coinstats_item, address: @address, blockchain: @blockchain).link + + if result.success? + redirect_to accounts_path, notice: t(".success", count: result.created_count), status: :see_other + else + error_msg = result.errors.join("; ").presence || t(".failed") + render_link_wallet_error(error_msg) + end + rescue Provider::Coinstats::Error => e + render_link_wallet_error(t(".error", message: e.message)) + rescue => e + Rails.logger.error("CoinStats link wallet error: #{e.class} - #{e.message}") + render_link_wallet_error(t(".error", message: e.message)) + end + + private + + def set_coinstats_item + @coinstats_item = Current.family.coinstats_items.find(params[:id]) + end + + def coinstats_item_params + params.require(:coinstats_item).permit( + :name, + :sync_start_date, + :api_key + ) + end + + def validate_api_key(api_key) + return true if api_key.blank? + + response = Provider::Coinstats.new(api_key).get_blockchains + if response.success? + true + else + @coinstats_item.errors.add(:api_key, t("coinstats_items.create.errors.validation_failed", message: response.error&.message)) + false + end + rescue => e + @coinstats_item.errors.add(:api_key, t("coinstats_items.create.errors.validation_failed", message: e.message)) + false + end + + def render_error_response(error_message) + if turbo_frame_request? + render turbo_stream: turbo_stream.replace( + "coinstats-providers-panel", + partial: "settings/providers/coinstats_panel", + locals: { error_message: error_message } + ), status: :unprocessable_entity + else + redirect_to settings_providers_path, alert: error_message, status: :unprocessable_entity + end + end + + def render_success_response(notice_key) + if turbo_frame_request? + flash.now[:notice] = t(notice_key, default: notice_key.to_s.humanize) + @coinstats_items = Current.family.coinstats_items.ordered + render turbo_stream: [ + turbo_stream.replace( + "coinstats-providers-panel", + partial: "settings/providers/coinstats_panel", + locals: { coinstats_items: @coinstats_items } + ), + *flash_notification_stream_items + ] + else + redirect_to settings_providers_path, notice: t(notice_key), status: :see_other + end + end + + def render_link_wallet_error(error_message) + @error_message = error_message + @coinstats_items = Current.family.coinstats_items.where.not(api_key: nil) + @blockchains = fetch_blockchain_options(@coinstats_items.first) + render :new, status: :unprocessable_entity + end + + def fetch_blockchain_options(coinstats_item) + return [] unless coinstats_item&.api_key.present? + + Provider::Coinstats.new(coinstats_item.api_key).blockchain_options + rescue Provider::Coinstats::Error => e + Rails.logger.error("CoinStats blockchain fetch failed: item_id=#{coinstats_item.id} error=#{e.class} message=#{e.message}") + flash.now[:alert] = t("coinstats_items.new.blockchain_fetch_error") + [] + rescue StandardError => e + Rails.logger.error("CoinStats blockchain fetch failed: item_id=#{coinstats_item.id} error=#{e.class} message=#{e.message}") + flash.now[:alert] = t("coinstats_items.new.blockchain_fetch_error") + [] + end +end diff --git a/app/controllers/enable_banking_items_controller.rb b/app/controllers/enable_banking_items_controller.rb index ae69543e7..d2e341387 100644 --- a/app/controllers/enable_banking_items_controller.rb +++ b/app/controllers/enable_banking_items_controller.rb @@ -269,13 +269,17 @@ class EnableBankingItemsController < ApplicationController end # Create the internal Account (uses save! internally, will raise on failure) + # Skip initial sync - provider sync will handle balance creation with correct currency account = Account.create_and_sync( - family: Current.family, - name: enable_banking_account.name, - balance: enable_banking_account.current_balance || 0, - currency: enable_banking_account.currency || "EUR", - accountable_type: accountable_type, - accountable_attributes: {} + { + family: Current.family, + name: enable_banking_account.name, + balance: enable_banking_account.current_balance || 0, + currency: enable_banking_account.currency || "EUR", + accountable_type: accountable_type, + accountable_attributes: {} + }, + skip_initial_sync: true ) # Link account to enable_banking_account via account_providers diff --git a/app/controllers/family_merchants_controller.rb b/app/controllers/family_merchants_controller.rb index e06e623ea..4d41db132 100644 --- a/app/controllers/family_merchants_controller.rb +++ b/app/controllers/family_merchants_controller.rb @@ -8,6 +8,15 @@ class FamilyMerchantsController < ApplicationController @family_merchants = Current.family.merchants.alphabetically @provider_merchants = Current.family.assigned_merchants.where(type: "ProviderMerchant").alphabetically + # Show recently unlinked ProviderMerchants (within last 30 days) + # Exclude merchants that are already assigned to transactions (they appear in provider_merchants) + recently_unlinked_ids = FamilyMerchantAssociation + .where(family: Current.family) + .recently_unlinked + .pluck(:merchant_id) + assigned_ids = @provider_merchants.pluck(:id) + @unlinked_merchants = ProviderMerchant.where(id: recently_unlinked_ids - assigned_ids).alphabetically + render layout: "settings" end @@ -32,24 +41,90 @@ class FamilyMerchantsController < ApplicationController end def update - @family_merchant.update!(merchant_params) - respond_to do |format| - format.html { redirect_to family_merchants_path, notice: t(".success") } - format.turbo_stream { render turbo_stream: turbo_stream.action(:redirect, family_merchants_path) } + if @merchant.is_a?(ProviderMerchant) + # Convert ProviderMerchant to FamilyMerchant for this family only + @family_merchant = @merchant.convert_to_family_merchant_for(Current.family, merchant_params) + respond_to do |format| + format.html { redirect_to family_merchants_path, notice: t(".converted_success") } + format.turbo_stream { render turbo_stream: turbo_stream.action(:redirect, family_merchants_path) } + end + elsif @merchant.update(merchant_params) + respond_to do |format| + format.html { redirect_to family_merchants_path, notice: t(".success") } + format.turbo_stream { render turbo_stream: turbo_stream.action(:redirect, family_merchants_path) } + end + else + render :edit, status: :unprocessable_entity end + rescue ActiveRecord::RecordInvalid => e + @family_merchant = e.record + render :edit, status: :unprocessable_entity end def destroy - @family_merchant.destroy! - redirect_to family_merchants_path, notice: t(".success") + if @merchant.is_a?(ProviderMerchant) + # Unlink from family's transactions only (don't delete the global merchant) + @merchant.unlink_from_family(Current.family) + redirect_to family_merchants_path, notice: t(".unlinked_success") + else + @merchant.destroy! + redirect_to family_merchants_path, notice: t(".success") + end + end + + def merge + @merchants = all_family_merchants + end + + def perform_merge + # Scope lookups to merchants valid for this family (FamilyMerchants + assigned ProviderMerchants) + valid_merchants = all_family_merchants + + target = valid_merchants.find_by(id: params[:target_id]) + unless target + return redirect_to merge_family_merchants_path, alert: t(".target_not_found") + end + + sources = valid_merchants.where(id: params[:source_ids]) + unless sources.any? + return redirect_to merge_family_merchants_path, alert: t(".invalid_merchants") + end + + merger = Merchant::Merger.new( + family: Current.family, + target_merchant: target, + source_merchants: sources + ) + + if merger.merge! + redirect_to family_merchants_path, notice: t(".success", count: merger.merged_count) + else + redirect_to merge_family_merchants_path, alert: t(".no_merchants_selected") + end + rescue Merchant::Merger::UnauthorizedMerchantError => e + redirect_to merge_family_merchants_path, alert: e.message end private def set_merchant - @family_merchant = Current.family.merchants.find(params[:id]) + # Find merchant that either belongs to family OR is assigned to family's transactions + @merchant = Current.family.merchants.find_by(id: params[:id]) || + Current.family.assigned_merchants.find(params[:id]) + @family_merchant = @merchant # For backwards compatibility with views end def merchant_params - params.require(:family_merchant).permit(:name, :color) + # Handle both family_merchant and provider_merchant param keys + key = params.key?(:family_merchant) ? :family_merchant : :provider_merchant + params.require(key).permit(:name, :color) + end + + def all_family_merchants + family_merchant_ids = Current.family.merchants.pluck(:id) + provider_merchant_ids = Current.family.assigned_merchants.where(type: "ProviderMerchant").pluck(:id) + combined_ids = (family_merchant_ids + provider_merchant_ids).uniq + + Merchant.where(id: combined_ids) + .order(Arel.sql("LOWER(COALESCE(name, ''))")) end end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 7b6040743..50891d323 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -26,14 +26,38 @@ class ImportsController < ApplicationController end def create + type = params.dig(:import, :type).to_s + type = "TransactionImport" unless Import::TYPES.include?(type) + account = Current.family.accounts.find_by(id: params.dig(:import, :account_id)) import = Current.family.imports.create!( - type: import_params[:type], + type: type, account: account, date_format: Current.family.date_format, ) - redirect_to import_upload_path(import) + if import_params[:csv_file].present? + file = import_params[:csv_file] + + if file.size > Import::MAX_CSV_SIZE + import.destroy + redirect_to new_import_path, alert: "File is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + return + end + + unless Import::ALLOWED_MIME_TYPES.include?(file.content_type) + import.destroy + redirect_to new_import_path, alert: "Invalid file type. Please upload a CSV file." + return + end + + # Stream reading is not fully applicable here as we store the raw string in the DB, + # but we have validated size beforehand to prevent memory exhaustion from massive files. + import.update!(raw_file_str: file.read) + redirect_to import_configuration_path(import), notice: "CSV uploaded successfully." + else + redirect_to import_upload_path(import) + end end def show @@ -70,6 +94,6 @@ class ImportsController < ApplicationController end def import_params - params.require(:import).permit(:type) + params.require(:import).permit(:csv_file) end end diff --git a/app/controllers/lunchflow_items_controller.rb b/app/controllers/lunchflow_items_controller.rb index c16cd20ef..2208090e7 100644 --- a/app/controllers/lunchflow_items_controller.rb +++ b/app/controllers/lunchflow_items_controller.rb @@ -182,13 +182,18 @@ class LunchflowItemsController < ApplicationController end # Create the internal Account with proper balance initialization + # Use lunchflow_account.currency (already parsed) and skip initial sync + # because the provider sync will set the correct currency from the balance API account = Account.create_and_sync( - family: Current.family, - name: account_data[:name], - balance: 0, # Initial balance will be set during sync - currency: account_data[:currency] || "USD", - accountable_type: accountable_type, - accountable_attributes: {} + { + family: Current.family, + name: account_data[:name], + balance: 0, # Initial balance will be set during sync + currency: lunchflow_account.currency || "USD", + accountable_type: accountable_type, + accountable_attributes: {} + }, + skip_initial_sync: true ) # Link account to lunchflow_account via account_providers join table @@ -605,13 +610,17 @@ class LunchflowItemsController < ApplicationController selected_subtype = "credit_card" if selected_type == "CreditCard" && selected_subtype.blank? # Create account with user-selected type and subtype (raises on failure) + # Skip initial sync - provider sync will handle balance creation with correct currency account = Account.create_and_sync( - family: Current.family, - name: lunchflow_account.name, - balance: lunchflow_account.current_balance || 0, - currency: lunchflow_account.currency || "USD", - accountable_type: selected_type, - accountable_attributes: selected_subtype.present? ? { subtype: selected_subtype } : {} + { + family: Current.family, + name: lunchflow_account.name, + balance: lunchflow_account.current_balance || 0, + currency: lunchflow_account.currency || "USD", + accountable_type: selected_type, + accountable_attributes: selected_subtype.present? ? { subtype: selected_subtype } : {} + }, + skip_initial_sync: true ) # Link account to lunchflow_account via account_providers join table (raises on failure) diff --git a/app/controllers/oidc_accounts_controller.rb b/app/controllers/oidc_accounts_controller.rb index 8e346fe79..6e22a1c93 100644 --- a/app/controllers/oidc_accounts_controller.rb +++ b/app/controllers/oidc_accounts_controller.rb @@ -98,14 +98,15 @@ class OidcAccountsController < ApplicationController return end - # Create user with a secure random password since they're using SSO - secure_password = SecureRandom.base58(32) + # Create SSO-only user without local password. + # Security: JIT users should NOT have password_digest set to prevent + # chained authentication attacks where SSO users gain local login access + # via password reset. @user = User.new( email: email, first_name: @pending_auth["first_name"], last_name: @pending_auth["last_name"], - password: secure_password, - password_confirmation: secure_password + skip_password_validation: true ) # Create new family for this user diff --git a/app/controllers/pages_controller.rb b/app/controllers/pages_controller.rb index 858e54a9b..db76e9d1d 100644 --- a/app/controllers/pages_controller.rb +++ b/app/controllers/pages_controller.rb @@ -5,16 +5,17 @@ class PagesController < ApplicationController def dashboard @balance_sheet = Current.family.balance_sheet + @investment_statement = Current.family.investment_statement @accounts = Current.family.accounts.visible.with_attached_logo family_currency = Current.family.currency - # Use the same period for all widgets (set by Periodable concern) + # Use IncomeStatement for all cashflow data (now includes categorized trades) income_totals = Current.family.income_statement.income_totals(period: @period) expense_totals = Current.family.income_statement.expense_totals(period: @period) @cashflow_sankey_data = build_cashflow_sankey_data(income_totals, expense_totals, family_currency) - @outflows_data = build_outflows_donut_data(expense_totals, family_currency) + @outflows_data = build_outflows_donut_data(expense_totals) @dashboard_sections = build_dashboard_sections @@ -81,6 +82,14 @@ class PagesController < ApplicationController visible: Current.family.accounts.any? && @outflows_data[:categories].present?, collapsible: true }, + { + key: "investment_summary", + title: "pages.dashboard.investment_summary.title", + partial: "pages/dashboard/investment_summary", + locals: { investment_statement: @investment_statement, period: @period }, + visible: Current.family.accounts.any? && @investment_statement.investment_accounts.any?, + collapsible: true + }, { key: "net_worth_chart", title: "pages.dashboard.net_worth_chart.title", @@ -117,12 +126,11 @@ class PagesController < ApplicationController Provider::Registry.get_provider(:github) end - def build_cashflow_sankey_data(income_totals, expense_totals, currency_symbol) + def build_cashflow_sankey_data(income_totals, expense_totals, currency) nodes = [] links = [] - node_indices = {} # Memoize node indices by a unique key: "type_categoryid" + node_indices = {} - # Helper to add/find node and return its index add_node = ->(unique_key, display_name, value, percentage, color) { node_indices[unique_key] ||= begin nodes << { name: display_name, value: value.to_f.round(2), percentage: percentage.to_f.round(1), color: color } @@ -130,93 +138,59 @@ class PagesController < ApplicationController end } - total_income_val = income_totals.total.to_f.round(2) - total_expense_val = expense_totals.total.to_f.round(2) + total_income = income_totals.total.to_f.round(2) + total_expense = expense_totals.total.to_f.round(2) - # --- Create Central Cash Flow Node --- - cash_flow_idx = add_node.call("cash_flow_node", "Cash Flow", total_income_val, 0, "var(--color-success)") + # Central Cash Flow node + cash_flow_idx = add_node.call("cash_flow_node", "Cash Flow", total_income, 100.0, "var(--color-success)") - # --- Process Income Side (Top-level categories only) --- + # Income side (top-level categories only) income_totals.category_totals.each do |ct| - # Skip subcategories – only include root income categories next if ct.category.parent_id.present? val = ct.total.to_f.round(2) next if val.zero? - percentage_of_total_income = total_income_val.zero? ? 0 : (val / total_income_val * 100).round(1) + percentage = total_income.zero? ? 0 : (val / total_income * 100).round(1) + color = ct.category.color.presence || Category::COLORS.sample - node_display_name = ct.category.name - node_color = ct.category.color.presence || Category::COLORS.sample - - current_cat_idx = add_node.call( - "income_#{ct.category.id}", - node_display_name, - val, - percentage_of_total_income, - node_color - ) - - links << { - source: current_cat_idx, - target: cash_flow_idx, - value: val, - color: node_color, - percentage: percentage_of_total_income - } + # Use name as fallback key for synthetic categories (no id) + node_key = "income_#{ct.category.id || ct.category.name}" + idx = add_node.call(node_key, ct.category.name, val, percentage, color) + links << { source: idx, target: cash_flow_idx, value: val, color: color, percentage: percentage } end - # --- Process Expense Side (Top-level categories only) --- + # Expense side (top-level categories only) expense_totals.category_totals.each do |ct| - # Skip subcategories – only include root expense categories to keep Sankey shallow next if ct.category.parent_id.present? val = ct.total.to_f.round(2) next if val.zero? - percentage_of_total_expense = total_expense_val.zero? ? 0 : (val / total_expense_val * 100).round(1) + percentage = total_expense.zero? ? 0 : (val / total_expense * 100).round(1) + color = ct.category.color.presence || Category::UNCATEGORIZED_COLOR - node_display_name = ct.category.name - node_color = ct.category.color.presence || Category::UNCATEGORIZED_COLOR - - current_cat_idx = add_node.call( - "expense_#{ct.category.id}", - node_display_name, - val, - percentage_of_total_expense, - node_color - ) - - links << { - source: cash_flow_idx, - target: current_cat_idx, - value: val, - color: node_color, - percentage: percentage_of_total_expense - } + # Use name as fallback key for synthetic categories (no id) + node_key = "expense_#{ct.category.id || ct.category.name}" + idx = add_node.call(node_key, ct.category.name, val, percentage, color) + links << { source: cash_flow_idx, target: idx, value: val, color: color, percentage: percentage } end - # --- Process Surplus --- - leftover = (total_income_val - total_expense_val).round(2) - if leftover.positive? - percentage_of_total_income_for_surplus = total_income_val.zero? ? 0 : (leftover / total_income_val * 100).round(1) - surplus_idx = add_node.call("surplus_node", "Surplus", leftover, percentage_of_total_income_for_surplus, "var(--color-success)") - links << { source: cash_flow_idx, target: surplus_idx, value: leftover, color: "var(--color-success)", percentage: percentage_of_total_income_for_surplus } + # Surplus/Deficit + net = (total_income - total_expense).round(2) + if net.positive? + percentage = total_income.zero? ? 0 : (net / total_income * 100).round(1) + idx = add_node.call("surplus_node", "Surplus", net, percentage, "var(--color-success)") + links << { source: cash_flow_idx, target: idx, value: net, color: "var(--color-success)", percentage: percentage } end - # Update Cash Flow and Income node percentages (relative to total income) - if node_indices["cash_flow_node"] - nodes[node_indices["cash_flow_node"]][:percentage] = 100.0 - end - # No primary income node anymore, percentages are on individual income cats relative to total_income_val - - { nodes: nodes, links: links, currency_symbol: Money::Currency.new(currency_symbol).symbol } + { nodes: nodes, links: links, currency_symbol: Money::Currency.new(currency).symbol } end - def build_outflows_donut_data(expense_totals, family_currency) + def build_outflows_donut_data(expense_totals) + currency_symbol = Money::Currency.new(expense_totals.currency).symbol total = expense_totals.total - # Only include top-level categories with non-zero amounts categories = expense_totals.category_totals .reject { |ct| ct.category.parent_id.present? || ct.total.zero? } .sort_by { |ct| -ct.total } @@ -228,10 +202,11 @@ class PagesController < ApplicationController currency: ct.currency, percentage: ct.weight.round(1), color: ct.category.color.presence || Category::UNCATEGORIZED_COLOR, - icon: ct.category.lucide_icon + icon: ct.category.lucide_icon, + clickable: !ct.category.other_investments? } end - { categories: categories, total: total.to_f.round(2), currency: family_currency, currency_symbol: Money::Currency.new(family_currency).symbol } + { categories: categories, total: total.to_f.round(2), currency: expense_totals.currency, currency_symbol: currency_symbol } end end diff --git a/app/controllers/password_resets_controller.rb b/app/controllers/password_resets_controller.rb index 76d0adb1a..482654a3a 100644 --- a/app/controllers/password_resets_controller.rb +++ b/app/controllers/password_resets_controller.rb @@ -11,12 +11,17 @@ class PasswordResetsController < ApplicationController def create if (user = User.find_by(email: params[:email])) - PasswordMailer.with( - user: user, - token: user.generate_token_for(:password_reset) - ).password_reset.deliver_later + # Security: Block password reset for SSO-only users. + # These users have no local password and must authenticate via SSO. + unless user.sso_only? + PasswordMailer.with( + user: user, + token: user.generate_token_for(:password_reset) + ).password_reset.deliver_later + end end + # Always redirect to pending step to prevent email enumeration redirect_to new_password_reset_path(step: "pending") end @@ -25,6 +30,13 @@ class PasswordResetsController < ApplicationController end def update + # Security: Block password setting for SSO-only users. + # Defense-in-depth: even if they somehow get a reset token, block the update. + if @user.sso_only? + redirect_to new_session_path, alert: t("password_resets.sso_only_user") + return + end + if @user.update(password_params) redirect_to new_session_path, notice: t(".success") else diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb index 96f3f7c17..bfb833acb 100644 --- a/app/controllers/reports_controller.rb +++ b/app/controllers/reports_controller.rb @@ -31,12 +31,15 @@ class ReportsController < ApplicationController # Build trend data (last 6 months) @trends_data = build_trends_data - # Spending patterns (weekday vs weekend) - @spending_patterns = build_spending_patterns + # Net worth metrics + @net_worth_metrics = build_net_worth_metrics # Transactions breakdown @transactions = build_transactions_breakdown + # Investment metrics (must be before build_reports_sections) + @investment_metrics = build_investment_metrics + # Build reports sections for collapsible/reorderable UI @reports_sections = build_reports_sections @@ -121,14 +124,30 @@ class ReportsController < ApplicationController def build_reports_sections all_sections = [ + { + key: "net_worth", + title: "reports.net_worth.title", + partial: "reports/net_worth", + locals: { net_worth_metrics: @net_worth_metrics }, + visible: Current.family.accounts.any?, + collapsible: true + }, { key: "trends_insights", title: "reports.trends.title", partial: "reports/trends_insights", - locals: { trends_data: @trends_data, spending_patterns: @spending_patterns }, + locals: { trends_data: @trends_data }, visible: Current.family.transactions.any?, collapsible: true }, + { + key: "investment_performance", + title: "reports.investment_performance.title", + partial: "reports/investment_performance", + locals: { investment_metrics: @investment_metrics }, + visible: @investment_metrics[:has_investments], + collapsible: true + }, { key: "transactions_breakdown", title: "reports.transactions_breakdown.title", @@ -299,61 +318,6 @@ class ReportsController < ApplicationController trends end - def build_spending_patterns - # Analyze weekday vs weekend spending - weekday_total = 0 - weekend_total = 0 - weekday_count = 0 - weekend_count = 0 - - # Build query matching income_statement logic: - # Expenses are transactions with positive amounts, regardless of category - expense_transactions = Transaction - .joins(:entry) - .joins(entry: :account) - .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) - .where(entries: { entryable_type: "Transaction", excluded: false, date: @period.date_range }) - .where(kind: [ "standard", "loan_payment" ]) - .where("entries.amount > 0") # Positive amount = expense (matching income_statement logic) - - # Sum up amounts by weekday vs weekend - expense_transactions.each do |transaction| - entry = transaction.entry - amount = entry.amount.abs - - if entry.date.wday.in?([ 0, 6 ]) # Sunday or Saturday - weekend_total += amount - weekend_count += 1 - else - weekday_total += amount - weekday_count += 1 - end - end - - weekday_avg = weekday_count.positive? ? (weekday_total / weekday_count) : 0 - weekend_avg = weekend_count.positive? ? (weekend_total / weekend_count) : 0 - - { - weekday_total: weekday_total, - weekend_total: weekend_total, - weekday_avg: weekday_avg, - weekend_avg: weekend_avg, - weekday_count: weekday_count, - weekend_count: weekend_count - } - end - - def default_spending_patterns - { - weekday_total: 0, - weekend_total: 0, - weekday_avg: 0, - weekend_avg: 0, - weekday_count: 0, - weekend_count: 0 - } - end - def build_transactions_breakdown # Base query: all transactions in the period # Exclude transfers, one-time, and CC payments (matching income_statement logic) @@ -368,25 +332,55 @@ class ReportsController < ApplicationController # Apply filters transactions = apply_transaction_filters(transactions) + # Get trades in the period (matching income_statement logic) + trades = Trade + .joins(:entry) + .joins(entry: :account) + .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) + .where(entries: { entryable_type: "Trade", excluded: false, date: @period.date_range }) + .includes(entry: :account, category: []) + # Get sort parameters sort_by = params[:sort_by] || "amount" sort_direction = params[:sort_direction] || "desc" # Group by category and type - all_transactions = transactions.to_a grouped_data = {} + family_currency = Current.family.currency - all_transactions.each do |transaction| + # Process transactions + transactions.each do |transaction| entry = transaction.entry is_expense = entry.amount > 0 type = is_expense ? "expense" : "income" category_name = transaction.category&.name || "Uncategorized" - category_color = transaction.category&.color || "#9CA3AF" + category_color = transaction.category&.color || Category::UNCATEGORIZED_COLOR + + # Convert to family currency + converted_amount = Money.new(entry.amount.abs, entry.currency).exchange_to(family_currency, fallback_rate: 1).amount key = [ category_name, type, category_color ] grouped_data[key] ||= { total: 0, count: 0 } grouped_data[key][:count] += 1 - grouped_data[key][:total] += entry.amount.abs + grouped_data[key][:total] += converted_amount + end + + # Process trades + trades.each do |trade| + entry = trade.entry + is_expense = entry.amount > 0 + type = is_expense ? "expense" : "income" + # Use "Other Investments" for trades without category + category_name = trade.category&.name || Category.other_investments_name + category_color = trade.category&.color || Category::OTHER_INVESTMENTS_COLOR + + # Convert to family currency + converted_amount = Money.new(entry.amount.abs, entry.currency).exchange_to(family_currency, fallback_rate: 1).amount + + key = [ category_name, type, category_color ] + grouped_data[key] ||= { total: 0, count: 0 } + grouped_data[key][:count] += 1 + grouped_data[key][:total] += converted_amount end # Convert to array @@ -408,6 +402,58 @@ class ReportsController < ApplicationController end end + def build_investment_metrics + investment_statement = Current.family.investment_statement + investment_accounts = investment_statement.investment_accounts + + return { has_investments: false } unless investment_accounts.any? + + period_totals = investment_statement.totals(period: @period) + + { + has_investments: true, + portfolio_value: investment_statement.portfolio_value_money, + unrealized_trend: investment_statement.unrealized_gains_trend, + period_contributions: period_totals.contributions, + period_withdrawals: period_totals.withdrawals, + top_holdings: investment_statement.top_holdings(limit: 5), + accounts: investment_accounts.to_a + } + end + + def build_net_worth_metrics + balance_sheet = Current.family.balance_sheet + currency = Current.family.currency + + # Current net worth + current_net_worth = balance_sheet.net_worth + total_assets = balance_sheet.assets.total + total_liabilities = balance_sheet.liabilities.total + + # Get net worth series for the period to calculate change + # The series.trend gives us the change from first to last value in the period + net_worth_series = balance_sheet.net_worth_series(period: @period) + trend = net_worth_series&.trend + + # Get asset and liability groups for breakdown + asset_groups = balance_sheet.assets.account_groups.map do |group| + { name: group.name, total: Money.new(group.total, currency) } + end.reject { |g| g[:total].zero? } + + liability_groups = balance_sheet.liabilities.account_groups.map do |group| + { name: group.name, total: Money.new(group.total, currency) } + end.reject { |g| g[:total].zero? } + + { + current_net_worth: Money.new(current_net_worth, currency), + total_assets: Money.new(total_assets, currency), + total_liabilities: Money.new(total_liabilities, currency), + trend: trend, + asset_groups: asset_groups, + liability_groups: liability_groups + } + end + def apply_transaction_filters(transactions) # Filter by category (including subcategories) if params[:filter_category_id].present? @@ -503,9 +549,19 @@ class ReportsController < ApplicationController transactions = apply_transaction_filters(transactions) - # Group transactions by category, type, and month - breakdown = {} + # Get trades in the period (matching income_statement logic) + trades = Trade + .joins(:entry) + .joins(entry: :account) + .where(accounts: { family_id: Current.family.id, status: [ "draft", "active" ] }) + .where(entries: { entryable_type: "Trade", excluded: false, date: @period.date_range }) + .includes(entry: :account, category: []) + # Group by category, type, and month + breakdown = {} + family_currency = Current.family.currency + + # Process transactions transactions.each do |transaction| entry = transaction.entry is_expense = entry.amount > 0 @@ -513,11 +569,33 @@ class ReportsController < ApplicationController category_name = transaction.category&.name || "Uncategorized" month_key = entry.date.beginning_of_month + # Convert to family currency + converted_amount = Money.new(entry.amount.abs, entry.currency).exchange_to(family_currency, fallback_rate: 1).amount + key = [ category_name, type ] breakdown[key] ||= { category: category_name, type: type, months: {}, total: 0 } breakdown[key][:months][month_key] ||= 0 - breakdown[key][:months][month_key] += entry.amount.abs - breakdown[key][:total] += entry.amount.abs + breakdown[key][:months][month_key] += converted_amount + breakdown[key][:total] += converted_amount + end + + # Process trades + trades.each do |trade| + entry = trade.entry + is_expense = entry.amount > 0 + type = is_expense ? "expense" : "income" + # Use "Other Investments" for trades without category + category_name = trade.category&.name || Category.other_investments_name + month_key = entry.date.beginning_of_month + + # Convert to family currency + converted_amount = Money.new(entry.amount.abs, entry.currency).exchange_to(family_currency, fallback_rate: 1).amount + + key = [ category_name, type ] + breakdown[key] ||= { category: category_name, type: type, months: {}, total: 0 } + breakdown[key][:months][month_key] ||= 0 + breakdown[key][:months][month_key] += converted_amount + breakdown[key][:total] += converted_amount end # Convert to array and sort by type and total (descending) diff --git a/app/controllers/rules_controller.rb b/app/controllers/rules_controller.rb index 24927b295..cd8a6eb6c 100644 --- a/app/controllers/rules_controller.rb +++ b/app/controllers/rules_controller.rb @@ -104,6 +104,30 @@ class RulesController < ApplicationController redirect_to rules_path, notice: "All rules deleted" end + def confirm_all + @rules = Current.family.rules + @total_affected_count = Rule.total_affected_resource_count(@rules) + + # Compute AI cost estimation if any rule has auto_categorize action + if @rules.any? { |r| r.actions.any? { |a| a.action_type == "auto_categorize" } } + llm_provider = Provider::Registry.get_provider(:openai) + + if llm_provider + @selected_model = Provider::Openai.effective_model + @estimated_cost = LlmUsage.estimate_auto_categorize_cost( + transaction_count: @total_affected_count, + category_count: Current.family.categories.count, + model: @selected_model + ) + end + end + end + + def apply_all + ApplyAllRulesJob.perform_later(Current.family) + redirect_back_or_to rules_path, notice: t("rules.apply_all.success") + end + private def set_rule @rule = Current.family.rules.find(params[:id]) diff --git a/app/controllers/settings/providers_controller.rb b/app/controllers/settings/providers_controller.rb index 43cd1eed1..4b9101fca 100644 --- a/app/controllers/settings/providers_controller.rb +++ b/app/controllers/settings/providers_controller.rb @@ -8,7 +8,7 @@ class Settings::ProvidersController < ApplicationController def show @breadcrumbs = [ [ "Home", root_path ], - [ "Bank Sync Providers", nil ] + [ "Sync Providers", nil ] ] prepare_show_context @@ -124,13 +124,14 @@ class Settings::ProvidersController < ApplicationController Provider::Factory.ensure_adapters_loaded @provider_configurations = Provider::ConfigurationRegistry.all.reject do |config| config.provider_key.to_s.casecmp("simplefin").zero? || config.provider_key.to_s.casecmp("lunchflow").zero? || \ - config.provider_key.to_s.casecmp("enable_banking").zero? + config.provider_key.to_s.casecmp("enable_banking").zero? || \ + config.provider_key.to_s.casecmp("coinstats").zero? end # Providers page only needs to know whether any SimpleFin/Lunchflow connections exist with valid credentials @simplefin_items = Current.family.simplefin_items.where.not(access_url: [ nil, "" ]).ordered.select(:id) @lunchflow_items = Current.family.lunchflow_items.where.not(api_key: [ nil, "" ]).ordered.select(:id) - # Enable Banking panel needs session info for status display - @enable_banking_items = Current.family.enable_banking_items.ordered + @enable_banking_items = Current.family.enable_banking_items.ordered # Enable Banking panel needs session info for status display + @coinstats_items = Current.family.coinstats_items.ordered # CoinStats panel needs account info for status display end end diff --git a/app/controllers/simplefin_items_controller.rb b/app/controllers/simplefin_items_controller.rb index 03a5322de..2aeb01f5d 100644 --- a/app/controllers/simplefin_items_controller.rb +++ b/app/controllers/simplefin_items_controller.rb @@ -21,43 +21,21 @@ class SimplefinItemsController < ApplicationController return render_error(t(".errors.blank_token"), context: :edit) if setup_token.blank? begin - # Create new SimpleFin item data with updated token - updated_item = Current.family.create_simplefin_item!( - setup_token: setup_token, - item_name: @simplefin_item.name + # Validate token shape early so the user gets immediate feedback. + claim_url = Base64.decode64(setup_token) + URI.parse(claim_url) + + # Updating a SimpleFin connection can involve network retries/backoff and account import. + # Do it asynchronously so web requests aren't blocked by retry sleeps. + SimplefinConnectionUpdateJob.perform_later( + family_id: Current.family.id, + old_simplefin_item_id: @simplefin_item.id, + setup_token: setup_token ) - # Ensure new simplefin_accounts are created & have account_id set - updated_item.import_latest_simplefin_data - - # Transfer accounts from old item to new item - ActiveRecord::Base.transaction do - @simplefin_item.simplefin_accounts.each do |old_account| - if old_account.account.present? - # Find matching account in new item by account_id - new_account = updated_item.simplefin_accounts.find_by(account_id: old_account.account_id) - if new_account - # Transfer the account directly to the new SimpleFin account - # This will automatically break the old association - old_account.account.update!(simplefin_account_id: new_account.id) - end - end - end - - # Mark old item for deletion - @simplefin_item.destroy_later - end - - # Clear any requires_update status on new item - updated_item.update!(status: :good) - if turbo_frame_request? - @simplefin_items = Current.family.simplefin_items.ordered - render turbo_stream: turbo_stream.replace( - "simplefin-providers-panel", - partial: "settings/providers/simplefin_panel", - locals: { simplefin_items: @simplefin_items } - ) + flash.now[:notice] = t(".success") + render turbo_stream: Array(flash_notification_stream_items) else redirect_to accounts_path, notice: t(".success"), status: :see_other end @@ -157,12 +135,16 @@ class SimplefinItemsController < ApplicationController end def setup_accounts - @simplefin_accounts = @simplefin_item.simplefin_accounts.includes(:account).where(accounts: { id: nil }) + # Only show unlinked accounts - check both legacy FK and AccountProvider + @simplefin_accounts = @simplefin_item.simplefin_accounts + .left_joins(:account, :account_provider) + .where(accounts: { id: nil }, account_providers: { id: nil }) @account_type_options = [ [ "Skip this account", "skip" ], [ "Checking or Savings Account", "Depository" ], [ "Credit Card", "CreditCard" ], [ "Investment Account", "Investment" ], + [ "Crypto Account", "Crypto" ], [ "Loan or Mortgage", "Loan" ], [ "Other Asset", "OtherAsset" ] ] @@ -208,25 +190,46 @@ class SimplefinItemsController < ApplicationController label: "Loan Type:", options: Loan::SUBTYPES.map { |k, v| [ v[:long], k ] } }, + "Crypto" => { + label: nil, + options: [], + message: "Crypto accounts track cryptocurrency holdings." + }, "OtherAsset" => { label: nil, options: [], message: "No additional options needed for Other Assets." } } + + # Detect stale accounts: linked in DB but no longer in upstream SimpleFin API + @stale_simplefin_accounts = detect_stale_simplefin_accounts + if @stale_simplefin_accounts.any? + # Build list of target accounts for "move transactions to" dropdown + # Only show accounts from this SimpleFin connection (excluding stale ones) + stale_account_ids = @stale_simplefin_accounts.map { |sfa| sfa.current_account&.id }.compact + @target_accounts = @simplefin_item.accounts + .reject { |acct| stale_account_ids.include?(acct.id) } + .sort_by(&:name) + end end def complete_account_setup account_types = params[:account_types] || {} account_subtypes = params[:account_subtypes] || {} + stale_account_actions = permitted_stale_account_actions # Update sync start date from form if params[:sync_start_date].present? @simplefin_item.update!(sync_start_date: params[:sync_start_date]) end - # Valid account types for this provider (plus OtherAsset which SimpleFIN UI allows) - valid_types = Provider::SimplefinAdapter.supported_account_types + [ "OtherAsset" ] + # Process stale account actions first + stale_results = process_stale_account_actions(stale_account_actions) + stale_action_errors = stale_results[:errors] || [] + + # Valid account types for this provider (plus Crypto and OtherAsset which SimpleFIN UI allows) + valid_types = Provider::SimplefinAdapter.supported_account_types + [ "Crypto", "OtherAsset" ] created_accounts = [] skipped_count = 0 @@ -269,6 +272,8 @@ class SimplefinItemsController < ApplicationController selected_subtype ) simplefin_account.update!(account: account) + # Also create AccountProvider for consistency with the new linking system + simplefin_account.ensure_account_provider! created_accounts << account end @@ -286,6 +291,17 @@ class SimplefinItemsController < ApplicationController else flash[:notice] = t(".no_accounts") end + + # Add stale account results to flash + if stale_results[:deleted] > 0 || stale_results[:moved] > 0 + stale_message = t(".stale_accounts_processed", deleted: stale_results[:deleted], moved: stale_results[:moved]) + flash[:notice] = [ flash[:notice], stale_message ].compact.join(" ") + end + + # Warn about any stale account action failures + if stale_action_errors.any? + flash[:alert] = t(".stale_accounts_errors", count: stale_action_errors.size) + end if turbo_frame_request? # Recompute data needed by Accounts#index partials @manual_accounts = Account.uncached { @@ -462,6 +478,24 @@ class SimplefinItemsController < ApplicationController params.require(:simplefin_item).permit(:setup_token, :sync_start_date) end + def permitted_stale_account_actions + return {} unless params[:stale_account_actions].is_a?(ActionController::Parameters) + + # Permit the nested structure: stale_account_actions[simplefin_account_id][action|target_account_id] + params[:stale_account_actions].to_unsafe_h.each_with_object({}) do |(simplefin_account_id, action_params), result| + next unless simplefin_account_id.present? && action_params.is_a?(Hash) + + # Validate simplefin_account_id is a valid UUID format to prevent injection + next unless simplefin_account_id.to_s.match?(/\A[0-9a-f-]+\z/i) + + permitted = {} + permitted[:action] = action_params[:action] if %w[delete move skip].include?(action_params[:action]) + permitted[:target_account_id] = action_params[:target_account_id] if action_params[:target_account_id].present? + + result[simplefin_account_id] = permitted if permitted[:action].present? + end + end + def render_error(message, setup_token = nil, context: :new) if context == :edit # Keep the persisted record and assign the token for re-render @@ -483,4 +517,106 @@ class SimplefinItemsController < ApplicationController render context, status: :unprocessable_entity end end + + # Detect stale SimpleFin accounts: linked in DB but no longer in upstream API + def detect_stale_simplefin_accounts + # Get upstream account IDs from the last sync's raw_payload + raw_payload = @simplefin_item.raw_payload + return [] if raw_payload.blank? + + upstream_ids = raw_payload.with_indifferent_access[:accounts]&.map { |a| a[:id].to_s } || [] + return [] if upstream_ids.empty? + + # Find SimplefinAccounts that are linked but not in upstream + @simplefin_item.simplefin_accounts + .includes(:account, account_provider: :account) + .select { |sfa| sfa.current_account.present? && !upstream_ids.include?(sfa.account_id) } + end + + # Process user-selected actions for stale accounts + def process_stale_account_actions(stale_actions) + results = { deleted: 0, moved: 0, skipped: 0, errors: [] } + return results if stale_actions.blank? + + stale_actions.each do |simplefin_account_id, action_params| + action = action_params[:action] + next if action.blank? || action == "skip" + + sfa = @simplefin_item.simplefin_accounts.find_by(id: simplefin_account_id) + next unless sfa + + account = sfa.current_account + next unless account + + case action + when "delete" + if handle_stale_account_delete(sfa, account) + results[:deleted] += 1 + else + results[:errors] << { account: account.name, action: "delete" } + end + when "move" + target_account_id = action_params[:target_account_id] + if target_account_id.present? && handle_stale_account_move(sfa, account, target_account_id) + results[:moved] += 1 + else + results[:errors] << { account: account.name, action: "move" } + end + else + results[:skipped] += 1 + end + end + + results + end + + def handle_stale_account_delete(simplefin_account, account) + ActiveRecord::Base.transaction do + # Destroy the Account (cascades to entries/holdings) + account.destroy! + # Destroy the SimplefinAccount + simplefin_account.destroy! + end + true + rescue => e + Rails.logger.error("Failed to delete stale account: #{e.class} - #{e.message}") + false + end + + def handle_stale_account_move(simplefin_account, source_account, target_account_id) + target_account = @simplefin_item.accounts.find { |acct| acct.id.to_s == target_account_id.to_s } + return false unless target_account + + ActiveRecord::Base.transaction do + # Handle transfers that would become invalid after moving entries. + # Transfers linking source entries to target entries would end up with both + # entries in the same account, violating transfer_has_different_accounts validation. + source_entry_ids = source_account.entries.pluck(:id) + target_entry_ids = target_account.entries.pluck(:id) + + if source_entry_ids.any? && target_entry_ids.any? + # Find and destroy transfers between source and target accounts + # Use find_each + destroy! to invoke Transfer's custom destroy! callbacks + # which reset transaction kinds to "standard" + Transfer.where(inflow_transaction_id: source_entry_ids, outflow_transaction_id: target_entry_ids) + .or(Transfer.where(inflow_transaction_id: target_entry_ids, outflow_transaction_id: source_entry_ids)) + .find_each(&:destroy!) + end + + # Move all entries to target account + source_account.entries.update_all(account_id: target_account.id) + + # Destroy the now-empty source account + source_account.destroy! + # Destroy the SimplefinAccount + simplefin_account.destroy! + end + + # Trigger sync on target account to recalculate balances (after commit) + target_account.sync_later + true + rescue => e + Rails.logger.error("Failed to move transactions from stale account: #{e.class} - #{e.message}") + false + end end diff --git a/app/controllers/trades_controller.rb b/app/controllers/trades_controller.rb index beb8fe5ec..283c172c7 100644 --- a/app/controllers/trades_controller.rb +++ b/app/controllers/trades_controller.rb @@ -54,7 +54,7 @@ class TradesController < ApplicationController def entry_params params.require(:entry).permit( :name, :date, :amount, :currency, :excluded, :notes, :nature, - entryable_attributes: [ :id, :qty, :price ] + entryable_attributes: [ :id, :qty, :price, :category_id ] ) end diff --git a/app/javascript/controllers/account_type_selector_controller.js b/app/javascript/controllers/account_type_selector_controller.js index 1f794ef0e..4504c41b7 100644 --- a/app/javascript/controllers/account_type_selector_controller.js +++ b/app/javascript/controllers/account_type_selector_controller.js @@ -42,4 +42,27 @@ export default class extends Controller { } } } + + clearWarning(event) { + // When user selects a subtype value, clear all warning styling + const select = event.target + if (select.value) { + // Clear the subtype dropdown warning + const warningContainer = select.closest('.ring-2') + if (warningContainer) { + warningContainer.classList.remove('ring-2', 'ring-warning/50', 'rounded-md', 'p-2', '-m-2') + const warningText = warningContainer.querySelector('.text-warning') + if (warningText) { + warningText.remove() + } + } + + // Clear the parent card's warning border + const card = this.element.closest('.border-2.border-warning') + if (card) { + card.classList.remove('border-2', 'border-warning', 'bg-warning/5') + card.classList.add('border', 'border-primary') + } + } + } } \ No newline at end of file diff --git a/app/javascript/controllers/drag_and_drop_import_controller.js b/app/javascript/controllers/drag_and_drop_import_controller.js new file mode 100644 index 000000000..8b8332ac2 --- /dev/null +++ b/app/javascript/controllers/drag_and_drop_import_controller.js @@ -0,0 +1,65 @@ +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["input", "form", "overlay"] + + dragDepth = 0 + + connect() { + this.boundDragOver = this.dragOver.bind(this) + this.boundDragEnter = this.dragEnter.bind(this) + this.boundDragLeave = this.dragLeave.bind(this) + this.boundDrop = this.drop.bind(this) + + // Listen on the document to catch drags anywhere + document.addEventListener("dragover", this.boundDragOver) + document.addEventListener("dragenter", this.boundDragEnter) + document.addEventListener("dragleave", this.boundDragLeave) + document.addEventListener("drop", this.boundDrop) + } + + disconnect() { + document.removeEventListener("dragover", this.boundDragOver) + document.removeEventListener("dragenter", this.boundDragEnter) + document.removeEventListener("dragleave", this.boundDragLeave) + document.removeEventListener("drop", this.boundDrop) + } + + dragEnter(event) { + event.preventDefault() + this.dragDepth++ + if (this.dragDepth === 1) { + this.overlayTarget.classList.remove("hidden") + } + } + + dragOver(event) { + event.preventDefault() + } + + dragLeave(event) { + event.preventDefault() + this.dragDepth-- + if (this.dragDepth <= 0) { + this.dragDepth = 0 + this.overlayTarget.classList.add("hidden") + } + } + + drop(event) { + event.preventDefault() + this.dragDepth = 0 + this.overlayTarget.classList.add("hidden") + + if (event.dataTransfer.files.length > 0) { + const file = event.dataTransfer.files[0] + // Simple validation + if (file.type === "text/csv" || file.name.toLowerCase().endsWith(".csv")) { + this.inputTarget.files = event.dataTransfer.files + this.formTarget.requestSubmit() + } else { + alert("Please upload a valid CSV file.") + } + } + } +} diff --git a/app/javascript/controllers/stale_account_action_controller.js b/app/javascript/controllers/stale_account_action_controller.js new file mode 100644 index 000000000..787ce55d9 --- /dev/null +++ b/app/javascript/controllers/stale_account_action_controller.js @@ -0,0 +1,25 @@ +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["moveRadio", "targetSelect"] + static values = { accountId: String } + + connect() { + this.updateTargetVisibility() + } + + updateTargetVisibility() { + if (!this.hasTargetSelectTarget || !this.hasMoveRadioTarget) return + + const moveRadio = this.moveRadioTarget + const targetSelect = this.targetSelectTarget + + if (moveRadio?.checked) { + targetSelect.disabled = false + targetSelect.classList.remove("opacity-50", "cursor-not-allowed") + } else { + targetSelect.disabled = true + targetSelect.classList.add("opacity-50", "cursor-not-allowed") + } + } +} diff --git a/app/jobs/apply_all_rules_job.rb b/app/jobs/apply_all_rules_job.rb new file mode 100644 index 000000000..f14da7091 --- /dev/null +++ b/app/jobs/apply_all_rules_job.rb @@ -0,0 +1,9 @@ +class ApplyAllRulesJob < ApplicationJob + queue_as :medium_priority + + def perform(family, execution_type: "manual") + family.rules.find_each do |rule| + RuleJob.perform_now(rule, ignore_attribute_locks: true, execution_type: execution_type) + end + end +end diff --git a/app/jobs/data_cleaner_job.rb b/app/jobs/data_cleaner_job.rb new file mode 100644 index 000000000..8cb22f283 --- /dev/null +++ b/app/jobs/data_cleaner_job.rb @@ -0,0 +1,17 @@ +class DataCleanerJob < ApplicationJob + queue_as :scheduled + + def perform + clean_old_merchant_associations + end + + private + def clean_old_merchant_associations + # Delete FamilyMerchantAssociation records older than 30 days + deleted_count = FamilyMerchantAssociation + .where(unlinked_at: ...30.days.ago) + .delete_all + + Rails.logger.info("DataCleanerJob: Deleted #{deleted_count} old merchant associations") if deleted_count > 0 + end +end diff --git a/app/jobs/simplefin_connection_update_job.rb b/app/jobs/simplefin_connection_update_job.rb new file mode 100644 index 000000000..850eaac8b --- /dev/null +++ b/app/jobs/simplefin_connection_update_job.rb @@ -0,0 +1,167 @@ +class SimplefinConnectionUpdateJob < ApplicationJob + queue_as :high_priority + + # Disable automatic retries for this job since the setup token is single-use. + # If the token claim succeeds but import fails, retrying would fail at claim. + discard_on Provider::Simplefin::SimplefinError do |job, error| + Rails.logger.error( + "SimplefinConnectionUpdateJob discarded: #{error.class} - #{error.message} " \ + "(family_id=#{job.arguments.first[:family_id]}, old_item_id=#{job.arguments.first[:old_simplefin_item_id]})" + ) + end + + def perform(family_id:, old_simplefin_item_id:, setup_token:) + family = Family.find(family_id) + old_item = family.simplefin_items.find(old_simplefin_item_id) + + # Step 1: Claim the token and create the new item. + # This is the critical step - if it fails, we can safely retry. + # If it succeeds, the token is consumed and we must not retry the claim. + updated_item = family.create_simplefin_item!( + setup_token: setup_token, + item_name: old_item.name + ) + + # Step 2: Import accounts from SimpleFin. + # If this fails, we have an orphaned item but the token is already consumed. + # We handle this gracefully by marking the item and continuing. + begin + updated_item.import_latest_simplefin_data + rescue => e + Rails.logger.error( + "SimplefinConnectionUpdateJob: import failed for new item #{updated_item.id}: " \ + "#{e.class} - #{e.message}. Item created but may need manual sync." + ) + # Mark the item as needing attention but don't fail the job entirely. + # The item exists and can be synced manually later. + updated_item.update!(status: :requires_update) + # Still proceed to transfer accounts and schedule old item deletion + end + + # Step 3: Transfer account links from old to new item. + # This is idempotent and safe to retry. + # Check for linked accounts via BOTH legacy FK and AccountProvider. + ActiveRecord::Base.transaction do + old_item.simplefin_accounts.includes(:account, account_provider: :account).each do |old_account| + # Get the linked account via either system + linked_account = old_account.current_account + next unless linked_account.present? + + new_simplefin_account = find_matching_simplefin_account(old_account, updated_item.simplefin_accounts) + next unless new_simplefin_account + + # Update legacy FK + linked_account.update!(simplefin_account_id: new_simplefin_account.id) + + # Also migrate AccountProvider if it exists + if old_account.account_provider.present? + old_account.account_provider.update!( + provider_type: "SimplefinAccount", + provider_id: new_simplefin_account.id + ) + else + # Create AccountProvider for consistency + new_simplefin_account.ensure_account_provider! + end + end + end + + # Schedule deletion outside transaction to avoid race condition where + # the job is enqueued even if the transaction rolls back + old_item.destroy_later + + # Only mark as good if import succeeded (status wasn't set to requires_update above) + updated_item.update!(status: :good) unless updated_item.requires_update? + end + + private + # Find a matching SimpleFin account in the new item's accounts. + # Uses a multi-tier matching strategy: + # 1. Exact account_id match (preferred) + # 2. Fingerprint match (name + institution + account_type) + # 3. Fuzzy name match with same institution (fallback) + def find_matching_simplefin_account(old_account, new_accounts) + exact_match = new_accounts.find_by(account_id: old_account.account_id) + return exact_match if exact_match + + old_fingerprint = account_fingerprint(old_account) + fingerprint_match = new_accounts.find { |new_account| account_fingerprint(new_account) == old_fingerprint } + return fingerprint_match if fingerprint_match + + old_institution = extract_institution_id(old_account) + old_name_normalized = normalize_account_name(old_account.name) + + new_accounts.find do |new_account| + new_institution = extract_institution_id(new_account) + new_name_normalized = normalize_account_name(new_account.name) + + next false unless old_institution.present? && old_institution == new_institution + + names_similar?(old_name_normalized, new_name_normalized) + end + end + + def account_fingerprint(simplefin_account) + institution_id = extract_institution_id(simplefin_account) + name_normalized = normalize_account_name(simplefin_account.name) + account_type = simplefin_account.account_type.to_s.downcase + + "#{institution_id}:#{name_normalized}:#{account_type}" + end + + def extract_institution_id(simplefin_account) + org_data = simplefin_account.org_data + return nil unless org_data.is_a?(Hash) + + org_data["id"] || org_data["domain"] || org_data["name"]&.downcase&.gsub(/\s+/, "_") + end + + def normalize_account_name(name) + return "" if name.blank? + + name.to_s + .downcase + .gsub(/[^a-z0-9]/, "") + end + + def names_similar?(name1, name2) + return false if name1.blank? || name2.blank? + + return true if name1 == name2 + return true if name1.include?(name2) || name2.include?(name1) + + longer = [ name1.length, name2.length ].max + return false if longer == 0 + + # Use Levenshtein distance for more accurate similarity + distance = levenshtein_distance(name1, name2) + similarity = 1.0 - (distance.to_f / longer) + similarity >= 0.8 + end + + # Compute Levenshtein edit distance between two strings + def levenshtein_distance(s1, s2) + m, n = s1.length, s2.length + return n if m.zero? + return m if n.zero? + + # Use a single array and update in place for memory efficiency + prev_row = (0..n).to_a + curr_row = [] + + (1..m).each do |i| + curr_row[0] = i + (1..n).each do |j| + cost = s1[i - 1] == s2[j - 1] ? 0 : 1 + curr_row[j] = [ + prev_row[j] + 1, # deletion + curr_row[j - 1] + 1, # insertion + prev_row[j - 1] + cost # substitution + ].min + end + prev_row, curr_row = curr_row, prev_row + end + + prev_row[n] + end +end diff --git a/app/jobs/sync_hourly_job.rb b/app/jobs/sync_hourly_job.rb new file mode 100644 index 000000000..07fc33e64 --- /dev/null +++ b/app/jobs/sync_hourly_job.rb @@ -0,0 +1,27 @@ +class SyncHourlyJob < ApplicationJob + queue_as :scheduled + sidekiq_options lock: :until_executed, on_conflict: :log + + # Provider item classes that opt-in to hourly syncing + HOURLY_SYNCABLES = [ + CoinstatsItem # https://coinstats.app/api-docs/rate-limits#plan-limits + ].freeze + + def perform + Rails.logger.info("Starting hourly sync") + HOURLY_SYNCABLES.each do |syncable_class| + sync_items(syncable_class) + end + Rails.logger.info("Completed hourly sync") + end + + private + + def sync_items(syncable_class) + syncable_class.active.find_each do |item| + item.sync_later + rescue => e + Rails.logger.error("Failed to sync #{syncable_class.name} #{item.id}: #{e.message}") + end + end +end diff --git a/app/models/account.rb b/app/models/account.rb index 4f96782b4..b861302c4 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -68,7 +68,7 @@ class Account < ApplicationRecord end class << self - def create_and_sync(attributes) + def create_and_sync(attributes, skip_initial_sync: false) attributes[:accountable_attributes] ||= {} # Ensure accountable is created, even if empty account = new(attributes.merge(cash_balance: attributes[:balance])) initial_balance = attributes.dig(:accountable_attributes, :initial_balance)&.to_d @@ -81,7 +81,9 @@ class Account < ApplicationRecord raise result.error if result.error end - account.sync_later + # Skip initial sync for linked accounts - the provider sync will handle balance creation + # after the correct currency is known + account.sync_later unless skip_initial_sync account end @@ -130,7 +132,8 @@ class Account < ApplicationRecord simplefin_account_id: simplefin_account.id } - create_and_sync(attributes) + # Skip initial sync - provider sync will handle balance creation with correct currency + create_and_sync(attributes, skip_initial_sync: true) end def create_from_enable_banking_account(enable_banking_account, account_type, subtype = nil) @@ -156,11 +159,13 @@ class Account < ApplicationRecord accountable_attributes = {} accountable_attributes[:subtype] = subtype if subtype.present? + # Skip initial sync - provider sync will handle balance creation with correct currency create_and_sync( attributes.merge( accountable_type: account_type, accountable_attributes: accountable_attributes - ) + ), + skip_initial_sync: true ) end @@ -196,6 +201,10 @@ class Account < ApplicationRecord read_attribute(:institution_domain).presence || provider&.institution_domain end + def logo_url + provider&.logo_url + end + def destroy_later mark_for_deletion! DestroyJob.perform_later(self) diff --git a/app/models/account/provider_import_adapter.rb b/app/models/account/provider_import_adapter.rb index 83f483fac..52964b165 100644 --- a/app/models/account/provider_import_adapter.rb +++ b/app/models/account/provider_import_adapter.rb @@ -75,6 +75,7 @@ class Account::ProviderImportAdapter existing = entry.transaction.extra || {} incoming = extra.is_a?(Hash) ? extra.deep_stringify_keys : {} entry.transaction.extra = existing.deep_merge(incoming) + entry.transaction.save! end entry.save! entry @@ -92,14 +93,42 @@ class Account::ProviderImportAdapter def find_or_create_merchant(provider_merchant_id:, name:, source:, website_url: nil, logo_url: nil) return nil unless provider_merchant_id.present? && name.present? - ProviderMerchant.find_or_create_by!( - provider_merchant_id: provider_merchant_id, - source: source - ) do |m| - m.name = name - m.website_url = website_url - m.logo_url = logo_url + # First try to find by provider_merchant_id (stable identifier derived from normalized name) + # This handles case variations in merchant names (e.g., "ACME Corp" vs "Acme Corp") + merchant = ProviderMerchant.find_by(provider_merchant_id: provider_merchant_id, source: source) + + # If not found by provider_merchant_id, try by exact name match (backwards compatibility) + merchant ||= ProviderMerchant.find_by(source: source, name: name) + + if merchant + # Update logo if provided and merchant doesn't have one (or has a different one) + # Best-effort: don't fail transaction import if logo update fails + if logo_url.present? && merchant.logo_url != logo_url + begin + merchant.update!(logo_url: logo_url) + rescue StandardError => e + Rails.logger.warn("Failed to update merchant logo: merchant_id=#{merchant.id} logo_url=#{logo_url} error=#{e.message}") + end + end + return merchant end + + # Create new merchant + begin + merchant = ProviderMerchant.create!( + source: source, + name: name, + provider_merchant_id: provider_merchant_id, + website_url: website_url, + logo_url: logo_url + ) + rescue ActiveRecord::RecordNotUnique + # Race condition - another process created the record + merchant = ProviderMerchant.find_by(provider_merchant_id: provider_merchant_id, source: source) || + ProviderMerchant.find_by(source: source, name: name) + end + + merchant end # Updates account balance from provider data diff --git a/app/models/account_import.rb b/app/models/account_import.rb index 0ffdcec4c..f5b790a09 100644 --- a/app/models/account_import.rb +++ b/app/models/account_import.rb @@ -42,7 +42,7 @@ class AccountImport < Import def dry_run { - accounts: rows.count + accounts: rows_count } end diff --git a/app/models/account_provider.rb b/app/models/account_provider.rb index bfb39f508..5ac8ef2cb 100644 --- a/app/models/account_provider.rb +++ b/app/models/account_provider.rb @@ -2,9 +2,16 @@ class AccountProvider < ApplicationRecord belongs_to :account belongs_to :provider, polymorphic: true + has_many :holdings, dependent: :nullify + validates :account_id, uniqueness: { scope: :provider_type } validates :provider_id, uniqueness: { scope: :provider_type } + # When unlinking a CoinStats account, also destroy the CoinstatsAccount record + # so it doesn't remain orphaned and count as "needs setup". + # Other providers may legitimately enter a "needs setup" state. + after_destroy :destroy_coinstats_provider_account, if: :coinstats_provider? + # Returns the provider adapter for this connection def adapter Provider::Factory.create_adapter(provider, account: account) @@ -15,4 +22,14 @@ class AccountProvider < ApplicationRecord def provider_name adapter&.provider_name || provider_type.underscore end + + private + + def coinstats_provider? + provider_type == "CoinstatsAccount" + end + + def destroy_coinstats_provider_account + provider&.destroy + end end diff --git a/app/models/assistant/configurable.rb b/app/models/assistant/configurable.rb index 1da95d14b..a2898c30b 100644 --- a/app/models/assistant/configurable.rb +++ b/app/models/assistant/configurable.rb @@ -17,6 +17,7 @@ module Assistant::Configurable [ Assistant::Function::GetTransactions, Assistant::Function::GetAccounts, + Assistant::Function::GetHoldings, Assistant::Function::GetBalanceSheet, Assistant::Function::GetIncomeStatement ] diff --git a/app/models/assistant/function/get_holdings.rb b/app/models/assistant/function/get_holdings.rb new file mode 100644 index 000000000..515888e8a --- /dev/null +++ b/app/models/assistant/function/get_holdings.rb @@ -0,0 +1,167 @@ +class Assistant::Function::GetHoldings < Assistant::Function + include Pagy::Backend + + SUPPORTED_ACCOUNT_TYPES = %w[Investment Crypto].freeze + + class << self + def default_page_size + 50 + end + + def name + "get_holdings" + end + + def description + <<~INSTRUCTIONS + Use this to search user's investment holdings by using various optional filters. + + This function is great for things like: + - Finding specific holdings or securities + - Getting portfolio composition and allocation + - Viewing investment performance and cost basis + + Note: This function only returns holdings from Investment and Crypto accounts. + + Note on pagination: + + This function can be paginated. You can expect the following properties in the response: + + - `total_pages`: The total number of pages of results + - `page`: The current page of results + - `page_size`: The number of results per page (this will always be #{default_page_size}) + - `total_results`: The total number of results for the given filters + - `total_value`: The total value of all holdings for the given filters + + Simple example (all current holdings): + + ``` + get_holdings({ + page: 1 + }) + ``` + + More complex example (various filters): + + ``` + get_holdings({ + page: 1, + accounts: ["Brokerage Account"], + securities: ["AAPL", "GOOGL"] + }) + ``` + INSTRUCTIONS + end + end + + def strict_mode? + false + end + + def params_schema + build_schema( + required: [ "page" ], + properties: { + page: { + type: "integer", + description: "Page number" + }, + accounts: { + type: "array", + description: "Filter holdings by account name (only Investment and Crypto accounts are supported)", + items: { enum: investment_account_names }, + minItems: 1, + uniqueItems: true + }, + securities: { + type: "array", + description: "Filter holdings by security ticker symbol", + items: { enum: family_security_tickers }, + minItems: 1, + uniqueItems: true + } + } + ) + end + + def call(params = {}) + holdings_query = build_holdings_query(params) + + pagy, paginated_holdings = pagy( + holdings_query.includes(:security, :account).order(amount: :desc), + page: params["page"] || 1, + limit: default_page_size + ) + + total_value = holdings_query.sum(:amount) + + normalized_holdings = paginated_holdings.map do |holding| + { + ticker: holding.ticker, + name: holding.name, + quantity: holding.qty.to_f, + price: holding.price.to_f, + currency: holding.currency, + amount: holding.amount.to_f, + formatted_amount: holding.amount_money.format, + weight: holding.weight&.round(2), + average_cost: holding.avg_cost.to_f, + formatted_average_cost: holding.avg_cost.format, + account: holding.account.name, + date: holding.date + } + end + + { + holdings: normalized_holdings, + total_results: pagy.count, + page: pagy.page, + page_size: default_page_size, + total_pages: pagy.pages, + total_value: Money.new(total_value, family.currency).format + } + end + + private + def default_page_size + self.class.default_page_size + end + + def build_holdings_query(params) + accounts = investment_accounts + + if params["accounts"].present? + accounts = accounts.where(name: params["accounts"]) + end + + holdings = Holding.where(account: accounts) + .where( + id: Holding.where(account: accounts) + .select("DISTINCT ON (account_id, security_id) id") + .where.not(qty: 0) + .order(:account_id, :security_id, date: :desc) + ) + + if params["securities"].present? + security_ids = family.securities.where(ticker: params["securities"]).pluck(:id) + holdings = holdings.where(security_id: security_ids) + end + + holdings + end + + def investment_accounts + family.accounts.visible.where(accountable_type: SUPPORTED_ACCOUNT_TYPES) + end + + def investment_account_names + @investment_account_names ||= investment_accounts.pluck(:name) + end + + def family_security_tickers + @family_security_tickers ||= Security + .where(id: Holding.where(account_id: investment_accounts.select(:id)).select(:security_id)) + .distinct + .pluck(:ticker) + end +end diff --git a/app/models/balance/chart_series_builder.rb b/app/models/balance/chart_series_builder.rb index dad108171..c988651b7 100644 --- a/app/models/balance/chart_series_builder.rb +++ b/app/models/balance/chart_series_builder.rb @@ -130,6 +130,7 @@ class Balance::ChartSeriesBuilder b.flows_factor FROM balances b WHERE b.account_id = accounts.id + AND b.currency = accounts.currency AND b.date <= d.date ORDER BY b.date DESC LIMIT 1 diff --git a/app/models/category.rb b/app/models/category.rb index 936e0ebb7..9561cd4ee 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -1,5 +1,6 @@ class Category < ApplicationRecord has_many :transactions, dependent: :nullify, class_name: "Transaction" + has_many :trades, dependent: :nullify has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping" belongs_to :family @@ -30,10 +31,15 @@ class Category < ApplicationRecord COLORS = %w[#e99537 #4da568 #6471eb #db5a54 #df4e92 #c44fe9 #eb5429 #61c9ea #805dee #6ad28a] UNCATEGORIZED_COLOR = "#737373" + OTHER_INVESTMENTS_COLOR = "#e99537" TRANSFER_COLOR = "#444CE7" PAYMENT_COLOR = "#db5a54" TRADE_COLOR = "#e99537" + # Synthetic category name keys for i18n + UNCATEGORIZED_NAME_KEY = "models.category.uncategorized" + OTHER_INVESTMENTS_NAME_KEY = "models.category.other_investments" + class Group attr_reader :category, :subcategories @@ -81,12 +87,30 @@ class Category < ApplicationRecord def uncategorized new( - name: "Uncategorized", + name: I18n.t(UNCATEGORIZED_NAME_KEY), color: UNCATEGORIZED_COLOR, lucide_icon: "circle-dashed" ) end + def other_investments + new( + name: I18n.t(OTHER_INVESTMENTS_NAME_KEY), + color: OTHER_INVESTMENTS_COLOR, + lucide_icon: "trending-up" + ) + end + + # Helper to get the localized name for uncategorized + def uncategorized_name + I18n.t(UNCATEGORIZED_NAME_KEY) + end + + # Helper to get the localized name for other investments + def other_investments_name + I18n.t(OTHER_INVESTMENTS_NAME_KEY) + end + private def default_categories [ @@ -110,7 +134,8 @@ class Category < ApplicationRecord [ "Loan Payments", "#e11d48", "credit-card", "expense" ], [ "Services", "#7c3aed", "briefcase", "expense" ], [ "Fees", "#6b7280", "receipt", "expense" ], - [ "Savings & Investments", "#059669", "piggy-bank", "expense" ] + [ "Savings & Investments", "#059669", "piggy-bank", "expense" ], + [ "Investment Contributions", "#0d9488", "trending-up", "expense" ] ] end end @@ -140,6 +165,21 @@ class Category < ApplicationRecord subcategory? ? "#{parent.name} > #{name}" : name end + # Predicate: is this the synthetic "Uncategorized" category? + def uncategorized? + !persisted? && name == I18n.t(UNCATEGORIZED_NAME_KEY) + end + + # Predicate: is this the synthetic "Other Investments" category? + def other_investments? + !persisted? && name == I18n.t(OTHER_INVESTMENTS_NAME_KEY) + end + + # Predicate: is this any synthetic (non-persisted) category? + def synthetic? + uncategorized? || other_investments? + end + private def category_level_limit if (subcategory? && parent.subcategory?) || (parent? && subcategory?) diff --git a/app/models/category_import.rb b/app/models/category_import.rb index a862bb77d..59f8095f4 100644 --- a/app/models/category_import.rb +++ b/app/models/category_import.rb @@ -42,7 +42,7 @@ class CategoryImport < Import end def dry_run - { categories: rows.count } + { categories: rows_count } end def csv_template diff --git a/app/models/coinstats_account.rb b/app/models/coinstats_account.rb new file mode 100644 index 000000000..dcab2e186 --- /dev/null +++ b/app/models/coinstats_account.rb @@ -0,0 +1,71 @@ +# Represents a single crypto token/coin within a CoinStats wallet. +# Each wallet address may have multiple CoinstatsAccounts (one per token). +class CoinstatsAccount < ApplicationRecord + include CurrencyNormalizable + + belongs_to :coinstats_item + + # Association through account_providers (standard pattern for all providers) + has_one :account_provider, as: :provider, dependent: :destroy + has_one :account, through: :account_provider, source: :account + + validates :name, :currency, presence: true + validates :account_id, uniqueness: { scope: :coinstats_item_id, allow_nil: true } + + # Alias for compatibility with provider adapter pattern + alias_method :current_account, :account + + # Updates account with latest balance data from CoinStats API. + # @param account_snapshot [Hash] Normalized balance data from API + def upsert_coinstats_snapshot!(account_snapshot) + # Convert to symbol keys or handle both string and symbol keys + snapshot = account_snapshot.with_indifferent_access + + # Build attributes to update + attrs = { + current_balance: snapshot[:balance] || snapshot[:current_balance], + currency: parse_currency(snapshot[:currency]) || "USD", + name: snapshot[:name], + account_status: snapshot[:status], + provider: snapshot[:provider], + institution_metadata: { + logo: snapshot[:institution_logo] + }.compact, + raw_payload: account_snapshot + } + + # Only set account_id if provided and not already set (preserves ID from initial creation) + if snapshot[:id].present? && account_id.blank? + attrs[:account_id] = snapshot[:id].to_s + end + + update!(attrs) + end + + # Stores transaction data from CoinStats API for later processing. + # @param transactions_snapshot [Hash, Array] Raw transactions response or array + def upsert_coinstats_transactions_snapshot!(transactions_snapshot) + # CoinStats API returns: { meta: { page, limit }, result: [...] } + # Extract just the result array for storage, or use directly if already an array + transactions_array = if transactions_snapshot.is_a?(Hash) + snapshot = transactions_snapshot.with_indifferent_access + snapshot[:result] || [] + elsif transactions_snapshot.is_a?(Array) + transactions_snapshot + else + [] + end + + assign_attributes( + raw_transactions_payload: transactions_array + ) + + save! + end + + private + + def log_invalid_currency(currency_value) + Rails.logger.warn("Invalid currency code '#{currency_value}' for CoinstatsAccount #{id}, defaulting to USD") + end +end diff --git a/app/models/coinstats_account/processor.rb b/app/models/coinstats_account/processor.rb new file mode 100644 index 000000000..70844d67f --- /dev/null +++ b/app/models/coinstats_account/processor.rb @@ -0,0 +1,68 @@ +# Processes a CoinStats account to update balance and import transactions. +# Updates the linked Account balance and delegates to transaction processor. +class CoinstatsAccount::Processor + include CurrencyNormalizable + + attr_reader :coinstats_account + + # @param coinstats_account [CoinstatsAccount] Account to process + def initialize(coinstats_account) + @coinstats_account = coinstats_account + end + + # Updates account balance and processes transactions. + # Skips processing if no linked account exists. + def process + unless coinstats_account.current_account.present? + Rails.logger.info "CoinstatsAccount::Processor - No linked account for coinstats_account #{coinstats_account.id}, skipping processing" + return + end + + Rails.logger.info "CoinstatsAccount::Processor - Processing coinstats_account #{coinstats_account.id}" + + begin + process_account! + rescue StandardError => e + Rails.logger.error "CoinstatsAccount::Processor - Failed to process account #{coinstats_account.id}: #{e.message}" + Rails.logger.error "Backtrace: #{e.backtrace.join("\n")}" + report_exception(e, "account") + raise + end + + process_transactions + end + + private + + # Updates the linked Account with current balance from CoinStats. + def process_account! + account = coinstats_account.current_account + balance = coinstats_account.current_balance || 0 + currency = parse_currency(coinstats_account.currency) || account.currency || "USD" + + account.update!( + balance: balance, + cash_balance: balance, + currency: currency + ) + end + + # Delegates transaction processing to the specialized processor. + def process_transactions + CoinstatsAccount::Transactions::Processor.new(coinstats_account).process + rescue StandardError => e + report_exception(e, "transactions") + end + + # Reports errors to Sentry with context tags. + # @param error [Exception] The error to report + # @param context [String] Processing context (e.g., "account", "transactions") + def report_exception(error, context) + Sentry.capture_exception(error) do |scope| + scope.set_tags( + coinstats_account_id: coinstats_account.id, + context: context + ) + end + end +end diff --git a/app/models/coinstats_account/transactions/processor.rb b/app/models/coinstats_account/transactions/processor.rb new file mode 100644 index 000000000..f5ee468f6 --- /dev/null +++ b/app/models/coinstats_account/transactions/processor.rb @@ -0,0 +1,138 @@ +# Processes stored transactions for a CoinStats account. +# Filters transactions by token and delegates to entry processor. +class CoinstatsAccount::Transactions::Processor + include CoinstatsTransactionIdentifiable + + attr_reader :coinstats_account + + # @param coinstats_account [CoinstatsAccount] Account with transactions to process + def initialize(coinstats_account) + @coinstats_account = coinstats_account + end + + # Processes all stored transactions for this account. + # Filters to relevant token and imports each transaction. + # @return [Hash] Result with :success, :total, :imported, :failed, :errors + def process + unless coinstats_account.raw_transactions_payload.present? + Rails.logger.info "CoinstatsAccount::Transactions::Processor - No transactions in raw_transactions_payload for coinstats_account #{coinstats_account.id}" + return { success: true, total: 0, imported: 0, failed: 0, errors: [] } + end + + # Filter transactions to only include ones for this specific token + # Multiple coinstats_accounts can share the same wallet address (one per token) + # but we only want to process transactions relevant to this token + relevant_transactions = filter_transactions_for_account(coinstats_account.raw_transactions_payload) + + total_count = relevant_transactions.count + Rails.logger.info "CoinstatsAccount::Transactions::Processor - Processing #{total_count} transactions for coinstats_account #{coinstats_account.id} (#{coinstats_account.name})" + + imported_count = 0 + failed_count = 0 + errors = [] + + relevant_transactions.each_with_index do |transaction_data, index| + begin + result = CoinstatsEntry::Processor.new( + transaction_data, + coinstats_account: coinstats_account + ).process + + if result.nil? + failed_count += 1 + transaction_id = extract_coinstats_transaction_id(transaction_data) + errors << { index: index, transaction_id: transaction_id, error: "No linked account" } + else + imported_count += 1 + end + rescue ArgumentError => e + failed_count += 1 + transaction_id = extract_coinstats_transaction_id(transaction_data) + error_message = "Validation error: #{e.message}" + Rails.logger.error "CoinstatsAccount::Transactions::Processor - #{error_message} (transaction #{transaction_id})" + errors << { index: index, transaction_id: transaction_id, error: error_message } + rescue => e + failed_count += 1 + transaction_id = extract_coinstats_transaction_id(transaction_data) + error_message = "#{e.class}: #{e.message}" + Rails.logger.error "CoinstatsAccount::Transactions::Processor - Error processing transaction #{transaction_id}: #{error_message}" + Rails.logger.error e.backtrace.join("\n") + errors << { index: index, transaction_id: transaction_id, error: error_message } + end + end + + result = { + success: failed_count == 0, + total: total_count, + imported: imported_count, + failed: failed_count, + errors: errors + } + + if failed_count > 0 + Rails.logger.warn "CoinstatsAccount::Transactions::Processor - Completed with #{failed_count} failures out of #{total_count} transactions" + else + Rails.logger.info "CoinstatsAccount::Transactions::Processor - Successfully processed #{imported_count} transactions" + end + + result + end + + private + + # Filters transactions to only include ones for this specific token. + # CoinStats returns all wallet transactions, but each CoinstatsAccount + # represents a single token, so we filter by matching coin ID or symbol. + # @param transactions [Array] Raw transactions from storage + # @return [Array] Transactions matching this account's token + def filter_transactions_for_account(transactions) + return [] unless transactions.present? + return transactions unless coinstats_account.account_id.present? + + account_id = coinstats_account.account_id.to_s.downcase + + transactions.select do |tx| + tx = tx.with_indifferent_access + + # Check coin ID in transactions[0].items[0].coin.id (most common location) + coin_id = tx.dig(:transactions, 0, :items, 0, :coin, :id)&.to_s&.downcase + + # Also check coinData for symbol match as fallback + coin_symbol = tx.dig(:coinData, :symbol)&.to_s&.downcase + + # Match if coin ID equals account_id, or if symbol matches account name precisely. + # We use strict matching to avoid false positives (e.g., "ETH" should not match + # "Ethereum Classic" which has symbol "ETC"). The symbol must appear as: + # - A whole word (bounded by word boundaries), OR + # - Inside parentheses like "(ETH)" which is common in wallet naming conventions + coin_id == account_id || + (coin_symbol.present? && symbol_matches_name?(coin_symbol, coinstats_account.name)) + end + end + + # Checks if a coin symbol matches the account name using strict matching. + # Avoids false positives from partial substring matches (e.g., "ETH" matching + # "Ethereum Classic (0x123...)" which should only match "ETC"). + # + # @param symbol [String] The coin symbol to match (already downcased) + # @param name [String, nil] The account name to match against + # @return [Boolean] true if symbol matches name precisely + def symbol_matches_name?(symbol, name) + return false if name.blank? + + normalized_name = name.to_s.downcase + + # Match symbol as a whole word using word boundaries, or within parentheses. + # Examples that SHOULD match: + # - "ETH" matches "ETH Wallet", "My ETH", "Ethereum (ETH)" + # - "BTC" matches "BTC", "(BTC) Savings", "Bitcoin (BTC)" + # Examples that should NOT match: + # - "ETH" should NOT match "Ethereum Classic" (symbol is "ETC") + # - "ETH" should NOT match "WETH Wrapped" (different token) + # - "BTC" should NOT match "BTCB" (different token) + word_boundary_pattern = /\b#{Regexp.escape(symbol)}\b/ + parenthesized_pattern = /\(#{Regexp.escape(symbol)}\)/ + + word_boundary_pattern.match?(normalized_name) || parenthesized_pattern.match?(normalized_name) + end +end diff --git a/app/models/coinstats_entry/processor.rb b/app/models/coinstats_entry/processor.rb new file mode 100644 index 000000000..6b966f60b --- /dev/null +++ b/app/models/coinstats_entry/processor.rb @@ -0,0 +1,270 @@ +# Processes a single CoinStats transaction into a local Transaction record. +# Extracts amount, date, and metadata from the CoinStats API format. +# +# CoinStats API transaction structure (from /wallet/transactions endpoint): +# { +# type: "Sent" | "Received" | "Swap" | ..., +# date: "2025-06-07T11:58:11.000Z", +# coinData: { count: -0.00636637, symbol: "ETH", currentValue: 29.21 }, +# profitLoss: { profit: -13.41, profitPercent: -84.44, currentValue: 29.21 }, +# hash: { id: "0x...", explorerUrl: "https://etherscan.io/tx/0x..." }, +# fee: { coin: { id, name, symbol, icon }, count: 0.00003, totalWorth: 0.08 }, +# transactions: [{ action: "Sent", items: [{ id, count, totalWorth, coin: {...} }] }] +# } +class CoinstatsEntry::Processor + include CoinstatsTransactionIdentifiable + + # @param coinstats_transaction [Hash] Raw transaction data from API + # @param coinstats_account [CoinstatsAccount] Parent account for context + def initialize(coinstats_transaction, coinstats_account:) + @coinstats_transaction = coinstats_transaction + @coinstats_account = coinstats_account + end + + # Imports the transaction into the linked account. + # @return [Transaction, nil] Created transaction or nil if no linked account + # @raise [ArgumentError] If transaction data is invalid + # @raise [StandardError] If import fails + def process + unless account.present? + Rails.logger.warn "CoinstatsEntry::Processor - No linked account for coinstats_account #{coinstats_account.id}, skipping transaction #{external_id}" + return nil + end + + import_adapter.import_transaction( + external_id: external_id, + amount: amount, + currency: currency, + date: date, + name: name, + source: "coinstats", + merchant: merchant, + notes: notes, + extra: extra_metadata + ) + rescue ArgumentError => e + Rails.logger.error "CoinstatsEntry::Processor - Validation error for transaction #{external_id rescue 'unknown'}: #{e.message}" + raise + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e + Rails.logger.error "CoinstatsEntry::Processor - Failed to save transaction #{external_id rescue 'unknown'}: #{e.message}" + raise StandardError.new("Failed to import transaction: #{e.message}") + rescue => e + Rails.logger.error "CoinstatsEntry::Processor - Unexpected error processing transaction #{external_id rescue 'unknown'}: #{e.class} - #{e.message}" + Rails.logger.error e.backtrace.join("\n") + raise StandardError.new("Unexpected error importing transaction: #{e.message}") + end + + private + + attr_reader :coinstats_transaction, :coinstats_account + + def extra_metadata + cs = {} + + # Store transaction hash and explorer URL + if hash_data.present? + cs["transaction_hash"] = hash_data[:id] if hash_data[:id].present? + cs["explorer_url"] = hash_data[:explorerUrl] if hash_data[:explorerUrl].present? + end + + # Store transaction type + cs["transaction_type"] = transaction_type if transaction_type.present? + + # Store coin/token info + if coin_data.present? + cs["symbol"] = coin_data[:symbol] if coin_data[:symbol].present? + cs["count"] = coin_data[:count] if coin_data[:count].present? + end + + # Store profit/loss info + if profit_loss.present? + cs["profit"] = profit_loss[:profit] if profit_loss[:profit].present? + cs["profit_percent"] = profit_loss[:profitPercent] if profit_loss[:profitPercent].present? + end + + # Store fee info + if fee_data.present? + cs["fee_amount"] = fee_data[:count] if fee_data[:count].present? + cs["fee_symbol"] = fee_data.dig(:coin, :symbol) if fee_data.dig(:coin, :symbol).present? + cs["fee_usd"] = fee_data[:totalWorth] if fee_data[:totalWorth].present? + end + + return nil if cs.empty? + { "coinstats" => cs } + end + + def import_adapter + @import_adapter ||= Account::ProviderImportAdapter.new(account) + end + + def account + coinstats_account.current_account + end + + def data + @data ||= coinstats_transaction.with_indifferent_access + end + + # Helper accessors for nested data structures + def hash_data + @hash_data ||= (data[:hash] || {}).with_indifferent_access + end + + def coin_data + @coin_data ||= (data[:coinData] || {}).with_indifferent_access + end + + def profit_loss + @profit_loss ||= (data[:profitLoss] || {}).with_indifferent_access + end + + def fee_data + @fee_data ||= (data[:fee] || {}).with_indifferent_access + end + + def transactions_data + @transactions_data ||= data[:transactions] || [] + end + + def transaction_type + data[:type] + end + + def external_id + tx_id = extract_coinstats_transaction_id(data) + raise ArgumentError, "CoinStats transaction missing unique identifier: #{data.inspect}" unless tx_id.present? + "coinstats_#{tx_id}" + end + + def name + tx_type = transaction_type || "Transaction" + symbol = coin_data[:symbol] + + # Get coin name from nested transaction items if available (used as fallback) + coin_name = transactions_data.dig(0, :items, 0, :coin, :name) + + if symbol.present? + "#{tx_type} #{symbol}" + elsif coin_name.present? + "#{tx_type} #{coin_name}" + else + tx_type.to_s + end + end + + def amount + # Use currentValue from coinData (USD value) or profitLoss + usd_value = coin_data[:currentValue] || profit_loss[:currentValue] || 0 + + parsed_amount = case usd_value + when String + BigDecimal(usd_value) + when Numeric + BigDecimal(usd_value.to_s) + else + BigDecimal("0") + end + + absolute_amount = parsed_amount.abs + + # App convention: negative amount = income (inflow), positive amount = expense (outflow) + # coinData.count is negative for outgoing transactions + coin_count = coin_data[:count] || 0 + + if coin_count.to_f < 0 || outgoing_transaction_type? + # Outgoing transaction = expense = positive + absolute_amount + else + # Incoming transaction = income = negative + -absolute_amount + end + rescue ArgumentError => e + Rails.logger.error "Failed to parse CoinStats transaction amount: #{usd_value.inspect} - #{e.message}" + raise + end + + def outgoing_transaction_type? + tx_type = (transaction_type || "").to_s.downcase + %w[sent send sell withdraw transfer_out swap_out].include?(tx_type) + end + + def currency + # CoinStats values are always in USD + "USD" + end + + def date + # CoinStats returns date as ISO 8601 string (e.g., "2025-06-07T11:58:11.000Z") + timestamp = data[:date] + + raise ArgumentError, "CoinStats transaction missing date" unless timestamp.present? + + case timestamp + when Integer, Float + Time.at(timestamp).to_date + when String + Time.parse(timestamp).to_date + when Time, DateTime + timestamp.to_date + when Date + timestamp + else + Rails.logger.error("CoinStats transaction has invalid date format: #{timestamp.inspect}") + raise ArgumentError, "Invalid date format: #{timestamp.inspect}" + end + rescue ArgumentError, TypeError => e + Rails.logger.error("CoinStats transaction date parsing failed: #{e.message}") + raise ArgumentError, "Invalid date format: #{timestamp.inspect}" + end + + def merchant + # Use the coinstats_account as the merchant source for consistency + # All transactions from the same account will have the same merchant and logo + merchant_name = coinstats_account.name + return nil unless merchant_name.present? + + # Use the account's logo (token icon) for the merchant + logo = coinstats_account.institution_metadata&.dig("logo") + + # Use the coinstats_account ID to ensure consistent merchant per account + @merchant ||= import_adapter.find_or_create_merchant( + provider_merchant_id: "coinstats_account_#{coinstats_account.id}", + name: merchant_name, + source: "coinstats", + logo_url: logo + ) + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error "CoinstatsEntry::Processor - Failed to create merchant '#{merchant_name}': #{e.message}" + nil + end + + def notes + parts = [] + + # Include coin/token details with count + symbol = coin_data[:symbol] + count = coin_data[:count] + if count.present? && symbol.present? + parts << "#{count} #{symbol}" + end + + # Include fee info + if fee_data[:count].present? && fee_data.dig(:coin, :symbol).present? + parts << "Fee: #{fee_data[:count]} #{fee_data.dig(:coin, :symbol)}" + end + + # Include profit/loss info + if profit_loss[:profit].present? + profit_formatted = profit_loss[:profit].to_f.round(2) + percent_formatted = profit_loss[:profitPercent].to_f.round(2) + parts << "P/L: $#{profit_formatted} (#{percent_formatted}%)" + end + + # Include explorer URL for reference + if hash_data[:explorerUrl].present? + parts << "Explorer: #{hash_data[:explorerUrl]}" + end + + parts.presence&.join(" | ") + end +end diff --git a/app/models/coinstats_item.rb b/app/models/coinstats_item.rb new file mode 100644 index 000000000..72da54d1f --- /dev/null +++ b/app/models/coinstats_item.rb @@ -0,0 +1,150 @@ +# Represents a CoinStats API connection for a family. +# Stores credentials and manages associated crypto wallet accounts. +class CoinstatsItem < ApplicationRecord + include Syncable, Provided, Unlinking + + enum :status, { good: "good", requires_update: "requires_update" }, default: :good + + # Checks if ActiveRecord Encryption is properly configured. + # @return [Boolean] true if encryption keys are available + def self.encryption_ready? + creds_ready = Rails.application.credentials.active_record_encryption.present? + env_ready = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"].present? && + ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"].present? && + ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"].present? + creds_ready || env_ready + end + + # Encrypt sensitive credentials if ActiveRecord encryption is configured + encrypts :api_key, deterministic: true if encryption_ready? + + validates :name, presence: true + validates :api_key, presence: true + + belongs_to :family + has_one_attached :logo + + has_many :coinstats_accounts, dependent: :destroy + has_many :accounts, through: :coinstats_accounts + + scope :active, -> { where(scheduled_for_deletion: false) } + scope :ordered, -> { order(created_at: :desc) } + scope :needs_update, -> { where(status: :requires_update) } + + # Schedules this item for async deletion. + def destroy_later + update!(scheduled_for_deletion: true) + DestroyJob.perform_later(self) + end + + # Fetches latest wallet data from CoinStats API and updates local records. + # @raise [StandardError] if provider is not configured or import fails + def import_latest_coinstats_data + provider = coinstats_provider + unless provider + Rails.logger.error "CoinstatsItem #{id} - Cannot import: CoinStats provider is not configured" + raise StandardError.new("CoinStats provider is not configured") + end + CoinstatsItem::Importer.new(self, coinstats_provider: provider).import + rescue => e + Rails.logger.error "CoinstatsItem #{id} - Failed to import data: #{e.message}" + raise + end + + # Processes holdings for all linked visible accounts. + # @return [Array] Results with success status per account + def process_accounts + return [] if coinstats_accounts.empty? + + results = [] + coinstats_accounts.includes(:account).joins(:account).merge(Account.visible).each do |coinstats_account| + begin + result = CoinstatsAccount::Processor.new(coinstats_account).process + results << { coinstats_account_id: coinstats_account.id, success: true, result: result } + rescue => e + Rails.logger.error "CoinstatsItem #{id} - Failed to process account #{coinstats_account.id}: #{e.message}" + results << { coinstats_account_id: coinstats_account.id, success: false, error: e.message } + end + end + + results + end + + # Queues balance sync jobs for all visible accounts. + # @param parent_sync [Sync, nil] Parent sync for tracking + # @param window_start_date [Date, nil] Start of sync window + # @param window_end_date [Date, nil] End of sync window + # @return [Array] Results with success status per account + def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) + return [] if accounts.empty? + + results = [] + accounts.visible.each do |account| + begin + account.sync_later( + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: window_end_date + ) + results << { account_id: account.id, success: true } + rescue => e + Rails.logger.error "CoinstatsItem #{id} - Failed to schedule sync for wallet #{account.id}: #{e.message}" + results << { account_id: account.id, success: false, error: e.message } + end + end + + results + end + + # Persists raw API response for debugging and reprocessing. + # @param accounts_snapshot [Hash] Raw API response data + def upsert_coinstats_snapshot!(accounts_snapshot) + assign_attributes(raw_payload: accounts_snapshot) + save! + end + + # @return [Boolean] true if at least one account has been linked + def has_completed_initial_setup? + accounts.any? + end + + # @return [String] Human-readable summary of sync status + def sync_status_summary + total_accounts = total_accounts_count + linked_count = linked_accounts_count + unlinked_count = unlinked_accounts_count + + if total_accounts == 0 + I18n.t("coinstats_items.coinstats_item.sync_status.no_accounts") + elsif unlinked_count == 0 + I18n.t("coinstats_items.coinstats_item.sync_status.all_synced", count: linked_count) + else + I18n.t("coinstats_items.coinstats_item.sync_status.partial_sync", linked_count: linked_count, unlinked_count: unlinked_count) + end + end + + # @return [Integer] Number of accounts with provider links + def linked_accounts_count + coinstats_accounts.joins(:account_provider).count + end + + # @return [Integer] Number of accounts without provider links + def unlinked_accounts_count + coinstats_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).count + end + + # @return [Integer] Total number of coinstats accounts + def total_accounts_count + coinstats_accounts.count + end + + # @return [String] Display name for the CoinStats connection + def institution_display_name + name.presence || "CoinStats" + end + + # @return [Boolean] true if API key is set + def credentials_configured? + api_key.present? + end +end diff --git a/app/models/coinstats_item/importer.rb b/app/models/coinstats_item/importer.rb new file mode 100644 index 000000000..8cf4e21f3 --- /dev/null +++ b/app/models/coinstats_item/importer.rb @@ -0,0 +1,315 @@ +# Imports wallet data from CoinStats API for linked accounts. +# Fetches balances and transactions, then updates local records. +class CoinstatsItem::Importer + include CoinstatsTransactionIdentifiable + + attr_reader :coinstats_item, :coinstats_provider + + # @param coinstats_item [CoinstatsItem] Item containing accounts to import + # @param coinstats_provider [Provider::Coinstats] API client instance + def initialize(coinstats_item, coinstats_provider:) + @coinstats_item = coinstats_item + @coinstats_provider = coinstats_provider + end + + # Imports balance and transaction data for all linked accounts. + # @return [Hash] Result with :success, :accounts_updated, :transactions_imported + def import + Rails.logger.info "CoinstatsItem::Importer - Starting import for item #{coinstats_item.id}" + + # CoinStats works differently from bank providers - wallets are added manually + # via the setup_accounts flow. During sync, we just update existing linked accounts. + + # Get all linked coinstats accounts (ones with account_provider associations) + linked_accounts = coinstats_item.coinstats_accounts + .joins(:account_provider) + .includes(:account) + + if linked_accounts.empty? + Rails.logger.info "CoinstatsItem::Importer - No linked accounts to sync for item #{coinstats_item.id}" + return { success: true, accounts_updated: 0, transactions_imported: 0 } + end + + accounts_updated = 0 + accounts_failed = 0 + transactions_imported = 0 + + # Fetch balance data using bulk endpoint + bulk_balance_data = fetch_balances_for_accounts(linked_accounts) + + # Fetch transaction data using bulk endpoint + bulk_transactions_data = fetch_transactions_for_accounts(linked_accounts) + + linked_accounts.each do |coinstats_account| + begin + result = update_account(coinstats_account, bulk_balance_data: bulk_balance_data, bulk_transactions_data: bulk_transactions_data) + accounts_updated += 1 if result[:success] + transactions_imported += result[:transactions_count] || 0 + rescue => e + accounts_failed += 1 + Rails.logger.error "CoinstatsItem::Importer - Failed to update account #{coinstats_account.id}: #{e.message}" + end + end + + Rails.logger.info "CoinstatsItem::Importer - Updated #{accounts_updated} accounts (#{accounts_failed} failed), #{transactions_imported} transactions" + + { + success: accounts_failed == 0, + accounts_updated: accounts_updated, + accounts_failed: accounts_failed, + transactions_imported: transactions_imported + } + end + + private + + # Fetch balance data for all linked accounts using the bulk endpoint + # @param linked_accounts [Array] Accounts to fetch balances for + # @return [Array, nil] Bulk balance data, or nil on error + def fetch_balances_for_accounts(linked_accounts) + # Extract unique wallet addresses and blockchains + wallets = linked_accounts.filter_map do |account| + raw = account.raw_payload || {} + address = raw["address"] || raw[:address] + blockchain = raw["blockchain"] || raw[:blockchain] + next unless address.present? && blockchain.present? + + { address: address, blockchain: blockchain } + end.uniq { |w| [ w[:address].downcase, w[:blockchain].downcase ] } + + return nil if wallets.empty? + + Rails.logger.info "CoinstatsItem::Importer - Fetching balances for #{wallets.size} wallet(s) via bulk endpoint" + # Build comma-separated string in format "blockchain:address" + wallets_param = wallets.map { |w| "#{w[:blockchain]}:#{w[:address]}" }.join(",") + response = coinstats_provider.get_wallet_balances(wallets_param) + response.success? ? response.data : nil + rescue => e + Rails.logger.warn "CoinstatsItem::Importer - Bulk balance fetch failed: #{e.message}" + nil + end + + # Fetch transaction data for all linked accounts using the bulk endpoint + # @param linked_accounts [Array] Accounts to fetch transactions for + # @return [Array, nil] Bulk transaction data, or nil on error + def fetch_transactions_for_accounts(linked_accounts) + # Extract unique wallet addresses and blockchains + wallets = linked_accounts.filter_map do |account| + raw = account.raw_payload || {} + address = raw["address"] || raw[:address] + blockchain = raw["blockchain"] || raw[:blockchain] + next unless address.present? && blockchain.present? + + { address: address, blockchain: blockchain } + end.uniq { |w| [ w[:address].downcase, w[:blockchain].downcase ] } + + return nil if wallets.empty? + + Rails.logger.info "CoinstatsItem::Importer - Fetching transactions for #{wallets.size} wallet(s) via bulk endpoint" + # Build comma-separated string in format "blockchain:address" + wallets_param = wallets.map { |w| "#{w[:blockchain]}:#{w[:address]}" }.join(",") + response = coinstats_provider.get_wallet_transactions(wallets_param) + response.success? ? response.data : nil + rescue => e + Rails.logger.warn "CoinstatsItem::Importer - Bulk transaction fetch failed: #{e.message}" + nil + end + + # Updates a single account with balance and transaction data. + # @param coinstats_account [CoinstatsAccount] Account to update + # @param bulk_balance_data [Array, nil] Pre-fetched balance data + # @param bulk_transactions_data [Array, nil] Pre-fetched transaction data + # @return [Hash] Result with :success and :transactions_count + def update_account(coinstats_account, bulk_balance_data:, bulk_transactions_data:) + # Get the wallet address and blockchain from the raw payload + raw = coinstats_account.raw_payload || {} + address = raw["address"] || raw[:address] + blockchain = raw["blockchain"] || raw[:blockchain] + + unless address.present? && blockchain.present? + Rails.logger.warn "CoinstatsItem::Importer - Missing address or blockchain for account #{coinstats_account.id}. Address: #{address.inspect}, Blockchain: #{blockchain.inspect}" + return { success: false, error: "Missing address or blockchain" } + end + + # Extract balance data for this specific wallet from the bulk response + balance_data = if bulk_balance_data.present? + coinstats_provider.extract_wallet_balance(bulk_balance_data, address, blockchain) + else + [] + end + + # Update the coinstats account with new balance data + coinstats_account.upsert_coinstats_snapshot!(normalize_balance_data(balance_data, coinstats_account)) + + # Extract and merge transactions from bulk response + transactions_count = fetch_and_merge_transactions(coinstats_account, address, blockchain, bulk_transactions_data) + + { success: true, transactions_count: transactions_count } + end + + # Extracts and merges new transactions for an account. + # Deduplicates by transaction ID to avoid duplicate imports. + # @param coinstats_account [CoinstatsAccount] Account to update + # @param address [String] Wallet address + # @param blockchain [String] Blockchain identifier + # @param bulk_transactions_data [Array, nil] Pre-fetched transaction data + # @return [Integer] Number of relevant transactions found + def fetch_and_merge_transactions(coinstats_account, address, blockchain, bulk_transactions_data) + # Extract transactions for this specific wallet from the bulk response + transactions_data = if bulk_transactions_data.present? + coinstats_provider.extract_wallet_transactions(bulk_transactions_data, address, blockchain) + else + [] + end + + new_transactions = transactions_data.is_a?(Array) ? transactions_data : (transactions_data[:result] || []) + return 0 if new_transactions.empty? + + # Filter transactions to only include those relevant to this coin/token + coin_id = coinstats_account.account_id + relevant_transactions = filter_transactions_by_coin(new_transactions, coin_id) + return 0 if relevant_transactions.empty? + + # Get existing transactions (already extracted as array) + existing_transactions = coinstats_account.raw_transactions_payload.to_a + + # Build a set of existing transaction IDs to avoid duplicates + existing_ids = existing_transactions.map { |tx| extract_coinstats_transaction_id(tx) }.compact.to_set + + # Filter to only new transactions + transactions_to_add = relevant_transactions.select do |tx| + tx_id = extract_coinstats_transaction_id(tx) + tx_id.present? && !existing_ids.include?(tx_id) + end + + if transactions_to_add.any? + # Merge new transactions with existing ones + merged_transactions = existing_transactions + transactions_to_add + coinstats_account.upsert_coinstats_transactions_snapshot!(merged_transactions) + Rails.logger.info "CoinstatsItem::Importer - Added #{transactions_to_add.count} new transactions for account #{coinstats_account.id}" + end + + relevant_transactions.count + end + + # Filter transactions to only include those relevant to a specific coin + # Transactions can be matched by: + # - coinData.symbol matching the coin (case-insensitive) + # - transactions[].items[].coin.id matching the coin_id + # @param transactions [Array] Array of transaction objects + # @param coin_id [String] The coin ID to filter by (e.g., "chainlink", "ethereum") + # @return [Array] Filtered transactions + def filter_transactions_by_coin(transactions, coin_id) + return [] if coin_id.blank? + + coin_id_downcase = coin_id.to_s.downcase + + transactions.select do |tx| + tx = tx.with_indifferent_access + + # Check nested transactions items for coin match + inner_transactions = tx[:transactions] || [] + inner_transactions.any? do |inner_tx| + inner_tx = inner_tx.with_indifferent_access + items = inner_tx[:items] || [] + items.any? do |item| + item = item.with_indifferent_access + coin = item[:coin] + next false unless coin.present? + + coin = coin.with_indifferent_access + coin[:id]&.downcase == coin_id_downcase + end + end + end + end + + # Normalizes API balance data to a consistent schema for storage. + # @param balance_data [Array] Raw token balances from API + # @param coinstats_account [CoinstatsAccount] Account for context + # @return [Hash] Normalized snapshot with id, balance, address, etc. + def normalize_balance_data(balance_data, coinstats_account) + # CoinStats get_wallet_balance returns an array of token balances directly + # Normalize it to match our expected schema + # Preserve existing address/blockchain from raw_payload + existing_raw = coinstats_account.raw_payload || {} + + # Find the matching token for this account to extract id, logo, and balance + matching_token = find_matching_token(balance_data, coinstats_account) + + # Calculate balance from the matching token only, not all tokens + # Each coinstats_account represents a single token/coin in the wallet + token_balance = calculate_token_balance(matching_token) + + { + # Use existing account_id if set, otherwise extract from matching token + id: coinstats_account.account_id.presence || matching_token&.dig(:coinId) || matching_token&.dig(:id), + name: coinstats_account.name, + balance: token_balance, + currency: "USD", # CoinStats returns values in USD + address: existing_raw["address"] || existing_raw[:address], + blockchain: existing_raw["blockchain"] || existing_raw[:blockchain], + # Extract logo from the matching token + institution_logo: matching_token&.dig(:imgUrl), + # Preserve original data + raw_balance_data: balance_data + } + end + + # Finds the token in balance_data that matches this account. + # Matches by account_id (coinId) first, then falls back to name. + # @param balance_data [Array] Token balances from API + # @param coinstats_account [CoinstatsAccount] Account to match + # @return [Hash, nil] Matching token data or nil + def find_matching_token(balance_data, coinstats_account) + tokens = normalize_tokens(balance_data).map(&:with_indifferent_access) + return nil if tokens.empty? + + # First try to match by account_id (coinId) if available + if coinstats_account.account_id.present? + account_id = coinstats_account.account_id.to_s + matching = tokens.find do |token| + token_id = (token[:coinId] || token[:id])&.to_s + token_id == account_id + end + return matching if matching + end + + # Fall back to matching by name (handles legacy accounts without account_id) + account_name = coinstats_account.name&.downcase + return nil if account_name.blank? + + tokens.find do |token| + token_name = token[:name]&.to_s&.downcase + token_symbol = token[:symbol]&.to_s&.downcase + + # Match if account name contains the token name or symbol, or vice versa + account_name.include?(token_name) || token_name.include?(account_name) || + (token_symbol.present? && (account_name.include?(token_symbol) || token_symbol == account_name)) + end + end + + # Normalizes various response formats to an array of tokens. + # @param balance_data [Array, Hash, nil] Raw balance response + # @return [Array] Array of token hashes + def normalize_tokens(balance_data) + if balance_data.is_a?(Array) + balance_data + elsif balance_data.is_a?(Hash) + balance_data[:result] || balance_data[:tokens] || [] + else + [] + end + end + + # Calculates USD balance from token amount and price. + # @param token [Hash, nil] Token with :amount/:balance and :price/:priceUsd + # @return [Float] Balance in USD (0 if token is nil) + def calculate_token_balance(token) + return 0 if token.blank? + + amount = token[:amount] || token[:balance] || 0 + price = token[:price] || token[:priceUsd] || 0 + (amount.to_f * price.to_f) + end +end diff --git a/app/models/coinstats_item/provided.rb b/app/models/coinstats_item/provided.rb new file mode 100644 index 000000000..08d859a52 --- /dev/null +++ b/app/models/coinstats_item/provided.rb @@ -0,0 +1,9 @@ +module CoinstatsItem::Provided + extend ActiveSupport::Concern + + def coinstats_provider + return nil unless credentials_configured? + + Provider::Coinstats.new(api_key) + end +end diff --git a/app/models/coinstats_item/sync_complete_event.rb b/app/models/coinstats_item/sync_complete_event.rb new file mode 100644 index 000000000..dfd6f5f8c --- /dev/null +++ b/app/models/coinstats_item/sync_complete_event.rb @@ -0,0 +1,29 @@ +# Broadcasts Turbo Stream updates when a CoinStats sync completes. +# Updates account views and notifies the family of sync completion. +class CoinstatsItem::SyncCompleteEvent + attr_reader :coinstats_item + + # @param coinstats_item [CoinstatsItem] The item that completed syncing + def initialize(coinstats_item) + @coinstats_item = coinstats_item + end + + # Broadcasts sync completion to update UI components. + def broadcast + # Update UI with latest account data + coinstats_item.accounts.each do |account| + account.broadcast_sync_complete + end + + # Update the CoinStats item view + coinstats_item.broadcast_replace_to( + coinstats_item.family, + target: "coinstats_item_#{coinstats_item.id}", + partial: "coinstats_items/coinstats_item", + locals: { coinstats_item: coinstats_item } + ) + + # Let family handle sync notifications + coinstats_item.family.broadcast_sync_complete + end +end diff --git a/app/models/coinstats_item/syncer.rb b/app/models/coinstats_item/syncer.rb new file mode 100644 index 000000000..459fa0926 --- /dev/null +++ b/app/models/coinstats_item/syncer.rb @@ -0,0 +1,61 @@ +# Orchestrates the sync process for a CoinStats connection. +# Imports data, processes holdings, and schedules account syncs. +class CoinstatsItem::Syncer + attr_reader :coinstats_item + + # @param coinstats_item [CoinstatsItem] Item to sync + def initialize(coinstats_item) + @coinstats_item = coinstats_item + end + + # Runs the full sync workflow: import, process, and schedule. + # @param sync [Sync] Sync record for status tracking + def perform_sync(sync) + # Phase 1: Import data from CoinStats API + sync.update!(status_text: "Importing wallets from CoinStats...") if sync.respond_to?(:status_text) + coinstats_item.import_latest_coinstats_data + + # Phase 2: Check account setup status and collect sync statistics + sync.update!(status_text: "Checking wallet configuration...") if sync.respond_to?(:status_text) + total_accounts = coinstats_item.coinstats_accounts.count + + linked_accounts = coinstats_item.coinstats_accounts.joins(:account_provider).joins(:account).merge(Account.visible) + unlinked_accounts = coinstats_item.coinstats_accounts.left_joins(:account_provider).where(account_providers: { id: nil }) + + sync_stats = { + total_accounts: total_accounts, + linked_accounts: linked_accounts.count, + unlinked_accounts: unlinked_accounts.count + } + + if unlinked_accounts.any? + coinstats_item.update!(pending_account_setup: true) + sync.update!(status_text: "#{unlinked_accounts.count} wallets need setup...") if sync.respond_to?(:status_text) + else + coinstats_item.update!(pending_account_setup: false) + end + + # Phase 3: Process holdings for linked accounts only + if linked_accounts.any? + sync.update!(status_text: "Processing holdings...") if sync.respond_to?(:status_text) + coinstats_item.process_accounts + + # Phase 4: Schedule balance calculations for linked accounts + sync.update!(status_text: "Calculating balances...") if sync.respond_to?(:status_text) + coinstats_item.schedule_account_syncs( + parent_sync: sync, + window_start_date: sync.window_start_date, + window_end_date: sync.window_end_date + ) + end + + if sync.respond_to?(:sync_stats) + sync.update!(sync_stats: sync_stats) + end + end + + # Hook called after sync completion. Currently a no-op. + def perform_post_sync + # no-op + end +end diff --git a/app/models/coinstats_item/unlinking.rb b/app/models/coinstats_item/unlinking.rb new file mode 100644 index 000000000..54a0a8b84 --- /dev/null +++ b/app/models/coinstats_item/unlinking.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Provides unlinking functionality for CoinStats items. +# Allows disconnecting provider accounts while preserving account data. +module CoinstatsItem::Unlinking + extend ActiveSupport::Concern + + # Removes all connections between this item and local accounts. + # Detaches AccountProvider links and nullifies associated Holdings. + # @param dry_run [Boolean] If true, returns results without making changes + # @return [Array] Results per account with :provider_account_id, :name, :provider_link_ids + def unlink_all!(dry_run: false) + results = [] + + coinstats_accounts.find_each do |provider_account| + links = AccountProvider.where(provider_type: CoinstatsAccount.name, provider_id: provider_account.id).to_a + link_ids = links.map(&:id) + result = { + provider_account_id: provider_account.id, + name: provider_account.name, + provider_link_ids: link_ids + } + results << result + + next if dry_run + + begin + ActiveRecord::Base.transaction do + # Detach holdings for any provider links found + if link_ids.any? + Holding.where(account_provider_id: link_ids).update_all(account_provider_id: nil) + end + + # Destroy all provider links + links.each do |ap| + ap.destroy! + end + end + rescue StandardError => e + Rails.logger.warn( + "CoinstatsItem Unlinker: failed to fully unlink provider account ##{provider_account.id} (links=#{link_ids.inspect}): #{e.class} - #{e.message}" + ) + # Record error for observability; continue with other accounts + result[:error] = e.message + end + end + + results + end +end diff --git a/app/models/coinstats_item/wallet_linker.rb b/app/models/coinstats_item/wallet_linker.rb new file mode 100644 index 000000000..a06a19a59 --- /dev/null +++ b/app/models/coinstats_item/wallet_linker.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +# Links a cryptocurrency wallet to CoinStats by fetching token balances +# and creating corresponding accounts for each token found. +class CoinstatsItem::WalletLinker + attr_reader :coinstats_item, :address, :blockchain + + Result = Struct.new(:success?, :created_count, :errors, keyword_init: true) + + # @param coinstats_item [CoinstatsItem] Parent item with API credentials + # @param address [String] Wallet address to link + # @param blockchain [String] Blockchain network identifier + def initialize(coinstats_item, address:, blockchain:) + @coinstats_item = coinstats_item + @address = address + @blockchain = blockchain + end + + # Fetches wallet balances and creates accounts for each token. + # @return [Result] Success status, created count, and any errors + def link + balance_data = fetch_balance_data + tokens = normalize_tokens(balance_data) + + return Result.new(success?: false, created_count: 0, errors: [ "No tokens found for wallet" ]) if tokens.empty? + + created_count = 0 + errors = [] + + tokens.each do |token_data| + result = create_account_from_token(token_data) + if result[:success] + created_count += 1 + else + errors << result[:error] + end + end + + # Trigger a sync if we created any accounts + coinstats_item.sync_later if created_count > 0 + + Result.new(success?: created_count > 0, created_count: created_count, errors: errors) + end + + private + + # Fetches balance data for this wallet from CoinStats API. + # @return [Array] Token balances for the wallet + def fetch_balance_data + provider = Provider::Coinstats.new(coinstats_item.api_key) + wallets_param = "#{blockchain}:#{address}" + response = provider.get_wallet_balances(wallets_param) + + return [] unless response.success? + + provider.extract_wallet_balance(response.data, address, blockchain) + end + + # Normalizes various balance data formats to an array of tokens. + # @param balance_data [Array, Hash, Object] Raw balance response + # @return [Array] Normalized array of token data + def normalize_tokens(balance_data) + if balance_data.is_a?(Array) + balance_data + elsif balance_data.is_a?(Hash) + balance_data[:result] || balance_data[:tokens] || [ balance_data ] + elsif balance_data.present? + [ balance_data ] + else + [] + end + end + + # Creates a CoinstatsAccount and linked Account for a token. + # @param token_data [Hash] Token balance data from API + # @return [Hash] Result with :success and optional :error + def create_account_from_token(token_data) + token = token_data.with_indifferent_access + account_name = build_account_name(token) + current_balance = calculate_balance(token) + token_id = (token[:coinId] || token[:id])&.to_s + + ActiveRecord::Base.transaction do + coinstats_account = coinstats_item.coinstats_accounts.create!( + name: account_name, + currency: "USD", + current_balance: current_balance, + account_id: token_id + ) + + # Store wallet metadata for future syncs + snapshot = build_snapshot(token, current_balance) + coinstats_account.upsert_coinstats_snapshot!(snapshot) + + account = coinstats_item.family.accounts.create!( + accountable: Crypto.new, + name: account_name, + balance: current_balance, + cash_balance: current_balance, + currency: coinstats_account.currency, + status: "active" + ) + + AccountProvider.create!(account: account, provider: coinstats_account) + + { success: true } + end + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e + Rails.logger.error("CoinstatsItem::WalletLinker - Failed to create account: #{e.message}") + { success: false, error: "Failed to create #{account_name || 'account'}: #{e.message}" } + rescue => e + Rails.logger.error("CoinstatsItem::WalletLinker - Unexpected error: #{e.class} - #{e.message}") + { success: false, error: "Unexpected error: #{e.message}" } + end + + # Builds a display name for the account from token and address. + # @param token [Hash] Token data with :name + # @return [String] Human-readable account name + def build_account_name(token) + token_name = token[:name].to_s.strip + truncated_address = address.present? ? "#{address.first(4)}...#{address.last(4)}" : nil + + if token_name.present? && truncated_address.present? + "#{token_name} (#{truncated_address})" + elsif token_name.present? + token_name + elsif truncated_address.present? + "#{blockchain.capitalize} (#{truncated_address})" + else + "Crypto Wallet" + end + end + + # Calculates USD balance from token amount and price. + # @param token [Hash] Token data with :amount/:balance and :price + # @return [Float] Balance in USD + def calculate_balance(token) + amount = token[:amount] || token[:balance] || token[:current_balance] || 0 + price = token[:price] || 0 + (amount.to_f * price.to_f) + end + + # Builds snapshot hash for storing in CoinstatsAccount. + # @param token [Hash] Token data from API + # @param current_balance [Float] Calculated USD balance + # @return [Hash] Snapshot with balance, address, and metadata + def build_snapshot(token, current_balance) + token.to_h.merge( + id: (token[:coinId] || token[:id])&.to_s, + balance: current_balance, + currency: "USD", + address: address, + blockchain: blockchain, + institution_logo: token[:imgUrl] + ) + end +end diff --git a/app/models/concerns/coinstats_transaction_identifiable.rb b/app/models/concerns/coinstats_transaction_identifiable.rb new file mode 100644 index 000000000..01fdfd6f9 --- /dev/null +++ b/app/models/concerns/coinstats_transaction_identifiable.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +# Shared logic for extracting unique transaction IDs from CoinStats API responses. +# Different blockchains return transaction IDs in different locations: +# - Ethereum/EVM: hash.id (transaction hash) +# - Bitcoin/UTXO: transactions[0].items[0].id +module CoinstatsTransactionIdentifiable + extend ActiveSupport::Concern + + private + + # Extracts a unique transaction ID from CoinStats transaction data. + # Handles different blockchain formats and generates fallback IDs. + # @param transaction_data [Hash] Raw transaction data from API + # @return [String, nil] Unique transaction identifier or nil + def extract_coinstats_transaction_id(transaction_data) + tx = transaction_data.is_a?(Hash) ? transaction_data.with_indifferent_access : {} + + # Try hash.id first (Ethereum/EVM chains) + hash_id = tx.dig(:hash, :id) + return hash_id if hash_id.present? + + # Try transactions[0].items[0].id (Bitcoin/UTXO chains) + item_id = tx.dig(:transactions, 0, :items, 0, :id) + return item_id if item_id.present? + + # Fallback: generate ID from multiple fields to reduce collision risk. + # Include as many distinguishing fields as possible since transactions + # with same date/type/amount are common (DCA, recurring purchases, batch trades). + fallback_id = build_fallback_transaction_id(tx) + return fallback_id if fallback_id.present? + + nil + end + + # Builds a fallback transaction ID from available fields. + # Uses a hash digest of combined fields to handle varying field availability + # while maintaining uniqueness across similar transactions. + # @param tx [HashWithIndifferentAccess] Transaction data + # @return [String, nil] Generated fallback ID or nil if insufficient data + def build_fallback_transaction_id(tx) + date = tx[:date] + type = tx[:type] + amount = tx.dig(:coinData, :count) + + # Require minimum fields for a valid fallback + return nil unless date.present? && type.present? && amount.present? + + # Collect additional distinguishing fields. + # Only use stable transaction data—avoid market-dependent values + # (currentValue, totalWorth, profit) that can change between API calls. + components = [ + date, + type, + amount, + tx.dig(:coinData, :symbol), + tx.dig(:fee, :count), + tx.dig(:fee, :coin, :symbol), + tx.dig(:transactions, 0, :action), + tx.dig(:transactions, 0, :items, 0, :coin, :id), + tx.dig(:transactions, 0, :items, 0, :count) + ].compact + + # Generate a hash digest for a fixed-length, collision-resistant ID + content = components.join("|") + "fallback_#{Digest::SHA256.hexdigest(content)[0, 16]}" + end +end diff --git a/app/models/concerns/currency_normalizable.rb b/app/models/concerns/currency_normalizable.rb index 3d1f62331..743392912 100644 --- a/app/models/concerns/currency_normalizable.rb +++ b/app/models/concerns/currency_normalizable.rb @@ -1,10 +1,10 @@ # Provides currency normalization and validation for provider data imports # # This concern provides a shared method to parse and normalize currency codes -# from external providers (Plaid, SimpleFIN, LunchFlow), ensuring: +# from external providers (Plaid, SimpleFIN, LunchFlow, Enable Banking), ensuring: # - Consistent uppercase formatting (e.g., "eur" -> "EUR") -# - Validation of 3-letter ISO currency codes -# - Proper handling of nil, empty, and invalid values +# - Validation against Money gem's known currencies (not just 3-letter format) +# - Proper handling of nil, empty, and invalid values (e.g., "XXX") # # Usage: # include CurrencyNormalizable @@ -23,6 +23,7 @@ module CurrencyNormalizable # parse_currency("usd") # => "USD" # parse_currency("EUR") # => "EUR" # parse_currency(" gbp ") # => "GBP" + # parse_currency("XXX") # => nil (not a valid Money currency) # parse_currency("invalid") # => nil (logs warning) # parse_currency(nil) # => nil # parse_currency("") # => nil @@ -33,8 +34,15 @@ module CurrencyNormalizable # Normalize to uppercase 3-letter code normalized = currency_value.to_s.strip.upcase - # Validate it's a reasonable currency code (3 letters) - if normalized.match?(/\A[A-Z]{3}\z/) + # Validate it's a 3-letter format first + unless normalized.match?(/\A[A-Z]{3}\z/) + log_invalid_currency(currency_value) + return nil + end + + # Validate against Money gem's known currencies + # This catches codes like "XXX" which are 3 letters but not valid for monetary operations + if valid_money_currency?(normalized) normalized else log_invalid_currency(currency_value) @@ -42,6 +50,17 @@ module CurrencyNormalizable end end + # Check if a currency code is valid in the Money gem + # + # @param code [String] Uppercase 3-letter currency code + # @return [Boolean] true if the Money gem recognizes this currency + def valid_money_currency?(code) + Money::Currency.new(code) + true + rescue Money::Currency::UnknownCurrencyError + false + end + # Log warning for invalid currency codes # Override this method in including classes to provide context-specific logging def log_invalid_currency(currency_value) diff --git a/app/models/concerns/simplefin_numeric_helpers.rb b/app/models/concerns/simplefin_numeric_helpers.rb new file mode 100644 index 000000000..542117ceb --- /dev/null +++ b/app/models/concerns/simplefin_numeric_helpers.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module SimplefinNumericHelpers + extend ActiveSupport::Concern + + private + + def to_decimal(value) + return BigDecimal("0") if value.nil? + case value + when BigDecimal then value + when String then BigDecimal(value) rescue BigDecimal("0") + when Numeric then BigDecimal(value.to_s) + else + BigDecimal("0") + end + end + + def same_sign?(a, b) + (a.positive? && b.positive?) || (a.negative? && b.negative?) + end +end diff --git a/app/models/concerns/sync_stats/collector.rb b/app/models/concerns/sync_stats/collector.rb new file mode 100644 index 000000000..8333a0d8d --- /dev/null +++ b/app/models/concerns/sync_stats/collector.rb @@ -0,0 +1,198 @@ +# frozen_string_literal: true + +# SyncStats::Collector provides shared methods for collecting sync statistics +# across different provider syncers. +# +# This concern standardizes the stat collection interface so all providers +# can report consistent sync summaries. +# +# @example Include in a syncer class +# class PlaidItem::Syncer +# include SyncStats::Collector +# +# def perform_sync(sync) +# # ... sync logic ... +# collect_setup_stats(sync, provider_accounts: plaid_item.plaid_accounts) +# collect_transaction_stats(sync, account_ids: account_ids, source: "plaid") +# # ... +# end +# end +# +module SyncStats + module Collector + extend ActiveSupport::Concern + + # Collects account setup statistics (total, linked, unlinked counts). + # + # @param sync [Sync] The sync record to update + # @param provider_accounts [ActiveRecord::Relation] The provider accounts (e.g., SimplefinAccount, PlaidAccount) + # @param linked_check [Proc, nil] Optional proc to check if an account is linked. If nil, uses default logic. + # @return [Hash] The setup stats that were collected + def collect_setup_stats(sync, provider_accounts:, linked_check: nil) + return {} unless sync.respond_to?(:sync_stats) + + total_accounts = provider_accounts.count + + # Count linked accounts - either via custom check or default association check + linked_count = if linked_check + provider_accounts.count { |pa| linked_check.call(pa) } + else + # Default: check for current_account method or account association + provider_accounts.count do |pa| + (pa.respond_to?(:current_account) && pa.current_account.present?) || + (pa.respond_to?(:account) && pa.account.present?) + end + end + + unlinked_count = total_accounts - linked_count + + setup_stats = { + "total_accounts" => total_accounts, + "linked_accounts" => linked_count, + "unlinked_accounts" => unlinked_count + } + + merge_sync_stats(sync, setup_stats) + setup_stats + end + + # Collects transaction statistics (imported, updated, seen, skipped). + # + # @param sync [Sync] The sync record to update + # @param account_ids [Array] The account IDs to count transactions for + # @param source [String] The transaction source (e.g., "simplefin", "plaid") + # @param window_start [Time, nil] Start of the sync window (defaults to sync.created_at or 30 minutes ago) + # @param window_end [Time, nil] End of the sync window (defaults to Time.current) + # @return [Hash] The transaction stats that were collected + def collect_transaction_stats(sync, account_ids:, source:, window_start: nil, window_end: nil) + return {} unless sync.respond_to?(:sync_stats) + return {} if account_ids.empty? + + window_start ||= sync.created_at || 30.minutes.ago + window_end ||= Time.current + + tx_scope = Entry.where(account_id: account_ids, source: source, entryable_type: "Transaction") + tx_imported = tx_scope.where(created_at: window_start..window_end).count + tx_updated = tx_scope.where(updated_at: window_start..window_end) + .where.not(created_at: window_start..window_end).count + tx_seen = tx_imported + tx_updated + + tx_stats = { + "tx_imported" => tx_imported, + "tx_updated" => tx_updated, + "tx_seen" => tx_seen, + "window_start" => window_start.iso8601, + "window_end" => window_end.iso8601 + } + + merge_sync_stats(sync, tx_stats) + tx_stats + end + + # Collects holdings statistics. + # + # @param sync [Sync] The sync record to update + # @param holdings_count [Integer] The number of holdings found/processed + # @param label [String] The label for the stat ("found" or "processed") + # @return [Hash] The holdings stats that were collected + def collect_holdings_stats(sync, holdings_count:, label: "found") + return {} unless sync.respond_to?(:sync_stats) + + key = label == "processed" ? "holdings_processed" : "holdings_found" + holdings_stats = { key => holdings_count } + + merge_sync_stats(sync, holdings_stats) + holdings_stats + end + + # Collects health/error statistics. + # + # @param sync [Sync] The sync record to update + # @param errors [Array, nil] Array of error objects with :message and optional :category + # @param rate_limited [Boolean] Whether the sync was rate limited + # @param rate_limited_at [Time, nil] When rate limiting occurred + # @return [Hash] The health stats that were collected + def collect_health_stats(sync, errors: nil, rate_limited: false, rate_limited_at: nil) + return {} unless sync.respond_to?(:sync_stats) + + health_stats = { + "import_started" => true + } + + if errors.present? + health_stats["errors"] = errors.map do |e| + e.is_a?(Hash) ? e.stringify_keys : { "message" => e.to_s } + end + health_stats["total_errors"] = errors.size + else + health_stats["total_errors"] = 0 + end + + if rate_limited + health_stats["rate_limited"] = true + health_stats["rate_limited_at"] = rate_limited_at&.iso8601 + end + + merge_sync_stats(sync, health_stats) + health_stats + end + + # Collects data quality warnings and notices. + # + # @param sync [Sync] The sync record to update + # @param warnings [Integer] Number of data warnings + # @param notices [Integer] Number of notices + # @param details [Array] Array of detail objects with :message and optional :severity + # @return [Hash] The data quality stats that were collected + def collect_data_quality_stats(sync, warnings: 0, notices: 0, details: []) + return {} unless sync.respond_to?(:sync_stats) + + quality_stats = { + "data_warnings" => warnings, + "notices" => notices + } + + if details.present? + quality_stats["data_quality_details"] = details.map do |d| + d.is_a?(Hash) ? d.stringify_keys : { "message" => d.to_s, "severity" => "info" } + end + end + + merge_sync_stats(sync, quality_stats) + quality_stats + end + + # Marks the sync as having started import (used for health indicator). + # + # @param sync [Sync] The sync record to update + def mark_import_started(sync) + return unless sync.respond_to?(:sync_stats) + + merge_sync_stats(sync, { "import_started" => true }) + end + + # Clears previous sync stats (useful at start of sync). + # + # @param sync [Sync] The sync record to update + def clear_sync_stats(sync) + return unless sync.respond_to?(:sync_stats) + + sync.update!(sync_stats: { "cleared_at" => Time.current.iso8601 }) + end + + private + + # Merges new stats into the existing sync_stats hash. + # + # @param sync [Sync] The sync record to update + # @param new_stats [Hash] The new stats to merge + def merge_sync_stats(sync, new_stats) + return unless sync.respond_to?(:sync_stats) + + existing = sync.sync_stats || {} + sync.update!(sync_stats: existing.merge(new_stats)) + rescue => e + Rails.logger.warn("SyncStats::Collector#merge_sync_stats failed: #{e.class} - #{e.message}") + end + end +end diff --git a/app/models/data_enrichment.rb b/app/models/data_enrichment.rb index c730e3693..0eb27ab7e 100644 --- a/app/models/data_enrichment.rb +++ b/app/models/data_enrichment.rb @@ -1,5 +1,5 @@ class DataEnrichment < ApplicationRecord belongs_to :enrichable, polymorphic: true - enum :source, { rule: "rule", plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking" } + enum :source, { rule: "rule", plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats" } end diff --git a/app/models/family.rb b/app/models/family.rb index 1df25797f..073574974 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -1,5 +1,5 @@ class Family < ApplicationRecord - include PlaidConnectable, SimplefinConnectable, LunchflowConnectable, EnableBankingConnectable, Syncable, AutoTransferMatchable, Subscribeable + include PlaidConnectable, SimplefinConnectable, LunchflowConnectable, EnableBankingConnectable, Syncable, AutoTransferMatchable, Subscribeable, CoinstatsConnectable DATE_FORMATS = [ [ "MM-DD-YYYY", "%m-%d-%Y" ], @@ -45,6 +45,15 @@ class Family < ApplicationRecord Merchant.where(id: merchant_ids) end + def available_merchants + assigned_ids = transactions.where.not(merchant_id: nil).pluck(:merchant_id).uniq + recently_unlinked_ids = FamilyMerchantAssociation + .where(family: self) + .recently_unlinked + .pluck(:merchant_id) + Merchant.where(id: (assigned_ids + recently_unlinked_ids).uniq) + end + def auto_categorize_transactions_later(transactions, rule_run_id: nil) AutoCategorizeJob.perform_later(self, transaction_ids: transactions.pluck(:id), rule_run_id: rule_run_id) end @@ -69,6 +78,10 @@ class Family < ApplicationRecord @income_statement ||= IncomeStatement.new(self) end + def investment_statement + @investment_statement ||= InvestmentStatement.new(self) + end + def eu? country != "US" && country != "CA" end diff --git a/app/models/family/auto_merchant_detector.rb b/app/models/family/auto_merchant_detector.rb index 667b15a68..c37a4f7b7 100644 --- a/app/models/family/auto_merchant_detector.rb +++ b/app/models/family/auto_merchant_detector.rb @@ -102,21 +102,33 @@ class Family::AutoMerchantDetector end def find_or_create_ai_merchant(auto_detection) - # Only use (source, name) for find_or_create since that's the uniqueness constraint - ProviderMerchant.find_or_create_by!( - source: "ai", - name: auto_detection.business_name - ) do |pm| - pm.website_url = auto_detection.business_url - if Setting.brand_fetch_client_id.present? - pm.logo_url = "#{default_logo_provider_url}/#{auto_detection.business_url}/icon/fallback/lettermark/w/40/h/40?c=#{Setting.brand_fetch_client_id}" - end + # Strategy 1: Find existing merchant by website_url (most reliable for deduplication) + if auto_detection.business_url.present? + existing = ProviderMerchant.find_by(website_url: auto_detection.business_url) + return existing if existing end + + # Strategy 2: Find by exact name match + existing = ProviderMerchant.find_by(source: "ai", name: auto_detection.business_name) + return existing if existing + + # Strategy 3: Create new merchant + ProviderMerchant.create!( + source: "ai", + name: auto_detection.business_name, + website_url: auto_detection.business_url, + logo_url: build_logo_url(auto_detection.business_url) + ) rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique # Race condition: another process created the merchant between our find and create ProviderMerchant.find_by(source: "ai", name: auto_detection.business_name) end + def build_logo_url(business_url) + return nil unless Setting.brand_fetch_client_id.present? && business_url.present? + "#{default_logo_provider_url}/#{business_url}/icon/fallback/lettermark/w/40/h/40?c=#{Setting.brand_fetch_client_id}" + end + def enhance_provider_merchant(merchant, auto_detection) updates = {} diff --git a/app/models/family/auto_transfer_matchable.rb b/app/models/family/auto_transfer_matchable.rb index 8566cbdcf..98c70775e 100644 --- a/app/models/family/auto_transfer_matchable.rb +++ b/app/models/family/auto_transfer_matchable.rb @@ -1,5 +1,5 @@ module Family::AutoTransferMatchable - def transfer_match_candidates + def transfer_match_candidates(date_window: 4) Entry.select([ "inflow_candidates.entryable_id as inflow_transaction_id", "outflow_candidates.entryable_id as outflow_transaction_id", @@ -10,7 +10,7 @@ module Family::AutoTransferMatchable inflow_candidates.amount < 0 AND outflow_candidates.amount > 0 AND inflow_candidates.account_id <> outflow_candidates.account_id AND - inflow_candidates.date BETWEEN outflow_candidates.date - 4 AND outflow_candidates.date + 4 + inflow_candidates.date BETWEEN outflow_candidates.date - #{date_window.to_i} AND outflow_candidates.date + #{date_window.to_i} ) ").joins(" LEFT JOIN transfers existing_transfers ON ( diff --git a/app/models/family/coinstats_connectable.rb b/app/models/family/coinstats_connectable.rb new file mode 100644 index 000000000..dc471fb6d --- /dev/null +++ b/app/models/family/coinstats_connectable.rb @@ -0,0 +1,35 @@ +# Adds CoinStats connection capabilities to Family. +# Allows families to create and manage CoinStats API connections. +module Family::CoinstatsConnectable + extend ActiveSupport::Concern + + included do + has_many :coinstats_items, dependent: :destroy + end + + # @return [Boolean] Whether the family can create CoinStats connections + def can_connect_coinstats? + # Families can configure their own Coinstats credentials + true + end + + # Creates a new CoinStats connection and triggers initial sync. + # @param api_key [String] CoinStats API key + # @param item_name [String, nil] Optional display name for the connection + # @return [CoinstatsItem] The created connection + def create_coinstats_item!(api_key:, item_name: nil) + coinstats_item = coinstats_items.create!( + name: item_name || "CoinStats Connection", + api_key: api_key + ) + + coinstats_item.sync_later + + coinstats_item + end + + # @return [Boolean] Whether the family has any configured CoinStats connections + def has_coinstats_credentials? + coinstats_items.where.not(api_key: nil).exists? + end +end diff --git a/app/models/family/syncer.rb b/app/models/family/syncer.rb index a16c8d6e1..862102509 100644 --- a/app/models/family/syncer.rb +++ b/app/models/family/syncer.rb @@ -10,7 +10,7 @@ class Family::Syncer family.sync_trial_status! Rails.logger.info("Applying rules for family #{family.id}") - family.rules.each do |rule| + family.rules.where(active: true).each do |rule| rule.apply_later end diff --git a/app/models/family_merchant_association.rb b/app/models/family_merchant_association.rb new file mode 100644 index 000000000..69a6f1adb --- /dev/null +++ b/app/models/family_merchant_association.rb @@ -0,0 +1,6 @@ +class FamilyMerchantAssociation < ApplicationRecord + belongs_to :family + belongs_to :merchant + + scope :recently_unlinked, -> { where(unlinked_at: 30.days.ago..).where.not(unlinked_at: nil) } +end diff --git a/app/models/holding.rb b/app/models/holding.rb index d3c117b4e..0b7f602af 100644 --- a/app/models/holding.rb +++ b/app/models/holding.rb @@ -28,32 +28,14 @@ class Holding < ApplicationRecord end # Basic approximation of cost-basis + # Uses pre-computed cost_basis if available (set during materialization), + # otherwise falls back to calculating from trades def avg_cost - trades = account.trades - .with_entry - .joins(ActiveRecord::Base.sanitize_sql_array([ - "LEFT JOIN exchange_rates ON ( - exchange_rates.date = entries.date AND - exchange_rates.from_currency = trades.currency AND - exchange_rates.to_currency = ? - )", account.currency - ])) - .where(security_id: security.id) - .where("trades.qty > 0 AND entries.date <= ?", date) + # Use stored cost_basis if available (eliminates N+1 queries) + return Money.new(cost_basis, currency) if cost_basis.present? - total_cost, total_qty = trades.pick( - Arel.sql("SUM(trades.price * trades.qty * COALESCE(exchange_rates.rate, 1))"), - Arel.sql("SUM(trades.qty)") - ) - - weighted_avg = - if total_qty && total_qty > 0 - total_cost / total_qty - else - price - end - - Money.new(weighted_avg || price, currency) + # Fallback to calculation for holdings without pre-computed cost_basis + calculate_avg_cost end def trend @@ -100,4 +82,32 @@ class Holding < ApplicationRecord current: amount_money, previous: start_amount end + + def calculate_avg_cost + trades = account.trades + .with_entry + .joins(ActiveRecord::Base.sanitize_sql_array([ + "LEFT JOIN exchange_rates ON ( + exchange_rates.date = entries.date AND + exchange_rates.from_currency = trades.currency AND + exchange_rates.to_currency = ? + )", account.currency + ])) + .where(security_id: security.id) + .where("trades.qty > 0 AND entries.date <= ?", date) + + total_cost, total_qty = trades.pick( + Arel.sql("SUM(trades.price * trades.qty * COALESCE(exchange_rates.rate, 1))"), + Arel.sql("SUM(trades.qty)") + ) + + weighted_avg = + if total_qty && total_qty > 0 + total_cost / total_qty + else + price + end + + Money.new(weighted_avg || price, currency) + end end diff --git a/app/models/holding/forward_calculator.rb b/app/models/holding/forward_calculator.rb index 43f91f7a6..ce490acba 100644 --- a/app/models/holding/forward_calculator.rb +++ b/app/models/holding/forward_calculator.rb @@ -3,6 +3,8 @@ class Holding::ForwardCalculator def initialize(account) @account = account + # Track cost basis per security: { security_id => { total_cost: BigDecimal, total_qty: BigDecimal } } + @cost_basis_tracker = Hash.new { |h, k| h[k] = { total_cost: BigDecimal("0"), total_qty: BigDecimal("0") } } end def calculate @@ -13,6 +15,7 @@ class Holding::ForwardCalculator account.start_date.upto(Date.current).each do |date| trades = portfolio_cache.get_trades(date: date) + update_cost_basis_tracker(trades) next_portfolio = transform_portfolio(current_portfolio, trades, direction: :forward) holdings += build_holdings(next_portfolio, date) current_portfolio = next_portfolio @@ -65,8 +68,36 @@ class Holding::ForwardCalculator qty: qty, price: price.price, currency: price.currency, - amount: qty * price.price + amount: qty * price.price, + cost_basis: cost_basis_for(security_id, price.currency) ) end.compact end + + # Updates cost basis tracker with buy trades (qty > 0) + # Uses weighted average cost method + def update_cost_basis_tracker(trade_entries) + trade_entries.each do |trade_entry| + trade = trade_entry.entryable + next unless trade.qty > 0 # Only track buys + + security_id = trade.security_id + tracker = @cost_basis_tracker[security_id] + + # Convert trade price to account currency if needed + trade_price = Money.new(trade.price, trade.currency) + converted_price = trade_price.exchange_to(account.currency, fallback_rate: 1).amount + + tracker[:total_cost] += converted_price * trade.qty + tracker[:total_qty] += trade.qty + end + end + + # Returns the current cost basis for a security, or nil if no buys recorded + def cost_basis_for(security_id, currency) + tracker = @cost_basis_tracker[security_id] + return nil if tracker[:total_qty].zero? + + tracker[:total_cost] / tracker[:total_qty] + end end diff --git a/app/models/holding/materializer.rb b/app/models/holding/materializer.rb index 6c1e8db19..fee335fd6 100644 --- a/app/models/holding/materializer.rb +++ b/app/models/holding/materializer.rb @@ -31,7 +31,7 @@ class Holding::Materializer account.holdings.upsert_all( @holdings.map { |h| h.attributes - .slice("date", "currency", "qty", "price", "amount", "security_id") + .slice("date", "currency", "qty", "price", "amount", "security_id", "cost_basis") .merge("account_id" => account.id, "updated_at" => current_time) }, unique_by: %i[account_id security_id date currency] ) diff --git a/app/models/holding/reverse_calculator.rb b/app/models/holding/reverse_calculator.rb index 656fc0d9b..2a4ea0375 100644 --- a/app/models/holding/reverse_calculator.rb +++ b/app/models/holding/reverse_calculator.rb @@ -8,6 +8,7 @@ class Holding::ReverseCalculator def calculate Rails.logger.tagged("Holding::ReverseCalculator") do + precompute_cost_basis holdings = calculate_holdings Holding.gapfill(holdings) end @@ -69,8 +70,47 @@ class Holding::ReverseCalculator qty: qty, price: price.price, currency: price.currency, - amount: qty * price.price + amount: qty * price.price, + cost_basis: cost_basis_for(security_id, date) ) end.compact end + + # Pre-compute cost basis for all securities at all dates using forward pass through trades + # Stores: { security_id => { date => cost_basis } } + def precompute_cost_basis + @cost_basis_by_date = Hash.new { |h, k| h[k] = {} } + tracker = Hash.new { |h, k| h[k] = { total_cost: BigDecimal("0"), total_qty: BigDecimal("0") } } + + trades = portfolio_cache.get_trades.sort_by(&:date) + trade_index = 0 + + account.start_date.upto(Date.current).each do |date| + # Process all trades up to and including this date + while trade_index < trades.size && trades[trade_index].date <= date + trade_entry = trades[trade_index] + trade = trade_entry.entryable + + if trade.qty > 0 # Only track buys + security_id = trade.security_id + trade_price = Money.new(trade.price, trade.currency) + converted_price = trade_price.exchange_to(account.currency, fallback_rate: 1).amount + + tracker[security_id][:total_cost] += converted_price * trade.qty + tracker[security_id][:total_qty] += trade.qty + end + trade_index += 1 + end + + # Store current cost basis snapshot for each security at this date + tracker.each do |security_id, data| + next if data[:total_qty].zero? + @cost_basis_by_date[security_id][date] = data[:total_cost] / data[:total_qty] + end + end + end + + def cost_basis_for(security_id, date) + @cost_basis_by_date.dig(security_id, date) + end end diff --git a/app/models/import.rb b/app/models/import.rb index 536209371..05b426573 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -2,6 +2,9 @@ class Import < ApplicationRecord MaxRowCountExceededError = Class.new(StandardError) MappingError = Class.new(StandardError) + MAX_CSV_SIZE = 10.megabytes + ALLOWED_MIME_TYPES = %w[text/csv text/plain application/vnd.ms-excel application/csv].freeze + TYPES = %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport].freeze SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative] SEPARATORS = [ [ "Comma (,)", "," ], [ "Semicolon (;)", ";" ] ].freeze @@ -36,6 +39,7 @@ class Import < ApplicationRecord validates :col_sep, inclusion: { in: SEPARATORS.map(&:last) } validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }, allow_nil: true validates :number_format, presence: true, inclusion: { in: NUMBER_FORMATS.keys } + validate :account_belongs_to_family has_many :rows, dependent: :destroy has_many :mappings, dependent: :destroy @@ -110,7 +114,7 @@ class Import < ApplicationRecord def dry_run mappings = { - transactions: rows.count, + transactions: rows_count, categories: Import::CategoryMapping.for_import(self).creational.count, tags: Import::TagMapping.for_import(self).creational.count } @@ -152,6 +156,7 @@ class Import < ApplicationRecord end rows.insert_all!(mapped_rows) + update_column(:rows_count, rows.count) end def sync_mappings @@ -181,7 +186,7 @@ class Import < ApplicationRecord end def configured? - uploaded? && rows.any? + uploaded? && rows_count > 0 end def cleaned? @@ -232,7 +237,7 @@ class Import < ApplicationRecord private def row_count_exceeded? - rows.count > max_row_count + rows_count > max_row_count end def import! @@ -288,4 +293,11 @@ class Import < ApplicationRecord def set_default_number_format self.number_format ||= "1,234.56" # Default to US/UK format end + + def account_belongs_to_family + return if account.nil? + return if account.family_id == family_id + + errors.add(:account, "must belong to your family") + end end diff --git a/app/models/import/row.rb b/app/models/import/row.rb index ef16d26e5..26525b6f4 100644 --- a/app/models/import/row.rb +++ b/app/models/import/row.rb @@ -1,5 +1,5 @@ class Import::Row < ApplicationRecord - belongs_to :import + belongs_to :import, counter_cache: true validates :amount, numericality: true, allow_blank: true validates :currency, presence: true diff --git a/app/models/income_statement.rb b/app/models/income_statement.rb index b30b352f0..6c4e16b90 100644 --- a/app/models/income_statement.rb +++ b/app/models/income_statement.rb @@ -11,10 +11,10 @@ class IncomeStatement @family = family end - def totals(transactions_scope: nil) + def totals(transactions_scope: nil, date_range:) transactions_scope ||= family.transactions.visible - result = totals_query(transactions_scope: transactions_scope) + result = totals_query(transactions_scope: transactions_scope, date_range: date_range) total_income = result.select { |t| t.classification == "income" }.sum(&:total) total_expense = result.select { |t| t.classification == "expense" }.sum(&:total) @@ -64,17 +64,26 @@ class IncomeStatement end def build_period_total(classification:, period:) - totals = totals_query(transactions_scope: family.transactions.visible.in_period(period)).select { |t| t.classification == classification } + totals = totals_query(transactions_scope: family.transactions.visible.in_period(period), date_range: period.date_range).select { |t| t.classification == classification } classification_total = totals.sum(&:total) uncategorized_category = family.categories.uncategorized + other_investments_category = family.categories.other_investments - category_totals = [ *categories, uncategorized_category ].map do |category| + category_totals = [ *categories, uncategorized_category, other_investments_category ].map do |category| subcategory = categories.find { |c| c.id == category.parent_id } - parent_category_total = totals.select { |t| t.category_id == category.id }&.sum(&:total) || 0 + parent_category_total = if category.uncategorized? + # Regular uncategorized: NULL category_id and NOT uncategorized investment + totals.select { |t| t.category_id.nil? && !t.is_uncategorized_investment }&.sum(&:total) || 0 + elsif category.other_investments? + # Other investments: NULL category_id AND is_uncategorized_investment + totals.select { |t| t.category_id.nil? && t.is_uncategorized_investment }&.sum(&:total) || 0 + else + totals.select { |t| t.category_id == category.id }&.sum(&:total) || 0 + end - children_totals = if category == uncategorized_category + children_totals = if category.synthetic? 0 else totals.select { |t| t.parent_category_id == category.id }&.sum(&:total) || 0 @@ -114,12 +123,12 @@ class IncomeStatement ]) { CategoryStats.new(family, interval:).call } end - def totals_query(transactions_scope:) + def totals_query(transactions_scope:, date_range:) sql_hash = Digest::MD5.hexdigest(transactions_scope.to_sql) Rails.cache.fetch([ "income_statement", "totals_query", family.id, sql_hash, family.entries_cache_version - ]) { Totals.new(family, transactions_scope: transactions_scope).call } + ]) { Totals.new(family, transactions_scope: transactions_scope, date_range: date_range).call } end def monetizable_currency diff --git a/app/models/income_statement/totals.rb b/app/models/income_statement/totals.rb index 858093a78..355212486 100644 --- a/app/models/income_statement/totals.rb +++ b/app/models/income_statement/totals.rb @@ -1,7 +1,11 @@ class IncomeStatement::Totals - def initialize(family, transactions_scope:) + def initialize(family, transactions_scope:, date_range:, include_trades: true) @family = family @transactions_scope = transactions_scope + @date_range = date_range + @include_trades = include_trades + + validate_date_range! end def call @@ -11,33 +15,54 @@ class IncomeStatement::Totals category_id: row["category_id"], classification: row["classification"], total: row["total"], - transactions_count: row["transactions_count"] + transactions_count: row["transactions_count"], + is_uncategorized_investment: row["is_uncategorized_investment"] ) end end private - TotalsRow = Data.define(:parent_category_id, :category_id, :classification, :total, :transactions_count) + TotalsRow = Data.define(:parent_category_id, :category_id, :classification, :total, :transactions_count, :is_uncategorized_investment) def query_sql ActiveRecord::Base.sanitize_sql_array([ - optimized_query_sql, + @include_trades ? combined_query_sql : transactions_only_query_sql, sql_params ]) end - # OPTIMIZED: Direct SUM aggregation without unnecessary time bucketing - # Eliminates CTE and intermediate date grouping for maximum performance - def optimized_query_sql + # Combined query that includes both transactions and trades + def combined_query_sql + <<~SQL + SELECT + category_id, + parent_category_id, + classification, + is_uncategorized_investment, + SUM(total) as total, + SUM(entry_count) as transactions_count + FROM ( + #{transactions_subquery_sql} + UNION ALL + #{trades_subquery_sql} + ) combined + GROUP BY category_id, parent_category_id, classification, is_uncategorized_investment; + SQL + end + + # Original transactions-only query (for backwards compatibility) + def transactions_only_query_sql <<~SQL SELECT c.id as category_id, c.parent_id as parent_category_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END as classification, ABS(SUM(ae.amount * COALESCE(er.rate, 1))) as total, - COUNT(ae.id) as transactions_count + COUNT(ae.id) as transactions_count, + false as is_uncategorized_investment FROM (#{@transactions_scope.to_sql}) at JOIN entries ae ON ae.entryable_id = at.id AND ae.entryable_type = 'Transaction' + JOIN accounts a ON a.id = ae.account_id LEFT JOIN categories c ON c.id = at.category_id LEFT JOIN exchange_rates er ON ( er.date = ae.date AND @@ -46,13 +71,82 @@ class IncomeStatement::Totals ) WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment') AND ae.excluded = false + AND a.family_id = :family_id + AND a.status IN ('draft', 'active') GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END; SQL end + def transactions_subquery_sql + <<~SQL + SELECT + c.id as category_id, + c.parent_id as parent_category_id, + CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END as classification, + ABS(SUM(ae.amount * COALESCE(er.rate, 1))) as total, + COUNT(ae.id) as entry_count, + false as is_uncategorized_investment + FROM (#{@transactions_scope.to_sql}) at + JOIN entries ae ON ae.entryable_id = at.id AND ae.entryable_type = 'Transaction' + JOIN accounts a ON a.id = ae.account_id + LEFT JOIN categories c ON c.id = at.category_id + LEFT JOIN exchange_rates er ON ( + er.date = ae.date AND + er.from_currency = ae.currency AND + er.to_currency = :target_currency + ) + WHERE at.kind NOT IN ('funds_movement', 'one_time', 'cc_payment') + AND ae.excluded = false + AND a.family_id = :family_id + AND a.status IN ('draft', 'active') + GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END + SQL + end + + def trades_subquery_sql + # Get trades for the same family and date range as transactions + # Trades without categories appear as "Uncategorized Investments" (separate from regular uncategorized) + <<~SQL + SELECT + c.id as category_id, + c.parent_id as parent_category_id, + CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END as classification, + ABS(SUM(ae.amount * COALESCE(er.rate, 1))) as total, + COUNT(ae.id) as entry_count, + CASE WHEN t.category_id IS NULL THEN true ELSE false END as is_uncategorized_investment + FROM trades t + JOIN entries ae ON ae.entryable_id = t.id AND ae.entryable_type = 'Trade' + JOIN accounts a ON a.id = ae.account_id + LEFT JOIN categories c ON c.id = t.category_id + LEFT JOIN exchange_rates er ON ( + er.date = ae.date AND + er.from_currency = ae.currency AND + er.to_currency = :target_currency + ) + WHERE a.family_id = :family_id + AND a.status IN ('draft', 'active') + AND ae.excluded = false + AND ae.date BETWEEN :start_date AND :end_date + GROUP BY c.id, c.parent_id, CASE WHEN ae.amount < 0 THEN 'income' ELSE 'expense' END, CASE WHEN t.category_id IS NULL THEN true ELSE false END + SQL + end + def sql_params { - target_currency: @family.currency + target_currency: @family.currency, + family_id: @family.id, + start_date: @date_range.begin, + end_date: @date_range.end } end + + def validate_date_range! + unless @date_range.is_a?(Range) + raise ArgumentError, "date_range must be a Range, got #{@date_range.class}" + end + + unless @date_range.begin.respond_to?(:to_date) && @date_range.end.respond_to?(:to_date) + raise ArgumentError, "date_range must contain date-like objects" + end + end end diff --git a/app/models/investment_statement.rb b/app/models/investment_statement.rb new file mode 100644 index 000000000..ee8daacfa --- /dev/null +++ b/app/models/investment_statement.rb @@ -0,0 +1,191 @@ +require "digest/md5" + +class InvestmentStatement + include Monetizable + + monetize :total_contributions, :total_dividends, :total_interest, :unrealized_gains + + attr_reader :family + + def initialize(family) + @family = family + end + + # Get totals for a specific period + def totals(period: Period.current_month) + trades_in_period = family.trades + .joins(:entry) + .where(entries: { date: period.date_range }) + + result = totals_query(trades_scope: trades_in_period) + + PeriodTotals.new( + contributions: Money.new(result[:contributions], family.currency), + withdrawals: Money.new(result[:withdrawals], family.currency), + dividends: Money.new(result[:dividends], family.currency), + interest: Money.new(result[:interest], family.currency), + trades_count: result[:trades_count], + currency: family.currency + ) + end + + # Net contributions (contributions - withdrawals) + def net_contributions(period: Period.current_month) + t = totals(period: period) + t.contributions - t.withdrawals + end + + # Total portfolio value across all investment accounts + def portfolio_value + investment_accounts.sum(&:balance) + end + + def portfolio_value_money + Money.new(portfolio_value, family.currency) + end + + # Total cash in investment accounts + def cash_balance + investment_accounts.sum(&:cash_balance) + end + + def cash_balance_money + Money.new(cash_balance, family.currency) + end + + # Total holdings value + def holdings_value + portfolio_value - cash_balance + end + + def holdings_value_money + Money.new(holdings_value, family.currency) + end + + # All current holdings across investment accounts + def current_holdings + return Holding.none unless investment_accounts.any? + + account_ids = investment_accounts.pluck(:id) + + # Get the latest holding for each security per account + Holding + .where(account_id: account_ids) + .where(currency: family.currency) + .where.not(qty: 0) + .where( + id: Holding + .where(account_id: account_ids) + .where(currency: family.currency) + .select("DISTINCT ON (holdings.account_id, holdings.security_id) holdings.id") + .order(Arel.sql("holdings.account_id, holdings.security_id, holdings.date DESC")) + ) + .includes(:security, :account) + .order(amount: :desc) + end + + # Top holdings by value + def top_holdings(limit: 5) + current_holdings.limit(limit) + end + + # Portfolio allocation by security type/sector (simplified for now) + def allocation + holdings = current_holdings.to_a + total = holdings.sum(&:amount) + + return [] if total.zero? + + holdings.map do |holding| + HoldingAllocation.new( + security: holding.security, + amount: holding.amount_money, + weight: (holding.amount / total * 100).round(2), + trend: holding.trend + ) + end + end + + # Unrealized gains across all holdings + def unrealized_gains + current_holdings.sum do |holding| + trend = holding.trend + trend ? trend.value : 0 + end + end + + # Total contributions (all time) - returns numeric for monetize + def total_contributions + all_time_totals.contributions&.amount || 0 + end + + # Total dividends (all time) - returns numeric for monetize + def total_dividends + all_time_totals.dividends&.amount || 0 + end + + # Total interest (all time) - returns numeric for monetize + def total_interest + all_time_totals.interest&.amount || 0 + end + + def unrealized_gains_trend + holdings = current_holdings.to_a + return nil if holdings.empty? + + current = holdings.sum(&:amount) + previous = holdings.sum { |h| h.qty * h.avg_cost.amount } + + Trend.new(current: current, previous: previous) + end + + # Day change across portfolio + def day_change + holdings = current_holdings.to_a + changes = holdings.map(&:day_change).compact + + return nil if changes.empty? + + current = changes.sum { |t| t.current.is_a?(Money) ? t.current.amount : t.current } + previous = changes.sum { |t| t.previous.is_a?(Money) ? t.previous.amount : t.previous } + + Trend.new( + current: Money.new(current, family.currency), + previous: Money.new(previous, family.currency) + ) + end + + # Investment accounts + def investment_accounts + @investment_accounts ||= family.accounts.visible.where(accountable_type: %w[Investment Crypto]) + end + + private + def all_time_totals + @all_time_totals ||= totals(period: Period.all_time) + end + + PeriodTotals = Data.define(:contributions, :withdrawals, :dividends, :interest, :trades_count, :currency) do + def net_flow + contributions - withdrawals + end + + def total_income + dividends + interest + end + end + + HoldingAllocation = Data.define(:security, :amount, :weight, :trend) + + def totals_query(trades_scope:) + sql_hash = Digest::MD5.hexdigest(trades_scope.to_sql) + + Rails.cache.fetch([ + "investment_statement", "totals_query", family.id, sql_hash, family.entries_cache_version + ]) { Totals.new(family, trades_scope: trades_scope).call } + end + + def monetizable_currency + family.currency + end +end diff --git a/app/models/investment_statement/totals.rb b/app/models/investment_statement/totals.rb new file mode 100644 index 000000000..65af7be6f --- /dev/null +++ b/app/models/investment_statement/totals.rb @@ -0,0 +1,56 @@ +class InvestmentStatement::Totals + def initialize(family, trades_scope:) + @family = family + @trades_scope = trades_scope + end + + def call + result = ActiveRecord::Base.connection.select_one(query_sql) + + { + contributions: result["contributions"]&.to_d || 0, + withdrawals: result["withdrawals"]&.to_d || 0, + dividends: 0, # Dividends come through as transactions, not trades + interest: 0, # Interest comes through as transactions, not trades + trades_count: result["trades_count"]&.to_i || 0 + } + end + + private + def query_sql + ActiveRecord::Base.sanitize_sql_array([ + aggregation_sql, + sql_params + ]) + end + + # Aggregate trades by direction (buy vs sell) + # Buys (qty > 0) = contributions (cash going out to buy securities) + # Sells (qty < 0) = withdrawals (cash coming in from selling securities) + def aggregation_sql + <<~SQL + SELECT + COALESCE(SUM(CASE WHEN t.qty > 0 THEN ABS(ae.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as contributions, + COALESCE(SUM(CASE WHEN t.qty < 0 THEN ABS(ae.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as withdrawals, + COUNT(t.id) as trades_count + FROM (#{@trades_scope.to_sql}) t + JOIN entries ae ON ae.entryable_id = t.id AND ae.entryable_type = 'Trade' + JOIN accounts a ON a.id = ae.account_id + LEFT JOIN exchange_rates er ON ( + er.date = ae.date AND + er.from_currency = ae.currency AND + er.to_currency = :target_currency + ) + WHERE a.family_id = :family_id + AND a.status IN ('draft', 'active') + AND ae.excluded = false + SQL + end + + def sql_params + { + family_id: @family.id, + target_currency: @family.currency + } + end +end diff --git a/app/models/lunchflow_account/investments/holdings_processor.rb b/app/models/lunchflow_account/investments/holdings_processor.rb new file mode 100644 index 000000000..a972f6f44 --- /dev/null +++ b/app/models/lunchflow_account/investments/holdings_processor.rb @@ -0,0 +1,184 @@ +class LunchflowAccount::Investments::HoldingsProcessor + def initialize(lunchflow_account) + @lunchflow_account = lunchflow_account + end + + def process + return if holdings_data.empty? + return unless [ "Investment", "Crypto" ].include?(account&.accountable_type) + + holdings_data.each do |lunchflow_holding| + begin + process_holding(lunchflow_holding) + rescue => e + symbol = lunchflow_holding.dig(:security, :tickerSymbol) rescue nil + ctx = symbol.present? ? " #{symbol}" : "" + Rails.logger.error "Error processing Lunchflow holding#{ctx}: #{e.message}" + end + end + end + + private + attr_reader :lunchflow_account + + def process_holding(lunchflow_holding) + # Support both symbol and string keys (JSONB returns string keys) + holding = lunchflow_holding.is_a?(Hash) ? lunchflow_holding.with_indifferent_access : {} + security_data = (holding[:security] || {}).with_indifferent_access + raw_data = holding[:raw] || {} + + symbol = security_data[:tickerSymbol].presence + security_name = security_data[:name].to_s.strip + + # Extract holding ID from nested raw data (e.g., raw.quiltt.id) + holding_id = extract_holding_id(raw_data) || generate_holding_id(holding) + + Rails.logger.debug({ + event: "lunchflow.holding.start", + lfa_id: lunchflow_account.id, + account_id: account&.id, + id: holding_id, + symbol: symbol, + name: security_name + }.to_json) + + # If symbol is missing but we have a name, create a synthetic ticker + if symbol.blank? && security_name.present? + normalized = security_name.gsub(/[^a-zA-Z0-9]/, "_").upcase.truncate(24, omission: "") + hash_suffix = Digest::MD5.hexdigest(security_name)[0..4].upcase + symbol = "CUSTOM:#{normalized}_#{hash_suffix}" + Rails.logger.info("Lunchflow: using synthetic ticker #{symbol} for holding #{holding_id} (#{security_name})") + end + + unless symbol.present? + Rails.logger.debug({ event: "lunchflow.holding.skip", reason: "no_symbol_or_name", id: holding_id }.to_json) + return + end + + security = resolve_security(symbol, security_name, security_data) + unless security.present? + Rails.logger.debug({ event: "lunchflow.holding.skip", reason: "unresolved_security", id: holding_id, symbol: symbol }.to_json) + return + end + + # Parse holding data from API response + qty = parse_decimal(holding[:quantity]) + price = parse_decimal(holding[:price]) + amount = parse_decimal(holding[:value]) + cost_basis = parse_decimal(holding[:costBasis]) + currency = holding[:currency].presence || security_data[:currency].presence || "USD" + + # Skip zero positions with no value + if qty.to_d.zero? && amount.to_d.zero? + Rails.logger.debug({ event: "lunchflow.holding.skip", reason: "zero_position", id: holding_id }.to_json) + return + end + + saved = import_adapter.import_holding( + security: security, + quantity: qty, + amount: amount, + currency: currency, + date: Date.current, + price: price, + cost_basis: cost_basis, + external_id: "lunchflow_#{holding_id}", + account_provider_id: lunchflow_account.account_provider&.id, + source: "lunchflow", + delete_future_holdings: false + ) + + Rails.logger.debug({ + event: "lunchflow.holding.saved", + account_id: account&.id, + holding_id: saved.id, + security_id: saved.security_id, + qty: saved.qty.to_s, + amount: saved.amount.to_s, + currency: saved.currency, + date: saved.date, + external_id: saved.external_id + }.to_json) + end + + def import_adapter + @import_adapter ||= Account::ProviderImportAdapter.new(account) + end + + def account + lunchflow_account.current_account + end + + def holdings_data + lunchflow_account.raw_holdings_payload || [] + end + + def extract_holding_id(raw_data) + # Try to find ID in nested provider data (e.g., raw.quiltt.id, raw.plaid.id, etc.) + return nil unless raw_data.is_a?(Hash) + + raw_data.each_value do |provider_data| + next unless provider_data.is_a?(Hash) + id = provider_data[:id] || provider_data["id"] + return id.to_s if id.present? + end + + nil + end + + def generate_holding_id(holding) + # Generate a stable ID based on holding content + # holding should already be with_indifferent_access from process_holding + security = holding[:security] || {} + content = [ + security[:tickerSymbol] || security["tickerSymbol"], + security[:name] || security["name"], + holding[:quantity], + holding[:value] + ].compact.join("-") + Digest::MD5.hexdigest(content)[0..11] + end + + def resolve_security(symbol, description, security_data) + # Normalize crypto tickers to a distinct namespace + sym = symbol.to_s.upcase + is_crypto_account = account&.accountable_type == "Crypto" + is_crypto_symbol = %w[BTC ETH SOL DOGE LTC BCH XRP ADA DOT AVAX].include?(sym) + + if !sym.include?(":") && (is_crypto_account || is_crypto_symbol) + sym = "CRYPTO:#{sym}" + end + + is_custom = sym.start_with?("CUSTOM:") + + begin + if is_custom + raise "Custom ticker - skipping resolver" + end + Security::Resolver.new(sym).resolve + rescue => e + Rails.logger.warn "Lunchflow: resolver failed for symbol=#{sym}: #{e.class} - #{e.message}; falling back to offline security" unless is_custom + Security.find_or_initialize_by(ticker: sym).tap do |sec| + sec.offline = true if sec.respond_to?(:offline) && sec.offline != true + sec.name = description.presence if sec.name.blank? && description.present? + sec.save! if sec.changed? + end + end + end + + def parse_decimal(value) + return BigDecimal("0") unless value.present? + + case value + when String + BigDecimal(value) + when Numeric + BigDecimal(value.to_s) + else + BigDecimal("0") + end + rescue ArgumentError => e + Rails.logger.error "Failed to parse Lunchflow decimal value #{value}: #{e.message}" + BigDecimal("0") + end +end diff --git a/app/models/lunchflow_account/processor.rb b/app/models/lunchflow_account/processor.rb index 4431080b7..b9c6b2184 100644 --- a/app/models/lunchflow_account/processor.rb +++ b/app/models/lunchflow_account/processor.rb @@ -25,6 +25,7 @@ class LunchflowAccount::Processor end process_transactions + process_investments end private @@ -67,6 +68,16 @@ class LunchflowAccount::Processor report_exception(e, "transactions") end + def process_investments + # Only process holdings for investment/crypto accounts with holdings support + return unless lunchflow_account.holdings_supported? + return unless [ "Investment", "Crypto" ].include?(lunchflow_account.current_account&.accountable_type) + + LunchflowAccount::Investments::HoldingsProcessor.new(lunchflow_account).process + rescue => e + report_exception(e, "holdings") + end + def report_exception(error, context) Sentry.capture_exception(error) do |scope| scope.set_tags( diff --git a/app/models/lunchflow_item/importer.rb b/app/models/lunchflow_item/importer.rb index 6d4c63e90..64af4089d 100644 --- a/app/models/lunchflow_item/importer.rb +++ b/app/models/lunchflow_item/importer.rb @@ -242,6 +242,14 @@ class LunchflowItem::Importer Rails.logger.warn "LunchflowItem::Importer - Failed to update balance for account #{lunchflow_account.account_id}: #{e.message}" end + # Fetch holdings for investment/crypto accounts + begin + fetch_and_store_holdings(lunchflow_account) + rescue => e + # Log but don't fail sync if holdings fetch fails + Rails.logger.warn "LunchflowItem::Importer - Failed to fetch holdings for account #{lunchflow_account.account_id}: #{e.message}" + end + { success: true, transactions_count: transactions_count } rescue Provider::Lunchflow::LunchflowError => e Rails.logger.error "LunchflowItem::Importer - Lunchflow API error for account #{lunchflow_account.id}: #{e.message}" @@ -299,6 +307,53 @@ class LunchflowItem::Importer end end + def fetch_and_store_holdings(lunchflow_account) + # Only fetch holdings for investment/crypto accounts + account = lunchflow_account.current_account + return unless account.present? + return unless [ "Investment", "Crypto" ].include?(account.accountable_type) + + # Skip if holdings are not supported for this account + unless lunchflow_account.holdings_supported? + Rails.logger.debug "LunchflowItem::Importer - Skipping holdings fetch for account #{lunchflow_account.account_id} (holdings not supported)" + return + end + + Rails.logger.info "LunchflowItem::Importer - Fetching holdings for account #{lunchflow_account.account_id}" + + begin + holdings_data = lunchflow_provider.get_account_holdings(lunchflow_account.account_id) + + # Validate response structure + unless holdings_data.is_a?(Hash) + Rails.logger.error "LunchflowItem::Importer - Invalid holdings_data format for account #{lunchflow_account.account_id}" + return + end + + # Check if holdings are not supported (501 response) + if holdings_data[:holdings_not_supported] + Rails.logger.info "LunchflowItem::Importer - Holdings not supported for account #{lunchflow_account.account_id}, disabling future requests" + lunchflow_account.update!(holdings_supported: false) + return + end + + # Store holdings payload for processing + holdings_array = holdings_data[:holdings] || [] + Rails.logger.info "LunchflowItem::Importer - Fetched #{holdings_array.count} holdings for account #{lunchflow_account.account_id}" + + lunchflow_account.update!(raw_holdings_payload: holdings_array) + rescue Provider::Lunchflow::LunchflowError => e + Rails.logger.error "LunchflowItem::Importer - Lunchflow API error fetching holdings for account #{lunchflow_account.id}: #{e.message}" + # Don't fail if holdings fetch fails + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error "LunchflowItem::Importer - Failed to save holdings for account #{lunchflow_account.id}: #{e.message}" + # Don't fail if holdings save fails + rescue => e + Rails.logger.error "LunchflowItem::Importer - Unexpected error fetching holdings for account #{lunchflow_account.id}: #{e.class} - #{e.message}" + # Don't fail if holdings fetch fails + end + end + def determine_sync_start_date(lunchflow_account) # Check if this account has any stored transactions # If not, treat it as a first sync for this account even if the item has been synced before diff --git a/app/models/lunchflow_item/syncer.rb b/app/models/lunchflow_item/syncer.rb index 4d10b2578..73147c4ef 100644 --- a/app/models/lunchflow_item/syncer.rb +++ b/app/models/lunchflow_item/syncer.rb @@ -1,4 +1,6 @@ class LunchflowItem::Syncer + include SyncStats::Collector + attr_reader :lunchflow_item def initialize(lunchflow_item) @@ -10,18 +12,13 @@ class LunchflowItem::Syncer sync.update!(status_text: "Importing accounts from Lunchflow...") if sync.respond_to?(:status_text) lunchflow_item.import_latest_lunchflow_data - # Phase 2: Check account setup status and collect sync statistics + # Phase 2: Collect setup statistics using shared concern sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text) - total_accounts = lunchflow_item.lunchflow_accounts.count - linked_accounts = lunchflow_item.lunchflow_accounts.joins(:account).merge(Account.visible) - unlinked_accounts = lunchflow_item.lunchflow_accounts.includes(:account).where(accounts: { id: nil }) + collect_setup_stats(sync, provider_accounts: lunchflow_item.lunchflow_accounts) - # Store sync statistics for display - sync_stats = { - total_accounts: total_accounts, - linked_accounts: linked_accounts.count, - unlinked_accounts: unlinked_accounts.count - } + # Check for unlinked accounts + linked_accounts = lunchflow_item.lunchflow_accounts.joins(:account_provider) + unlinked_accounts = lunchflow_item.lunchflow_accounts.left_joins(:account_provider).where(account_providers: { id: nil }) # Set pending_account_setup if there are unlinked accounts if unlinked_accounts.any? @@ -31,9 +28,10 @@ class LunchflowItem::Syncer lunchflow_item.update!(pending_account_setup: false) end - # Phase 3: Process transactions for linked accounts only + # Phase 3: Process transactions and holdings for linked accounts only if linked_accounts.any? - sync.update!(status_text: "Processing transactions...") if sync.respond_to?(:status_text) + sync.update!(status_text: "Processing transactions and holdings...") if sync.respond_to?(:status_text) + mark_import_started(sync) Rails.logger.info "LunchflowItem::Syncer - Processing #{linked_accounts.count} linked accounts" lunchflow_item.process_accounts Rails.logger.info "LunchflowItem::Syncer - Finished processing accounts" @@ -45,14 +43,19 @@ class LunchflowItem::Syncer window_start_date: sync.window_start_date, window_end_date: sync.window_end_date ) + + # Phase 5: Collect transaction statistics + account_ids = linked_accounts.includes(:account_provider).filter_map { |la| la.current_account&.id } + collect_transaction_stats(sync, account_ids: account_ids, source: "lunchflow") else Rails.logger.info "LunchflowItem::Syncer - No linked accounts to process" end - # Store sync statistics in the sync record for status display - if sync.respond_to?(:sync_stats) - sync.update!(sync_stats: sync_stats) - end + # Mark sync health + collect_health_stats(sync, errors: nil) + rescue => e + collect_health_stats(sync, errors: [ { message: e.message, category: "sync_error" } ]) + raise end def perform_post_sync diff --git a/app/models/merchant/merger.rb b/app/models/merchant/merger.rb new file mode 100644 index 000000000..7baa78e67 --- /dev/null +++ b/app/models/merchant/merger.rb @@ -0,0 +1,54 @@ +class Merchant::Merger + class UnauthorizedMerchantError < StandardError; end + + attr_reader :family, :target_merchant, :source_merchants, :merged_count + + def initialize(family:, target_merchant:, source_merchants:) + @family = family + @target_merchant = target_merchant + @merged_count = 0 + + validate_merchant_belongs_to_family!(target_merchant, "Target merchant") + + sources = Array(source_merchants) + sources.each { |m| validate_merchant_belongs_to_family!(m, "Source merchant '#{m.name}'") } + + @source_merchants = sources.reject { |m| m.id == target_merchant.id } + end + + private + + def validate_merchant_belongs_to_family!(merchant, label) + return if family_merchant_ids.include?(merchant.id) + + raise UnauthorizedMerchantError, "#{label} does not belong to this family" + end + + def family_merchant_ids + @family_merchant_ids ||= begin + family_ids = family.merchants.pluck(:id) + assigned_ids = family.assigned_merchants.pluck(:id) + (family_ids + assigned_ids).uniq + end + end + + public + + def merge! + return false if source_merchants.empty? + + Merchant.transaction do + source_merchants.each do |source| + # Reassign family's transactions to target + family.transactions.where(merchant_id: source.id).update_all(merchant_id: target_merchant.id) + + # Delete FamilyMerchant, keep ProviderMerchant (it may be used by other families) + source.destroy! if source.is_a?(FamilyMerchant) + + @merged_count += 1 + end + end + + true + end +end diff --git a/app/models/mint_import.rb b/app/models/mint_import.rb index d62da471d..932353d8a 100644 --- a/app/models/mint_import.rb +++ b/app/models/mint_import.rb @@ -18,6 +18,7 @@ class MintImport < Import end rows.insert_all!(mapped_rows) + update_column(:rows_count, rows.count) end def import! diff --git a/app/models/plaid_item/syncer.rb b/app/models/plaid_item/syncer.rb index b76c37b67..58c2baccd 100644 --- a/app/models/plaid_item/syncer.rb +++ b/app/models/plaid_item/syncer.rb @@ -1,4 +1,6 @@ class PlaidItem::Syncer + include SyncStats::Collector + attr_reader :plaid_item def initialize(plaid_item) @@ -6,21 +8,60 @@ class PlaidItem::Syncer end def perform_sync(sync) - # Loads item metadata, accounts, transactions, and other data to our DB + # Phase 1: Import data from Plaid API + sync.update!(status_text: "Importing accounts from Plaid...") if sync.respond_to?(:status_text) plaid_item.import_latest_plaid_data - # Processes the raw Plaid data and updates internal domain objects - plaid_item.process_accounts + # Phase 2: Collect setup statistics + sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text) + collect_setup_stats(sync, provider_accounts: plaid_item.plaid_accounts) - # All data is synced, so we can now run an account sync to calculate historical balances and more - plaid_item.schedule_account_syncs( - parent_sync: sync, - window_start_date: sync.window_start_date, - window_end_date: sync.window_end_date - ) + # Check for unlinked accounts and update pending_account_setup flag + unlinked_count = plaid_item.plaid_accounts.count { |pa| pa.current_account.nil? } + if unlinked_count > 0 + plaid_item.update!(pending_account_setup: true) if plaid_item.respond_to?(:pending_account_setup=) + sync.update!(status_text: "#{unlinked_count} accounts need setup...") if sync.respond_to?(:status_text) + else + plaid_item.update!(pending_account_setup: false) if plaid_item.respond_to?(:pending_account_setup=) + end + + # Phase 3: Process the raw Plaid data and updates internal domain objects + linked_accounts = plaid_item.plaid_accounts.select { |pa| pa.current_account.present? } + if linked_accounts.any? + sync.update!(status_text: "Processing transactions...") if sync.respond_to?(:status_text) + mark_import_started(sync) + plaid_item.process_accounts + + # Phase 4: Schedule balance calculations + sync.update!(status_text: "Calculating balances...") if sync.respond_to?(:status_text) + plaid_item.schedule_account_syncs( + parent_sync: sync, + window_start_date: sync.window_start_date, + window_end_date: sync.window_end_date + ) + + # Phase 5: Collect transaction and holdings statistics + account_ids = linked_accounts.filter_map { |pa| pa.current_account&.id } + collect_transaction_stats(sync, account_ids: account_ids, source: "plaid") + collect_holdings_stats(sync, holdings_count: count_holdings(linked_accounts), label: "processed") + end + + # Mark sync health + collect_health_stats(sync, errors: nil) + rescue => e + collect_health_stats(sync, errors: [ { message: e.message, category: "sync_error" } ]) + raise end def perform_post_sync # no-op end + + private + + def count_holdings(plaid_accounts) + plaid_accounts.sum do |pa| + Array(pa.raw_investments_payload).size + end + end end diff --git a/app/models/provider/coinstats.rb b/app/models/provider/coinstats.rb new file mode 100644 index 000000000..e9d406ab9 --- /dev/null +++ b/app/models/provider/coinstats.rb @@ -0,0 +1,184 @@ +# API client for CoinStats cryptocurrency data provider. +# Handles authentication and requests to the CoinStats OpenAPI. +class Provider::Coinstats < Provider + include HTTParty + + # Subclass so errors caught in this provider are raised as Provider::Coinstats::Error + Error = Class.new(Provider::Error) + + BASE_URL = "https://openapiv1.coinstats.app" + + headers "User-Agent" => "Sure Finance CoinStats Client (https://github.com/we-promise/sure)" + default_options.merge!(verify: true, ssl_verify_mode: OpenSSL::SSL::VERIFY_PEER, timeout: 120) + + attr_reader :api_key + + # @param api_key [String] CoinStats API key for authentication + def initialize(api_key) + @api_key = api_key + end + + # Get the list of blockchains supported by CoinStats + # https://coinstats.app/api-docs/openapi/get-blockchains + def get_blockchains + with_provider_response do + res = self.class.get("#{BASE_URL}/wallet/blockchains", headers: auth_headers) + handle_response(res) + end + end + + # Returns blockchain options formatted for select dropdowns + # @return [Array] Array of [label, value] pairs sorted alphabetically + def blockchain_options + response = get_blockchains + + unless response.success? + Rails.logger.warn("CoinStats: failed to fetch blockchains: #{response.error&.message}") + return [] + end + + raw_blockchains = response.data + items = if raw_blockchains.is_a?(Array) + raw_blockchains + elsif raw_blockchains.respond_to?(:dig) && raw_blockchains[:data].is_a?(Array) + raw_blockchains[:data] + else + [] + end + + items.filter_map do |b| + b = b.with_indifferent_access + value = b[:connectionId] || b[:id] || b[:name] + next unless value.present? + + label = b[:name].presence || value.to_s + [ label, value ] + end.uniq { |_label, value| value }.sort_by { |label, _| label.to_s.downcase } + rescue StandardError => e + Rails.logger.warn("CoinStats: failed to fetch blockchains: #{e.class} - #{e.message}") + [] + end + + # Get cryptocurrency balances for multiple wallets in a single request + # https://coinstats.app/api-docs/openapi/get-wallet-balances + # @param wallets [String] Comma-separated list of wallet addresses in format "blockchain:address" + # Example: "ethereum:0x123abc,bitcoin:bc1qxyz" + # @return [Provider::Response] Response with wallet balance data + def get_wallet_balances(wallets) + return with_provider_response { [] } if wallets.blank? + + with_provider_response do + res = self.class.get( + "#{BASE_URL}/wallet/balances", + headers: auth_headers, + query: { wallets: wallets } + ) + handle_response(res) + end + end + + # Extract balance data for a specific wallet from bulk response + # @param bulk_data [Array] Response from get_wallet_balances + # @param address [String] Wallet address to find + # @param blockchain [String] Blockchain/connectionId to find + # @return [Array] Token balances for the wallet, or empty array if not found + def extract_wallet_balance(bulk_data, address, blockchain) + return [] unless bulk_data.is_a?(Array) + + wallet_data = bulk_data.find do |entry| + entry = entry.with_indifferent_access + entry[:address]&.downcase == address&.downcase && + (entry[:connectionId]&.downcase == blockchain&.downcase || + entry[:blockchain]&.downcase == blockchain&.downcase) + end + + return [] unless wallet_data + + wallet_data = wallet_data.with_indifferent_access + wallet_data[:balances] || [] + end + + # Get transaction data for multiple wallet addresses in a single request + # https://coinstats.app/api-docs/openapi/get-wallet-transactions + # @param wallets [String] Comma-separated list of wallet addresses in format "blockchain:address" + # Example: "ethereum:0x123abc,bitcoin:bc1qxyz" + # @return [Provider::Response] Response with wallet transaction data + def get_wallet_transactions(wallets) + return with_provider_response { [] } if wallets.blank? + + with_provider_response do + res = self.class.get( + "#{BASE_URL}/wallet/transactions", + headers: auth_headers, + query: { wallets: wallets } + ) + handle_response(res) + end + end + + # Extract transaction data for a specific wallet from bulk response + # The transactions API returns {result: Array, meta: {...}} + # All transactions in the response belong to the requested wallets + # @param bulk_data [Hash, Array] Response from get_wallet_transactions + # @param address [String] Wallet address to filter by (currently unused as API returns flat list) + # @param blockchain [String] Blockchain/connectionId to filter by (currently unused) + # @return [Array] Transactions for the wallet, or empty array if not found + def extract_wallet_transactions(bulk_data, address, blockchain) + # Handle Hash response with :result key (current API format) + if bulk_data.is_a?(Hash) + bulk_data = bulk_data.with_indifferent_access + return bulk_data[:result] || [] + end + + # Handle legacy Array format (per-wallet structure) + return [] unless bulk_data.is_a?(Array) + + wallet_data = bulk_data.find do |entry| + entry = entry.with_indifferent_access + entry[:address]&.downcase == address&.downcase && + (entry[:connectionId]&.downcase == blockchain&.downcase || + entry[:blockchain]&.downcase == blockchain&.downcase) + end + + return [] unless wallet_data + + wallet_data = wallet_data.with_indifferent_access + wallet_data[:transactions] || [] + end + + private + + def auth_headers + { + "X-API-KEY" => api_key, + "Accept" => "application/json" + } + end + + # The CoinStats API uses standard HTTP status codes to indicate the success or failure of requests. + # https://coinstats.app/api-docs/errors + def handle_response(response) + case response.code + when 200 + JSON.parse(response.body, symbolize_names: true) + when 400 + raise Error, "CoinStats: #{response.code} Bad Request - Invalid parameters or request format #{response.body}" + when 401 + raise Error, "CoinStats: #{response.code} Unauthorized - Invalid or missing API key #{response.body}" + when 403 + raise Error, "CoinStats: #{response.code} Forbidden - #{response.body}" + when 404 + raise Error, "CoinStats: #{response.code} Not Found - Resource not found #{response.body}" + when 409 + raise Error, "CoinStats: #{response.code} Conflict - Resource conflict #{response.body}" + when 429 + raise Error, "CoinStats: #{response.code} Too Many Requests - Rate limit exceeded #{response.body}" + when 500 + raise Error, "CoinStats: #{response.code} Internal Server Error - Server error #{response.body}" + when 503 + raise Error, "CoinStats: #{response.code} Service Unavailable - #{response.body}" + else + raise Error, "CoinStats: #{response.code} Unexpected Error - #{response.body}" + end + end +end diff --git a/app/models/provider/coinstats_adapter.rb b/app/models/provider/coinstats_adapter.rb new file mode 100644 index 000000000..d0ac64964 --- /dev/null +++ b/app/models/provider/coinstats_adapter.rb @@ -0,0 +1,119 @@ +# Provider adapter for CoinStats cryptocurrency wallet integration. +# Handles sync operations and institution metadata for crypto accounts. +class Provider::CoinstatsAdapter < Provider::Base + include Provider::Syncable + include Provider::InstitutionMetadata + + # Register this adapter with the factory + Provider::Factory.register("CoinstatsAccount", self) + + # @return [Array] Account types supported by this provider + def self.supported_account_types + %w[Crypto] + end + + # Returns connection configurations for this provider + # @param family [Family] The family to check connection eligibility + # @return [Array] Connection config with name, description, and paths + def self.connection_configs(family:) + return [] unless family.can_connect_coinstats? + + [ { + key: "coinstats", + name: "CoinStats", + description: "Connect to your crypto wallet via CoinStats", + can_connect: true, + new_account_path: ->(accountable_type, return_to) { + Rails.application.routes.url_helpers.new_coinstats_item_path( + accountable_type: accountable_type, + return_to: return_to + ) + }, + # CoinStats wallets are linked via the link_wallet action, not via existing account selection + existing_account_path: nil + } ] + end + + # @return [String] Unique identifier for this provider + def provider_name + "coinstats" + end + + # Build a Coinstats provider instance with family-specific credentials + # @param family [Family] The family to get credentials for (required) + # @return [Provider::Coinstats, nil] Returns nil if credentials are not configured + def self.build_provider(family: nil) + return nil unless family.present? + + # Get family-specific credentials + coinstats_item = family.coinstats_items.where.not(api_key: nil).first + return nil unless coinstats_item&.credentials_configured? + + Provider::Coinstats.new(coinstats_item.api_key) + end + + # @return [String] URL path for triggering a sync + def sync_path + Rails.application.routes.url_helpers.sync_coinstats_item_path(item) + end + + # @return [CoinstatsItem] The parent item containing API credentials + def item + provider_account.coinstats_item + end + + # @return [Boolean] Whether holdings can be manually deleted + def can_delete_holdings? + false + end + + # Extracts institution domain from metadata, deriving from URL if needed. + # @return [String, nil] Domain name or nil if unavailable + def institution_domain + metadata = provider_account.institution_metadata + return nil unless metadata.present? + + domain = metadata["domain"] + url = metadata["url"] + + # Derive domain from URL if missing + if domain.blank? && url.present? + begin + domain = URI.parse(url).host&.gsub(/^www\./, "") + rescue URI::InvalidURIError + Rails.logger.warn("Invalid institution URL for Coinstats account #{provider_account.id}: #{url}") + end + end + + domain + end + + # @return [String, nil] Institution display name + def institution_name + metadata = provider_account.institution_metadata + return nil unless metadata.present? + + metadata["name"] + end + + # @return [String, nil] Institution website URL + def institution_url + metadata = provider_account.institution_metadata + return nil unless metadata.present? + + metadata["url"] + end + + # @return [nil] CoinStats doesn't provide institution colors + def institution_color + nil # CoinStats doesn't provide institution colors + end + + # @return [String, nil] URL for institution/token logo + def logo_url + metadata = provider_account.institution_metadata + return nil unless metadata.present? + + metadata["logo"] + end +end diff --git a/app/models/provider/institution_metadata.rb b/app/models/provider/institution_metadata.rb index 2998d3aa4..63ecce59b 100644 --- a/app/models/provider/institution_metadata.rb +++ b/app/models/provider/institution_metadata.rb @@ -27,6 +27,12 @@ module Provider::InstitutionMetadata nil end + # Returns the institution/account logo URL (direct image URL) + # @return [String, nil] The logo URL or nil if not available + def logo_url + nil + end + # Returns a hash of all institution metadata # @return [Hash] Hash containing institution metadata def institution_metadata @@ -34,7 +40,8 @@ module Provider::InstitutionMetadata domain: institution_domain, name: institution_name, url: institution_url, - color: institution_color + color: institution_color, + logo_url: logo_url }.compact end end diff --git a/app/models/provider/lunchflow.rb b/app/models/provider/lunchflow.rb index dfd5f5109..8ff44d682 100644 --- a/app/models/provider/lunchflow.rb +++ b/app/models/provider/lunchflow.rb @@ -78,6 +78,31 @@ class Provider::Lunchflow raise LunchflowError.new("Exception during GET request: #{e.message}", :request_failed) end + # Get holdings for a specific account (investment accounts only) + # Returns: { holdings: [...], totalValue: N, currency: "USD" } + # Returns { holdings_not_supported: true } if API returns 501 + def get_account_holdings(account_id) + path = "/accounts/#{ERB::Util.url_encode(account_id.to_s)}/holdings" + + response = self.class.get( + "#{@base_url}#{path}", + headers: auth_headers + ) + + # Handle 501 specially - indicates holdings not supported for this account + if response.code == 501 + return { holdings_not_supported: true } + end + + handle_response(response) + rescue SocketError, Net::OpenTimeout, Net::ReadTimeout => e + Rails.logger.error "Lunch Flow API: GET #{path} failed: #{e.class}: #{e.message}" + raise LunchflowError.new("Exception during GET request: #{e.message}", :request_failed) + rescue => e + Rails.logger.error "Lunch Flow API: Unexpected error during GET #{path}: #{e.class}: #{e.message}" + raise LunchflowError.new("Exception during GET request: #{e.message}", :request_failed) + end + private def auth_headers diff --git a/app/models/provider/simplefin.rb b/app/models/provider/simplefin.rb index 8fff2a297..5ceb2ecad 100644 --- a/app/models/provider/simplefin.rb +++ b/app/models/provider/simplefin.rb @@ -9,6 +9,22 @@ class Provider::Simplefin headers "User-Agent" => "Sure Finance SimpleFin Client" default_options.merge!(verify: true, ssl_verify_mode: OpenSSL::SSL::VERIFY_PEER, timeout: 120) + # Retry configuration for transient network failures + MAX_RETRIES = 3 + INITIAL_RETRY_DELAY = 2 # seconds + MAX_RETRY_DELAY = 30 # seconds + + # Errors that are safe to retry (transient network issues) + RETRYABLE_ERRORS = [ + SocketError, + Net::OpenTimeout, + Net::ReadTimeout, + Errno::ECONNRESET, + Errno::ECONNREFUSED, + Errno::ETIMEDOUT, + EOFError + ].freeze + def initialize end @@ -16,7 +32,11 @@ class Provider::Simplefin # Decode the base64 setup token to get the claim URL claim_url = Base64.decode64(setup_token) - response = HTTParty.post(claim_url) + # Use retry logic for transient network failures during token claim + # Claim should be fast; keep request-path latency bounded. + response = with_retries("POST /claim", max_retries: 1, sleep: false) do + HTTParty.post(claim_url, timeout: 15) + end case response.code when 200 @@ -49,19 +69,12 @@ class Provider::Simplefin accounts_url = "#{access_url}/accounts" accounts_url += "?#{URI.encode_www_form(query_params)}" unless query_params.empty? - # The access URL already contains HTTP Basic Auth credentials - begin - response = HTTParty.get(accounts_url) - rescue SocketError, Net::OpenTimeout, Net::ReadTimeout => e - Rails.logger.error "SimpleFin API: GET /accounts failed: #{e.class}: #{e.message}" - raise SimplefinError.new("Exception during GET request: #{e.message}", :request_failed) - rescue => e - Rails.logger.error "SimpleFin API: Unexpected error during GET /accounts: #{e.class}: #{e.message}" - raise SimplefinError.new("Exception during GET request: #{e.message}", :request_failed) + # Use retry logic with exponential backoff for transient network failures + response = with_retries("GET /accounts") do + HTTParty.get(accounts_url) end - case response.code when 200 JSON.parse(response.body, symbolize_names: true) @@ -72,6 +85,12 @@ class Provider::Simplefin raise SimplefinError.new("Access URL is no longer valid", :access_forbidden) when 402 raise SimplefinError.new("Payment required to access this account", :payment_required) + when 429 + Rails.logger.warn "SimpleFin API: Rate limited - #{response.body}" + raise SimplefinError.new("SimpleFin rate limit exceeded. Please try again later.", :rate_limited) + when 500..599 + Rails.logger.error "SimpleFin API: Server error - Code: #{response.code}, Body: #{response.body}" + raise SimplefinError.new("SimpleFin server error (#{response.code}). Please try again later.", :server_error) else Rails.logger.error "SimpleFin API: Unexpected response - Code: #{response.code}, Body: #{response.body}" raise SimplefinError.new("Failed to fetch accounts: #{response.code} #{response.message} - #{response.body}", :fetch_failed) @@ -97,4 +116,55 @@ class Provider::Simplefin @error_type = error_type end end + + private + + # Execute a block with retry logic and exponential backoff for transient network errors. + # This helps handle temporary network issues that cause autosync failures while + # manual sync (with user retry) succeeds. + def with_retries(operation_name, max_retries: MAX_RETRIES, sleep: true) + retries = 0 + + begin + yield + rescue *RETRYABLE_ERRORS => e + retries += 1 + + if retries <= max_retries + delay = calculate_retry_delay(retries) + Rails.logger.warn( + "SimpleFin API: #{operation_name} failed (attempt #{retries}/#{max_retries}): " \ + "#{e.class}: #{e.message}. Retrying in #{delay}s..." + ) + Kernel.sleep(delay) if sleep && delay.to_f.positive? + retry + else + Rails.logger.error( + "SimpleFin API: #{operation_name} failed after #{max_retries} retries: " \ + "#{e.class}: #{e.message}" + ) + raise SimplefinError.new( + "Network error after #{max_retries} retries: #{e.message}", + :network_error + ) + end + rescue SimplefinError => e + # Preserve original error type and message. + raise + rescue => e + # Non-retryable errors are logged and re-raised immediately + Rails.logger.error "SimpleFin API: #{operation_name} failed with non-retryable error: #{e.class}: #{e.message}" + raise SimplefinError.new("Exception during #{operation_name}: #{e.message}", :request_failed) + end + end + + # Calculate delay with exponential backoff and jitter + def calculate_retry_delay(retry_count) + # Exponential backoff: 2^retry * initial_delay + base_delay = INITIAL_RETRY_DELAY * (2 ** (retry_count - 1)) + # Add jitter (0-25% of base delay) to prevent thundering herd + jitter = base_delay * rand * 0.25 + # Cap at max delay + [ base_delay + jitter, MAX_RETRY_DELAY ].min + end end diff --git a/app/models/provider/yahoo_finance.rb b/app/models/provider/yahoo_finance.rb index bb008b5f6..eff80232f 100644 --- a/app/models/provider/yahoo_finance.rb +++ b/app/models/provider/yahoo_finance.rb @@ -260,7 +260,7 @@ class Provider::YahooFinance < Provider prices << Price.new( symbol: symbol, - date: Time.at(timestamp).to_date, + date: Time.at(timestamp).utc.to_date, price: normalized_price, currency: normalized_currency, exchange_operating_mic: exchange_operating_mic @@ -390,7 +390,7 @@ class Provider::YahooFinance < Provider symbol = "#{from}#{to}=X" fetch_chart_data(symbol, start_date, end_date) do |timestamp, close_rate| Rate.new( - date: Time.at(timestamp).to_date, + date: Time.at(timestamp).utc.to_date, from: from, to: to, rate: close_rate.to_f @@ -402,7 +402,7 @@ class Provider::YahooFinance < Provider symbol = "#{to}#{from}=X" rates = fetch_chart_data(symbol, start_date, end_date) do |timestamp, close_rate| Rate.new( - date: Time.at(timestamp).to_date, + date: Time.at(timestamp).utc.to_date, from: from, to: to, rate: (1.0 / close_rate.to_f).round(8) diff --git a/app/models/provider_merchant.rb b/app/models/provider_merchant.rb index e6a7341d3..5bc7cea4f 100644 --- a/app/models/provider_merchant.rb +++ b/app/models/provider_merchant.rb @@ -1,6 +1,36 @@ class ProviderMerchant < Merchant - enum :source, { plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking" } + enum :source, { plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats" } validates :name, uniqueness: { scope: [ :source ] } validates :source, presence: true + + # Convert this ProviderMerchant to a FamilyMerchant for a specific family. + # Only affects transactions belonging to that family. + # Returns the newly created FamilyMerchant. + def convert_to_family_merchant_for(family, attributes = {}) + transaction do + family_merchant = family.merchants.create!( + name: attributes[:name].presence || name, + color: attributes[:color].presence || FamilyMerchant::COLORS.sample, + logo_url: logo_url, + website_url: website_url + ) + + # Update only this family's transactions to point to new merchant + family.transactions.where(merchant_id: id).update_all(merchant_id: family_merchant.id) + + family_merchant + end + end + + # Unlink from family's transactions (set merchant_id to null). + # Does NOT delete the ProviderMerchant since it may be used by other families. + # Tracks the unlink in FamilyMerchantAssociation so it shows as "recently unlinked". + def unlink_from_family(family) + family.transactions.where(merchant_id: id).update_all(merchant_id: nil) + + # Track that this merchant was unlinked from this family + association = FamilyMerchantAssociation.find_or_initialize_by(family: family, merchant: self) + association.update!(unlinked_at: Time.current) + end end diff --git a/app/models/rule.rb b/app/models/rule.rb index 13087f93b..cd18deadc 100644 --- a/app/models/rule.rb +++ b/app/models/rule.rb @@ -40,6 +40,20 @@ class Rule < ApplicationRecord matching_resources_scope.count end + # Calculates total unique resources affected across multiple rules + # This handles overlapping rules by deduplicating transaction IDs + def self.total_affected_resource_count(rules) + return 0 if rules.empty? + + # Collect all unique transaction IDs matched by any rule + transaction_ids = Set.new + rules.each do |rule| + transaction_ids.merge(rule.send(:matching_resources_scope).pluck(:id)) + end + + transaction_ids.size + end + def apply(ignore_attribute_locks: false, rule_run: nil) total_modified = 0 total_async_jobs = 0 diff --git a/app/models/rule_import.rb b/app/models/rule_import.rb index e214481aa..d2a2d07ca 100644 --- a/app/models/rule_import.rb +++ b/app/models/rule_import.rb @@ -20,7 +20,7 @@ class RuleImport < Import end def dry_run - { rules: rows.count } + { rules: rows_count } end def csv_template diff --git a/app/models/security.rb b/app/models/security.rb index ca5b38c4b..e91d04b90 100644 --- a/app/models/security.rb +++ b/app/models/security.rb @@ -27,7 +27,25 @@ class Security < ApplicationRecord ) end + def brandfetch_icon_url(width: 40, height: 40) + return nil unless Setting.brand_fetch_client_id.present? && website_url.present? + + domain = extract_domain(website_url) + return nil unless domain.present? + + "https://cdn.brandfetch.io/#{domain}/icon/fallback/lettermark/w/#{width}/h/#{height}?c=#{Setting.brand_fetch_client_id}" + end + private + + def extract_domain(url) + uri = URI.parse(url) + host = uri.host || url + host.sub(/\Awww\./, "") + rescue URI::InvalidURIError + nil + end + def upcase_symbols self.ticker = ticker.upcase self.exchange_operating_mic = exchange_operating_mic.upcase if exchange_operating_mic.present? diff --git a/app/models/security/price.rb b/app/models/security/price.rb index 4143a0c8e..4feb765a7 100644 --- a/app/models/security/price.rb +++ b/app/models/security/price.rb @@ -3,4 +3,13 @@ class Security::Price < ApplicationRecord validates :date, :price, :currency, presence: true validates :date, uniqueness: { scope: %i[security_id currency] } + + # Provisional prices from recent days that should be re-fetched + # - Must be provisional (gap-filled) + # - Must be from the last few days (configurable, default 7) + # - Includes weekends: they get fixed via cascade when weekday prices are fetched + scope :refetchable_provisional, ->(lookback_days: 7) { + where(provisional: true) + .where(date: lookback_days.days.ago.to_date..Date.current) + } end diff --git a/app/models/security/price/importer.rb b/app/models/security/price/importer.rb index 6fb064cbe..def62fc03 100644 --- a/app/models/security/price/importer.rb +++ b/app/models/security/price/importer.rb @@ -2,6 +2,8 @@ class Security::Price::Importer MissingSecurityPriceError = Class.new(StandardError) MissingStartPriceError = Class.new(StandardError) + PROVISIONAL_LOOKBACK_DAYS = 7 + def initialize(security:, security_provider:, start_date:, end_date:, clear_cache: false) @security = security @security_provider = security_provider @@ -24,6 +26,7 @@ class Security::Price::Importer end prev_price_value = start_price_value + prev_currency = prev_price_currency || db_price_currency || "USD" unless prev_price_value.present? Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{start_date}") @@ -40,28 +43,53 @@ class Security::Price::Importer end gapfilled_prices = effective_start_date.upto(end_date).map do |date| - db_price_value = db_prices[date]&.price - provider_price_value = provider_prices[date]&.price - provider_currency = provider_prices[date]&.currency + db_price = db_prices[date] + db_price_value = db_price&.price + provider_price = provider_prices[date] + provider_price_value = provider_price&.price + provider_currency = provider_price&.currency - chosen_price = if clear_cache - provider_price_value || db_price_value # overwrite when possible + has_provider_price = provider_price_value.present? && provider_price_value.to_f > 0 + has_db_price = db_price_value.present? && db_price_value.to_f > 0 + is_provisional = db_price&.provisional + + # Choose price and currency from the same source to avoid mismatches + chosen_price, chosen_currency = if clear_cache || is_provisional + # For provisional/cache clear: only use provider price, let gap-fill handle missing + # This ensures stale DB values don't persist when provider has no weekend data + [ provider_price_value, provider_currency ] + elsif has_db_price + # For non-provisional with valid DB price: preserve existing value (user edits) + [ db_price_value, db_price&.currency ] else - db_price_value || provider_price_value # fill gaps + # Fill gaps with provider data + [ provider_price_value, provider_currency ] end # Gap-fill using LOCF (last observation carried forward) - # Treat nil or zero prices as invalid and use previous price + # Treat nil or zero prices as invalid and use previous price/currency + used_locf = false if chosen_price.nil? || chosen_price.to_f <= 0 chosen_price = prev_price_value + chosen_currency = prev_currency + used_locf = true end prev_price_value = chosen_price + prev_currency = chosen_currency || prev_currency + + provisional = determine_provisional_status( + date: date, + has_provider_price: has_provider_price, + used_locf: used_locf, + existing_provisional: db_price&.provisional + ) { security_id: security.id, date: date, price: chosen_price, - currency: provider_currency || prev_price_currency || db_price_currency || "USD" + currency: chosen_currency || "USD", + provisional: provisional } end @@ -73,7 +101,7 @@ class Security::Price::Importer def provider_prices @provider_prices ||= begin - provider_fetch_start_date = effective_start_date - 5.days + provider_fetch_start_date = effective_start_date - PROVISIONAL_LOOKBACK_DAYS.days response = security_provider.fetch_security_prices( symbol: security.ticker, @@ -104,33 +132,97 @@ class Security::Price::Importer end def all_prices_exist? + return false if has_refetchable_provisional_prices? db_prices.count == expected_count end + def has_refetchable_provisional_prices? + Security::Price.where(security_id: security.id, date: start_date..end_date) + .refetchable_provisional(lookback_days: PROVISIONAL_LOOKBACK_DAYS) + .exists? + end + def expected_count (start_date..end_date).count end # Skip over ranges that already exist unless clearing cache + # Also includes dates with refetchable provisional prices def effective_start_date return start_date if clear_cache - (start_date..end_date).detect { |d| !db_prices.key?(d) } || end_date + refetchable_dates = Security::Price.where(security_id: security.id, date: start_date..end_date) + .refetchable_provisional(lookback_days: PROVISIONAL_LOOKBACK_DAYS) + .pluck(:date) + .to_set + + (start_date..end_date).detect do |d| + !db_prices.key?(d) || refetchable_dates.include?(d) + end || end_date end def start_price_value - provider_price_value = provider_prices.select { |date, _| date <= start_date } - .max_by { |date, _| date } - &.last&.price - db_price_value = db_prices[start_date]&.price - provider_price_value || db_price_value + # When processing full range (first sync), use original behavior + if effective_start_date == start_date + provider_price_value = provider_prices.select { |date, _| date <= start_date } + .max_by { |date, _| date } + &.last&.price + db_price_value = db_prices[start_date]&.price + + return provider_price_value if provider_price_value.present? && provider_price_value.to_f > 0 + return db_price_value if db_price_value.present? && db_price_value.to_f > 0 + return nil + end + + # For partial range (effective_start_date > start_date), use recent data + # This prevents stale prices from old trade dates propagating to current gap-fills + cutoff_date = effective_start_date + + # First try provider prices (most recent before cutoff) + provider_price_value = provider_prices + .select { |date, _| date < cutoff_date } + .max_by { |date, _| date } + &.last&.price + + return provider_price_value if provider_price_value.present? && provider_price_value.to_f > 0 + + # Fall back to most recent DB price before cutoff + currency = prev_price_currency || db_price_currency + Security::Price + .where(security_id: security.id) + .where("date < ?", cutoff_date) + .where("price > 0") + .where(provisional: false) + .then { |q| currency.present? ? q.where(currency: currency) : q } + .order(date: :desc) + .limit(1) + .pick(:price) + end + + def determine_provisional_status(date:, has_provider_price:, used_locf:, existing_provisional:) + # Provider returned real price => NOT provisional + return false if has_provider_price + + # Gap-filled (LOCF) => provisional if recent (including weekends) + # Weekend prices inherit uncertainty from Friday and get fixed via cascade + # when the next weekday sync fetches correct Friday price + if used_locf + is_recent = date >= PROVISIONAL_LOOKBACK_DAYS.days.ago.to_date + return is_recent + end + + # Otherwise preserve existing status + existing_provisional || false end def upsert_rows(rows) batch_size = 200 total_upsert_count = 0 + now = Time.current - rows.each_slice(batch_size) do |batch| + rows_with_timestamps = rows.map { |row| row.merge(updated_at: now) } + + rows_with_timestamps.each_slice(batch_size) do |batch| ids = Security::Price.upsert_all( batch, unique_by: %i[security_id date currency], diff --git a/app/models/security/provided.rb b/app/models/security/provided.rb index 58b7f50e6..1fbb1f272 100644 --- a/app/models/security/provided.rb +++ b/app/models/security/provided.rb @@ -68,7 +68,7 @@ module Security::Provided return end - if self.name.present? && self.logo_url.present? && !clear_cache + if self.name.present? && (self.logo_url.present? || self.website_url.present?) && !clear_cache return end @@ -81,6 +81,7 @@ module Security::Provided update( name: response.data.name, logo_url: response.data.logo_url, + website_url: response.data.links ) else Rails.logger.warn("Failed to fetch security info for #{ticker} from #{provider.class.name}: #{response.error.message}") diff --git a/app/models/simplefin/account_type_mapper.rb b/app/models/simplefin/account_type_mapper.rb index e01f4adb0..18eb8cbcf 100644 --- a/app/models/simplefin/account_type_mapper.rb +++ b/app/models/simplefin/account_type_mapper.rb @@ -12,6 +12,11 @@ module Simplefin CREDIT_NAME_KEYWORDS = /\b(credit|card)\b/i.freeze CREDIT_BRAND_KEYWORDS = /\b(visa|mastercard|amex|american express|discover|apple card|freedom unlimited|quicksilver)\b/i.freeze LOAN_KEYWORDS = /\b(loan|mortgage|heloc|line of credit|loc)\b/i.freeze + CHECKING_KEYWORDS = /\b(checking|chequing|dda|demand deposit)\b/i.freeze + SAVINGS_KEYWORDS = /\b(savings|sav|money market|mma|high.yield)\b/i.freeze + CRYPTO_KEYWORDS = /\b(bitcoin|btc|ethereum|eth|crypto|cryptocurrency|litecoin|dogecoin|solana)\b/i.freeze + # "Cash" as a standalone name (not "cash back", "cash rewards", etc.) + CASH_ACCOUNT_PATTERN = /\A\s*cash\s*\z/i.freeze # Explicit investment subtype tokens mapped to known SUBTYPES keys EXPLICIT_INVESTMENT_TOKENS = { @@ -53,18 +58,23 @@ module Simplefin end end - # 1) Holdings present => Investment (high confidence) + # 1) Crypto keywords → Crypto account (check before holdings since crypto accounts may have holdings) + if CRYPTO_KEYWORDS.match?(nm) + return Inference.new(accountable_type: "Crypto", confidence: :high) + end + + # 2) Holdings present => Investment (high confidence) if holdings_present # Do not guess generic retirement; explicit tokens handled above return Inference.new(accountable_type: "Investment", subtype: nil, confidence: :high) end - # 2) Name suggests LOAN (high confidence) + # 3) Name suggests LOAN (high confidence) if LOAN_KEYWORDS.match?(nm) return Inference.new(accountable_type: "Loan", confidence: :high) end - # 3) Credit card signals + # 4) Credit card signals # - Name contains credit/card (medium to high) # - Card brands (Visa/Mastercard/Amex/Discover/Apple Card) → high # - Or negative balance with available-balance present (medium) @@ -76,14 +86,26 @@ module Simplefin return Inference.new(accountable_type: "CreditCard", confidence: :high) end - # 4) Retirement keywords without holdings still point to Investment (retirement) + # 5) Retirement keywords without holdings still point to Investment (retirement) if RETIREMENT_KEYWORDS.match?(nm) # If the name contains 'brokerage', avoid forcing retirement subtype subtype = BROKERAGE_KEYWORD.match?(nm) ? nil : "retirement" return Inference.new(accountable_type: "Investment", subtype: subtype, confidence: :high) end - # 5) Default + # 6) Checking/Savings/Cash accounts (high confidence when name explicitly says so) + if CHECKING_KEYWORDS.match?(nm) + return Inference.new(accountable_type: "Depository", subtype: "checking", confidence: :high) + end + if SAVINGS_KEYWORDS.match?(nm) + return Inference.new(accountable_type: "Depository", subtype: "savings", confidence: :high) + end + # "Cash" as a standalone account name (like Cash App's "Cash" account) → checking + if CASH_ACCOUNT_PATTERN.match?(nm) + return Inference.new(accountable_type: "Depository", subtype: "checking", confidence: :high) + end + + # 7) Default - unknown account type, let user decide Inference.new(accountable_type: "Depository", confidence: :low) end end diff --git a/app/models/simplefin_account.rb b/app/models/simplefin_account.rb index 3961bea63..d505d41e9 100644 --- a/app/models/simplefin_account.rb +++ b/app/models/simplefin_account.rb @@ -80,7 +80,7 @@ class SimplefinAccount < ApplicationRecord end def parse_currency(currency_value) - return "USD" if currency_value.nil? + return "USD" if currency_value.blank? # SimpleFin currency can be a 3-letter code or a URL for custom currencies if currency_value.start_with?("http") diff --git a/app/models/simplefin_account/investments/holdings_processor.rb b/app/models/simplefin_account/investments/holdings_processor.rb index ae7723206..052c28cb5 100644 --- a/app/models/simplefin_account/investments/holdings_processor.rb +++ b/app/models/simplefin_account/investments/holdings_processor.rb @@ -9,13 +9,29 @@ class SimplefinAccount::Investments::HoldingsProcessor holdings_data.each do |simplefin_holding| begin - symbol = simplefin_holding["symbol"] + symbol = simplefin_holding["symbol"].presence holding_id = simplefin_holding["id"] + description = simplefin_holding["description"].to_s.strip Rails.logger.debug({ event: "simplefin.holding.start", sfa_id: simplefin_account.id, account_id: account&.id, id: holding_id, symbol: symbol, raw: simplefin_holding }.to_json) - unless symbol.present? && holding_id.present? - Rails.logger.debug({ event: "simplefin.holding.skip", reason: "missing_symbol_or_id", id: holding_id, symbol: symbol }.to_json) + unless holding_id.present? + Rails.logger.debug({ event: "simplefin.holding.skip", reason: "missing_id", id: holding_id, symbol: symbol }.to_json) + next + end + + # If symbol is missing but we have a description, create a synthetic ticker + # This allows tracking holdings like 401k funds that don't have standard symbols + # Append a hash suffix to ensure uniqueness for similar descriptions + if symbol.blank? && description.present? + normalized = description.gsub(/[^a-zA-Z0-9]/, "_").upcase.truncate(24, omission: "") + hash_suffix = Digest::MD5.hexdigest(description)[0..4].upcase + symbol = "CUSTOM:#{normalized}_#{hash_suffix}" + Rails.logger.info("SimpleFin: using synthetic ticker #{symbol} for holding #{holding_id} (#{description})") + end + + unless symbol.present? + Rails.logger.debug({ event: "simplefin.holding.skip", reason: "no_symbol_or_description", id: holding_id }.to_json) next end @@ -57,7 +73,7 @@ class SimplefinAccount::Investments::HoldingsProcessor security: security, quantity: qty, amount: computed_amount, - currency: simplefin_holding["currency"] || "USD", + currency: simplefin_holding["currency"].presence || "USD", date: holding_date, price: price, cost_basis: cost_basis, @@ -101,14 +117,23 @@ class SimplefinAccount::Investments::HoldingsProcessor if !sym.include?(":") && (is_crypto_account || is_crypto_symbol || mentions_crypto) sym = "CRYPTO:#{sym}" end + + # Custom tickers (from holdings without symbols) should always be offline + is_custom = sym.start_with?("CUSTOM:") + # Use Security::Resolver to find or create the security, but be resilient begin + if is_custom + # Skip resolver for custom tickers - create offline security directly + raise "Custom ticker - skipping resolver" + end Security::Resolver.new(sym).resolve rescue => e # If provider search fails or any unexpected error occurs, fall back to an offline security - Rails.logger.warn "SimpleFin: resolver failed for symbol=#{sym}: #{e.class} - #{e.message}; falling back to offline security" + Rails.logger.warn "SimpleFin: resolver failed for symbol=#{sym}: #{e.class} - #{e.message}; falling back to offline security" unless is_custom Security.find_or_initialize_by(ticker: sym).tap do |sec| sec.offline = true if sec.respond_to?(:offline) && sec.offline != true + sec.name = description.presence if sec.name.blank? && description.present? sec.save! if sec.changed? end end diff --git a/app/models/simplefin_account/investments/transactions_processor.rb b/app/models/simplefin_account/investments/transactions_processor.rb index 6644a64dc..09d89f3ab 100644 --- a/app/models/simplefin_account/investments/transactions_processor.rb +++ b/app/models/simplefin_account/investments/transactions_processor.rb @@ -1,85 +1,33 @@ # SimpleFin Investment transactions processor -# Processes investment-specific transactions like trades, dividends, etc. +# +# NOTE: SimpleFIN transactions (dividends, contributions, etc.) for investment accounts +# are already processed by SimplefinAccount::Transactions::Processor, which handles ALL +# account types including investments. That processor uses SimplefinEntry::Processor +# which captures full metadata (merchant, notes, extra data). +# +# This processor is intentionally a no-op for transactions to avoid: +# 1. Duplicate processing of the same transactions +# 2. Overwriting richer data with less complete data +# +# Unlike Plaid (which has a separate investment_transactions endpoint), SimpleFIN returns +# all transactions in a single `transactions` array regardless of account type. +# +# Holdings are processed separately by SimplefinAccount::Investments::HoldingsProcessor. class SimplefinAccount::Investments::TransactionsProcessor def initialize(simplefin_account) @simplefin_account = simplefin_account end def process - return unless simplefin_account.current_account&.accountable_type == "Investment" - return unless simplefin_account.raw_transactions_payload.present? - - transactions_data = simplefin_account.raw_transactions_payload - - transactions_data.each do |transaction_data| - process_investment_transaction(transaction_data) - end + # Intentionally a no-op for transactions. + # SimpleFIN investment transactions are already processed by the regular + # SimplefinAccount::Transactions::Processor which handles all account types. + # + # This avoids duplicate processing and ensures the richer metadata from + # SimplefinEntry::Processor (merchant, notes, extra) is preserved. + Rails.logger.debug "SimplefinAccount::Investments::TransactionsProcessor - Skipping (transactions handled by SimplefinAccount::Transactions::Processor)" end private attr_reader :simplefin_account - - def account - simplefin_account.current_account - end - - def process_investment_transaction(transaction_data) - data = transaction_data.with_indifferent_access - - amount = parse_amount(data[:amount]) - posted_date = parse_date(data[:posted]) - external_id = "simplefin_#{data[:id]}" - - # Use the unified import adapter for consistent handling - import_adapter.import_transaction( - external_id: external_id, - amount: amount, - currency: account.currency, - date: posted_date, - name: data[:description] || "Investment transaction", - source: "simplefin" - ) - rescue => e - Rails.logger.error("Failed to process SimpleFin investment transaction #{data[:id]}: #{e.message}") - end - - def import_adapter - @import_adapter ||= Account::ProviderImportAdapter.new(account) - end - - def parse_amount(amount_value) - parsed_amount = case amount_value - when String - BigDecimal(amount_value) - when Numeric - BigDecimal(amount_value.to_s) - else - BigDecimal("0") - end - - # SimpleFin uses banking convention, Maybe expects opposite - -parsed_amount - rescue ArgumentError => e - Rails.logger.error "Failed to parse SimpleFin investment transaction amount: #{amount_value.inspect} - #{e.message}" - BigDecimal("0") - end - - def parse_date(date_value) - case date_value - when String - Date.parse(date_value) - when Integer, Float - Time.at(date_value).to_date - when Time, DateTime - date_value.to_date - when Date - date_value - else - Rails.logger.error("SimpleFin investment transaction has invalid date value: #{date_value.inspect}") - raise ArgumentError, "Invalid date format: #{date_value.inspect}" - end - rescue ArgumentError, TypeError => e - Rails.logger.error("Failed to parse SimpleFin investment transaction date '#{date_value}': #{e.message}") - raise ArgumentError, "Unable to parse transaction date: #{date_value.inspect}" - end end diff --git a/app/models/simplefin_account/liabilities/overpayment_analyzer.rb b/app/models/simplefin_account/liabilities/overpayment_analyzer.rb new file mode 100644 index 000000000..0c2a16ad6 --- /dev/null +++ b/app/models/simplefin_account/liabilities/overpayment_analyzer.rb @@ -0,0 +1,225 @@ +# frozen_string_literal: true + +# Classifies a SimpleFIN liability balance as :debt (owe, show positive) +# or :credit (overpaid, show negative) using recent transaction history. +# +# Notes: +# - Preferred signal: already-imported Entry records for the linked Account +# (they are in Maybe's convention: expenses/charges > 0, payments < 0). +# - Fallback signal: provider raw transactions payload with amounts converted +# to Maybe convention by negating SimpleFIN's banking convention. +# - Returns :unknown when evidence is insufficient; callers should fallback +# to existing sign-only normalization. +class SimplefinAccount::Liabilities::OverpaymentAnalyzer + include SimplefinNumericHelpers + Result = Struct.new(:classification, :reason, :metrics, keyword_init: true) + + DEFAULTS = { + window_days: 120, + min_txns: 10, + min_payments: 2, + epsilon_base: BigDecimal("0.50"), + statement_guard_days: 5, + sticky_days: 7 + }.freeze + + def initialize(simplefin_account, observed_balance:, now: Time.current) + @sfa = simplefin_account + @observed = to_decimal(observed_balance) + @now = now + end + + def call + return unknown("flag disabled") unless enabled? + return unknown("no-account") unless (account = @sfa.current_account) + + # Only applicable for liabilities + return unknown("not-liability") unless %w[CreditCard Loan].include?(account.accountable_type) + + # Near-zero observed balances are too noisy to infer + return unknown("near-zero-balance") if @observed.abs <= epsilon_base + + # Sticky cache via Rails.cache to avoid DB schema changes + sticky = read_sticky + if sticky && sticky[:expires_at] > @now + return Result.new(classification: sticky[:value].to_sym, reason: "sticky_hint", metrics: {}) + end + + txns = gather_transactions(account) + return unknown("insufficient-txns") if txns.size < min_txns + + metrics = compute_metrics(txns) + cls, reason = classify(metrics) + + if %i[credit debt].include?(cls) + write_sticky(cls) + end + + Result.new(classification: cls, reason: reason, metrics: metrics) + end + + private + + def enabled? + # Setting override takes precedence, then ENV, then default enabled + setting_val = Setting["simplefin_cc_overpayment_detection"] + return parse_bool(setting_val) unless setting_val.nil? + + env_val = ENV["SIMPLEFIN_CC_OVERPAYMENT_HEURISTIC"] + return parse_bool(env_val) if env_val.present? + + true # Default enabled + end + + def parse_bool(value) + case value + when true, false then value + when String then %w[1 true yes on].include?(value.downcase) + else false + end + end + + def window_days + val = Setting["simplefin_cc_overpayment_window_days"] + v = (val.presence || DEFAULTS[:window_days]).to_i + v > 0 ? v : DEFAULTS[:window_days] + end + + def min_txns + val = Setting["simplefin_cc_overpayment_min_txns"] + v = (val.presence || DEFAULTS[:min_txns]).to_i + v > 0 ? v : DEFAULTS[:min_txns] + end + + def min_payments + val = Setting["simplefin_cc_overpayment_min_payments"] + v = (val.presence || DEFAULTS[:min_payments]).to_i + v > 0 ? v : DEFAULTS[:min_payments] + end + + def epsilon_base + val = Setting["simplefin_cc_overpayment_epsilon_base"] + d = to_decimal(val.presence || DEFAULTS[:epsilon_base]) + d > 0 ? d : DEFAULTS[:epsilon_base] + end + + def statement_guard_days + val = Setting["simplefin_cc_overpayment_statement_guard_days"] + v = (val.presence || DEFAULTS[:statement_guard_days]).to_i + v >= 0 ? v : DEFAULTS[:statement_guard_days] + end + + def sticky_days + val = Setting["simplefin_cc_overpayment_sticky_days"] + v = (val.presence || DEFAULTS[:sticky_days]).to_i + v > 0 ? v : DEFAULTS[:sticky_days] + end + + def gather_transactions(account) + start_date = (@now.to_date - window_days.days) + + # Prefer materialized entries + entries = account.entries.where("date >= ?", start_date).select(:amount, :date) + txns = entries.map { |e| { amount: to_decimal(e.amount), date: e.date } } + return txns if txns.size >= min_txns + + # Fallback: provider raw payload + raw = Array(@sfa.raw_transactions_payload) + raw_txns = raw.filter_map do |tx| + h = tx.with_indifferent_access + amt = convert_provider_amount(h[:amount]) + d = ( + Simplefin::DateUtils.parse_provider_date(h[:posted]) || + Simplefin::DateUtils.parse_provider_date(h[:transacted_at]) + ) + next nil unless d + next nil if d < start_date + { amount: amt, date: d } + end + raw_txns + rescue => e + Rails.logger.debug("SimpleFIN transaction gathering failed for sfa=#{@sfa.id}: #{e.class} - #{e.message}") + [] + end + + def compute_metrics(txns) + charges = BigDecimal("0") + payments = BigDecimal("0") + payments_count = 0 + recent_payment = false + guard_since = (@now.to_date - statement_guard_days.days) + + txns.each do |t| + amt = to_decimal(t[:amount]) + if amt.positive? + charges += amt + elsif amt.negative? + payments += -amt + payments_count += 1 + recent_payment ||= (t[:date] >= guard_since) + end + end + + net = charges - payments + { + charges_total: charges, + payments_total: payments, + payments_count: payments_count, + tx_count: txns.size, + net: net, + observed: @observed, + window_days: window_days, + recent_payment: recent_payment + } + end + + def classify(m) + # Boundary guard: a single very recent payment may create temporary credit before charges post + if m[:recent_payment] && m[:payments_count] <= 2 + return [ :unknown, "statement-guard" ] + end + + eps = [ epsilon_base, (@observed.abs * BigDecimal("0.005")) ].max + + # Overpayment (credit): payments exceed charges by at least the observed balance (within eps) + if (m[:payments_total] - m[:charges_total]) >= (@observed.abs - eps) + return [ :credit, "payments>=charges+observed-eps" ] + end + + # Debt: charges exceed payments beyond epsilon + if (m[:charges_total] - m[:payments_total]) > eps && m[:payments_count] >= min_payments + return [ :debt, "charges>payments+eps" ] + end + + [ :unknown, "ambiguous" ] + end + + def convert_provider_amount(val) + amt = case val + when String then BigDecimal(val) rescue BigDecimal("0") + when Numeric then BigDecimal(val.to_s) + else BigDecimal("0") + end + # Negate to convert banking convention (expenses negative) -> Maybe convention + -amt + end + + def read_sticky + Rails.cache.read(sticky_key) + end + + def write_sticky(value) + Rails.cache.write(sticky_key, { value: value.to_s, expires_at: @now + sticky_days.days }, expires_in: sticky_days.days) + end + + def sticky_key + id = @sfa.id || "tmp:#{@sfa.object_id}" + "simplefin:sfa:#{id}:liability_sign_hint" + end + + # numeric coercion handled by SimplefinNumericHelpers#to_decimal + + def unknown(reason) + Result.new(classification: :unknown, reason: reason, metrics: {}) + end +end diff --git a/app/models/simplefin_account/processor.rb b/app/models/simplefin_account/processor.rb index 4c82db4b4..569f515cd 100644 --- a/app/models/simplefin_account/processor.rb +++ b/app/models/simplefin_account/processor.rb @@ -1,4 +1,5 @@ class SimplefinAccount::Processor + include SimplefinNumericHelpers attr_reader :simplefin_account def initialize(simplefin_account) @@ -39,15 +40,90 @@ class SimplefinAccount::Processor # Update account balance and cash balance from latest SimpleFin data account = simplefin_account.current_account - balance = simplefin_account.current_balance || simplefin_account.available_balance || 0 - # Normalize balances for liabilities (SimpleFIN typically uses opposite sign) - # App convention: - # - Liabilities: positive => you owe; negative => provider owes you (overpayment/credit) - # Since providers often send the opposite sign, ALWAYS invert for liabilities so - # that both debt and overpayment cases are represented correctly. - if [ "CreditCard", "Loan" ].include?(account.accountable_type) - balance = -balance + # Extract raw values from SimpleFIN snapshot + bal = to_decimal(simplefin_account.current_balance) + avail = to_decimal(simplefin_account.available_balance) + + # Choose an observed value prioritizing posted balance first + observed = bal.nonzero? ? bal : avail + + # Determine if this should be treated as a liability for normalization + is_linked_liability = [ "CreditCard", "Loan" ].include?(account.accountable_type) + raw = (simplefin_account.raw_payload || {}).with_indifferent_access + org = (simplefin_account.org_data || {}).with_indifferent_access + inferred = Simplefin::AccountTypeMapper.infer( + name: simplefin_account.name, + holdings: raw[:holdings], + extra: simplefin_account.extra, + balance: bal, + available_balance: avail, + institution: org[:name] + ) rescue nil + is_mapper_liability = inferred && [ "CreditCard", "Loan" ].include?(inferred.accountable_type) + is_liability = is_linked_liability || is_mapper_liability + + if is_mapper_liability && !is_linked_liability + Rails.logger.warn( + "SimpleFIN liability normalization: linked account #{account.id} type=#{account.accountable_type} " \ + "appears to be liability via mapper (#{inferred.accountable_type}). Normalizing as liability; consider relinking." + ) + end + + balance = observed + if is_liability + # 1) Try transaction-history heuristic when enabled + begin + result = SimplefinAccount::Liabilities::OverpaymentAnalyzer + .new(simplefin_account, observed_balance: observed) + .call + + case result.classification + when :credit + balance = -observed.abs + Rails.logger.info( + "SimpleFIN overpayment heuristic: classified as credit for sfa=#{simplefin_account.id}, " \ + "observed=#{observed.to_s('F')} metrics=#{result.metrics.slice(:charges_total, :payments_total, :tx_count).inspect}" + ) + Sentry.add_breadcrumb(Sentry::Breadcrumb.new( + category: "simplefin", + message: "liability_sign=credit", + data: { sfa_id: simplefin_account.id, observed: observed.to_s("F") } + )) rescue nil + when :debt + balance = observed.abs + Rails.logger.info( + "SimpleFIN overpayment heuristic: classified as debt for sfa=#{simplefin_account.id}, " \ + "observed=#{observed.to_s('F')} metrics=#{result.metrics.slice(:charges_total, :payments_total, :tx_count).inspect}" + ) + Sentry.add_breadcrumb(Sentry::Breadcrumb.new( + category: "simplefin", + message: "liability_sign=debt", + data: { sfa_id: simplefin_account.id, observed: observed.to_s("F") } + )) rescue nil + else + # 2) Fall back to existing sign-only logic (log unknown for observability) + begin + obs = { + reason: result.reason, + tx_count: result.metrics[:tx_count], + charges_total: result.metrics[:charges_total], + payments_total: result.metrics[:payments_total], + observed: observed.to_s("F") + }.compact + Rails.logger.info("SimpleFIN overpayment heuristic: unknown; falling back #{obs.inspect}") + rescue + # no-op + end + balance = normalize_liability_balance(observed, bal, avail) + end + rescue NameError + # Analyzer not loaded; keep legacy behavior + balance = normalize_liability_balance(observed, bal, avail) + rescue => e + Rails.logger.warn("SimpleFIN overpayment heuristic error for sfa=#{simplefin_account.id}: #{e.class} - #{e.message}") + balance = normalize_liability_balance(observed, bal, avail) + end end # Calculate cash balance correctly for investment accounts @@ -98,4 +174,19 @@ class SimplefinAccount::Processor ) end end + + # Helpers + # to_decimal and same_sign? provided by SimplefinNumericHelpers concern + + def normalize_liability_balance(observed, bal, avail) + both_present = bal.nonzero? && avail.nonzero? + if both_present && same_sign?(bal, avail) + if bal.positive? && avail.positive? + return -observed.abs + elsif bal.negative? && avail.negative? + return observed.abs + end + end + -observed + end end diff --git a/app/models/simplefin_account/transactions/processor.rb b/app/models/simplefin_account/transactions/processor.rb index ea4725066..da4c96dda 100644 --- a/app/models/simplefin_account/transactions/processor.rb +++ b/app/models/simplefin_account/transactions/processor.rb @@ -6,18 +6,40 @@ class SimplefinAccount::Transactions::Processor end def process - return unless simplefin_account.raw_transactions_payload.present? + transactions = simplefin_account.raw_transactions_payload.to_a + acct = simplefin_account.current_account + acct_info = acct ? "Account id=#{acct.id} name='#{acct.name}' type=#{acct.accountable_type}" : "NO LINKED ACCOUNT" + + if transactions.empty? + Rails.logger.info "SimplefinAccount::Transactions::Processor - No transactions in raw_transactions_payload for simplefin_account #{simplefin_account.id} (#{simplefin_account.name}) - #{acct_info}" + return + end + + Rails.logger.info "SimplefinAccount::Transactions::Processor - Processing #{transactions.count} transactions for simplefin_account #{simplefin_account.id} (#{simplefin_account.name}) - #{acct_info}" + + # Log first few transaction IDs for debugging + sample_ids = transactions.first(3).map { |t| t.is_a?(Hash) ? (t[:id] || t["id"]) : nil }.compact + Rails.logger.info "SimplefinAccount::Transactions::Processor - Sample transaction IDs: #{sample_ids.inspect}" + + processed_count = 0 + error_count = 0 # Each entry is processed inside a transaction, but to avoid locking up the DB when # there are hundreds or thousands of transactions, we process them individually. - simplefin_account.raw_transactions_payload.each do |transaction_data| + transactions.each do |transaction_data| SimplefinEntry::Processor.new( transaction_data, simplefin_account: simplefin_account ).process + processed_count += 1 rescue => e - Rails.logger.error "Error processing SimpleFin transaction: #{e.message}" + error_count += 1 + tx_id = transaction_data.is_a?(Hash) ? (transaction_data[:id] || transaction_data["id"]) : nil + Rails.logger.error "SimplefinAccount::Transactions::Processor - Error processing transaction #{tx_id}: #{e.class} - #{e.message}" + Rails.logger.error e.backtrace.first(5).join("\n") if e.backtrace end + + Rails.logger.info "SimplefinAccount::Transactions::Processor - Completed for simplefin_account #{simplefin_account.id}: #{processed_count} processed, #{error_count} errors" end private diff --git a/app/models/simplefin_entry/processor.rb b/app/models/simplefin_entry/processor.rb index de0802c90..4ef68449e 100644 --- a/app/models/simplefin_entry/processor.rb +++ b/app/models/simplefin_entry/processor.rb @@ -34,12 +34,13 @@ class SimplefinEntry::Processor # Include provider-supplied extra hash if present sf["extra"] = data[:extra] if data[:extra].is_a?(Hash) - # Pending detection: honor provider flag or infer from missing/zero posted with present transacted_at - posted_val = data[:posted] - posted_missing = posted_val.blank? || posted_val == 0 || posted_val == "0" - if ActiveModel::Type::Boolean.new.cast(data[:pending]) || (posted_missing && data[:transacted_at].present?) + # Pending detection: only use explicit provider flag + # We always set the key (true or false) to ensure deep_merge overwrites any stale value + if ActiveModel::Type::Boolean.new.cast(data[:pending]) sf["pending"] = true Rails.logger.debug("SimpleFIN: flagged pending transaction #{external_id}") + else + sf["pending"] = false end # FX metadata: when tx currency differs from account currency @@ -87,7 +88,7 @@ class SimplefinEntry::Processor elsif description.present? description else - data[:memo] || "Unknown transaction" + data[:memo] || I18n.t("transactions.unknown_name") end end @@ -142,7 +143,7 @@ class SimplefinEntry::Processor def posted_date val = data[:posted] - # Treat 0 / "0" as missing to avoid Unix epoch 1970-01-01 for pendings + # Treat 0 / "0" as missing to avoid Unix epoch 1970-01-01 return nil if val == 0 || val == "0" Simplefin::DateUtils.parse_provider_date(val) end diff --git a/app/models/simplefin_item.rb b/app/models/simplefin_item.rb index cfad9c19d..45acd07b7 100644 --- a/app/models/simplefin_item.rb +++ b/app/models/simplefin_item.rb @@ -56,9 +56,164 @@ class SimplefinItem < ApplicationRecord end def process_accounts - simplefin_accounts.joins(:account).each do |simplefin_account| + # Process accounts linked via BOTH legacy FK and AccountProvider + # Use direct query to ensure fresh data from DB, bypassing any association cache + all_accounts = SimplefinAccount.where(simplefin_item_id: id).includes(:account, :linked_account, account_provider: :account).to_a + + Rails.logger.info "=" * 60 + Rails.logger.info "SimplefinItem#process_accounts START - Item #{id} (#{name})" + Rails.logger.info " Total SimplefinAccounts: #{all_accounts.count}" + + # Log all accounts for debugging + all_accounts.each do |sfa| + acct = sfa.current_account + Rails.logger.info " - SimplefinAccount id=#{sfa.id} sf_account_id=#{sfa.account_id} name='#{sfa.name}'" + Rails.logger.info " linked_account: #{sfa.linked_account&.id || 'nil'}, account: #{sfa.account&.id || 'nil'}, current_account: #{acct&.id || 'nil'}" + Rails.logger.info " raw_transactions_payload count: #{sfa.raw_transactions_payload.to_a.count}" + end + + # First, try to repair stale linkages (old SimplefinAccount linked but new one has data) + repair_stale_linkages(all_accounts) + + # Re-fetch after repairs - use direct query for fresh data + all_accounts = SimplefinAccount.where(simplefin_item_id: id).includes(:account, :linked_account, account_provider: :account).to_a + + linked = all_accounts.select { |sfa| sfa.current_account.present? } + unlinked = all_accounts.reject { |sfa| sfa.current_account.present? } + + Rails.logger.info "SimplefinItem#process_accounts - After repair: #{linked.count} linked, #{unlinked.count} unlinked" + + # Log unlinked accounts with transactions for debugging + unlinked_with_txns = unlinked.select { |sfa| sfa.raw_transactions_payload.to_a.any? } + if unlinked_with_txns.any? + Rails.logger.warn "SimplefinItem#process_accounts - #{unlinked_with_txns.count} UNLINKED account(s) have transactions that won't be processed:" + unlinked_with_txns.each do |sfa| + Rails.logger.warn " - SimplefinAccount id=#{sfa.id} name='#{sfa.name}' sf_account_id=#{sfa.account_id} txn_count=#{sfa.raw_transactions_payload.to_a.count}" + end + end + + linked.each do |simplefin_account| + acct = simplefin_account.current_account + Rails.logger.info "SimplefinItem#process_accounts - Processing: SimplefinAccount id=#{simplefin_account.id} name='#{simplefin_account.name}' -> Account id=#{acct.id} name='#{acct.name}' type=#{acct.accountable_type}" SimplefinAccount::Processor.new(simplefin_account).process end + + Rails.logger.info "SimplefinItem#process_accounts END" + Rails.logger.info "=" * 60 + end + + # Repairs stale linkages when user re-adds institution in SimpleFIN. + # When a user deletes and re-adds an institution in SimpleFIN, new account IDs are generated. + # This causes old SimplefinAccounts to remain "linked" but stale (no new data), + # while new SimplefinAccounts have data but are unlinked. + # This method detects such cases and transfers the linkage from old to new. + def repair_stale_linkages(all_accounts) + linked = all_accounts.select { |sfa| sfa.current_account.present? } + unlinked = all_accounts.reject { |sfa| sfa.current_account.present? } + + Rails.logger.info "SimplefinItem#repair_stale_linkages - #{linked.count} linked, #{unlinked.count} unlinked SimplefinAccounts" + + # Find unlinked accounts that have transactions + unlinked_with_data = unlinked.select { |sfa| sfa.raw_transactions_payload.to_a.any? } + + if unlinked_with_data.any? + Rails.logger.info "SimplefinItem#repair_stale_linkages - Found #{unlinked_with_data.count} unlinked SimplefinAccount(s) with transactions:" + unlinked_with_data.each do |sfa| + Rails.logger.info " - id=#{sfa.id} name='#{sfa.name}' account_id=#{sfa.account_id} txn_count=#{sfa.raw_transactions_payload.to_a.count}" + end + end + + return if unlinked_with_data.empty? + + # For each unlinked account with data, try to find a matching linked account + unlinked_with_data.each do |new_sfa| + # Find linked SimplefinAccount with same name (case-insensitive). + stale_matches = linked.select do |old_sfa| + old_sfa.name.to_s.downcase.strip == new_sfa.name.to_s.downcase.strip + end + + if stale_matches.size > 1 + Rails.logger.warn "SimplefinItem#repair_stale_linkages - Multiple linked accounts match '#{new_sfa.name}': #{stale_matches.map(&:id).join(', ')}. Using first match." + end + + stale_match = stale_matches.first + next unless stale_match + + account = stale_match.current_account + Rails.logger.info "SimplefinItem#repair_stale_linkages - Found matching accounts:" + Rails.logger.info " - OLD: SimplefinAccount id=#{stale_match.id} account_id=#{stale_match.account_id} txn_count=#{stale_match.raw_transactions_payload.to_a.count}" + Rails.logger.info " - NEW: SimplefinAccount id=#{new_sfa.id} account_id=#{new_sfa.account_id} txn_count=#{new_sfa.raw_transactions_payload.to_a.count}" + Rails.logger.info " - Linked to Account: '#{account.name}' (id=#{account.id})" + + # Transfer the linkage from old to new + begin + # Merge transactions from old to new before transferring + old_transactions = stale_match.raw_transactions_payload.to_a + new_transactions = new_sfa.raw_transactions_payload.to_a + if old_transactions.any? + Rails.logger.info "SimplefinItem#repair_stale_linkages - Merging #{old_transactions.count} transactions from old SimplefinAccount" + merged = merge_transactions(old_transactions, new_transactions) + new_sfa.update!(raw_transactions_payload: merged) + end + + # Check if linked via legacy FK (use to_s for UUID comparison safety) + if account.simplefin_account_id.to_s == stale_match.id.to_s + account.simplefin_account_id = new_sfa.id + account.save! + end + + # Check if linked via AccountProvider + if stale_match.account_provider.present? + Rails.logger.info "SimplefinItem#repair_stale_linkages - Transferring AccountProvider linkage from SimplefinAccount #{stale_match.id} to #{new_sfa.id}" + stale_match.account_provider.update!(provider: new_sfa) + end + + # If the new one doesn't have an AccountProvider yet, create one + new_sfa.ensure_account_provider! + + Rails.logger.info "SimplefinItem#repair_stale_linkages - Successfully transferred linkage for Account '#{account.name}' to SimplefinAccount id=#{new_sfa.id}" + + # Clear transactions from stale SimplefinAccount and leave it orphaned + # We don't destroy it because has_one :account, dependent: :nullify would nullify the FK we just set + # IMPORTANT: Use update_all to bypass AR associations - stale_match.update! would + # trigger autosave on the preloaded account association, reverting the FK we just set! + SimplefinAccount.where(id: stale_match.id).update_all(raw_transactions_payload: [], raw_holdings_payload: []) + Rails.logger.info "SimplefinItem#repair_stale_linkages - Cleared data from stale SimplefinAccount id=#{stale_match.id} (leaving orphaned)" + rescue => e + Rails.logger.error "SimplefinItem#repair_stale_linkages - Failed to transfer linkage: #{e.class} - #{e.message}" + Rails.logger.error e.backtrace.first(5).join("\n") if e.backtrace + end + end + end + + # Merge two arrays of transactions, deduplicating by ID. + # Fallback: uses composite key [posted, amount, description] when ID/fitid missing. + # + # Known edge cases with composite key fallback: + # 1. False positives: Two distinct transactions with identical posted/amount/description + # will be incorrectly merged (rare but possible). + # 2. Type inconsistency: If posted varies in type (String vs Integer), keys won't match. + # 3. Description variations: Minor differences (whitespace, case) prevent matching. + # + # SimpleFIN typically provides transaction IDs, so this fallback is rarely needed. + def merge_transactions(old_txns, new_txns) + by_id = {} + + # Add old transactions first + old_txns.each do |tx| + t = tx.with_indifferent_access + key = t[:id] || t[:fitid] || [ t[:posted], t[:amount], t[:description] ] + by_id[key] = tx + end + + # Add new transactions (overwrite old with same ID) + new_txns.each do |tx| + t = tx.with_indifferent_access + key = t[:id] || t[:fitid] || [ t[:posted], t[:amount], t[:description] ] + by_id[key] = tx + end + + by_id.values end def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) @@ -192,6 +347,58 @@ class SimplefinItem < ApplicationRecord end end + # Detect if sync data appears stale (no new transactions for extended period) + # Returns a hash with :stale (boolean) and :message (string) if stale + def stale_sync_status + return { stale: false } unless last_synced_at.present? + + # Check if last sync was more than 3 days ago + days_since_sync = (Date.current - last_synced_at.to_date).to_i + if days_since_sync > 3 + return { + stale: true, + days_since_sync: days_since_sync, + message: "Last successful sync was #{days_since_sync} days ago. Your SimpleFin connection may need attention." + } + end + + # Check if linked accounts have recent transactions + linked_accounts = accounts + return { stale: false } if linked_accounts.empty? + + # Find the most recent transaction date across all linked accounts + latest_transaction_date = Entry.where(account_id: linked_accounts.map(&:id)) + .where(entryable_type: "Transaction") + .maximum(:date) + + if latest_transaction_date.present? + days_since_transaction = (Date.current - latest_transaction_date).to_i + if days_since_transaction > 14 + return { + stale: true, + days_since_transaction: days_since_transaction, + message: "No new transactions in #{days_since_transaction} days. Check your SimpleFin dashboard to ensure your bank connections are active." + } + end + end + + { stale: false } + end + + # Check if the SimpleFin connection needs user attention + def needs_attention? + requires_update? || stale_sync_status[:stale] || pending_account_setup? + end + + # Get a summary of issues requiring attention + def attention_summary + issues = [] + issues << "Connection needs update" if requires_update? + issues << stale_sync_status[:message] if stale_sync_status[:stale] + issues << "Accounts need setup" if pending_account_setup? + issues + end + private def remove_simplefin_item # SimpleFin doesn't require server-side cleanup like Plaid diff --git a/app/models/simplefin_item/importer.rb b/app/models/simplefin_item/importer.rb index d7d7e7226..f4d3d9e3a 100644 --- a/app/models/simplefin_item/importer.rb +++ b/app/models/simplefin_item/importer.rb @@ -1,5 +1,6 @@ require "set" class SimplefinItem::Importer + include SimplefinNumericHelpers class RateLimitedError < StandardError; end attr_reader :simplefin_item, :simplefin_provider, :sync @@ -15,20 +16,32 @@ class SimplefinItem::Importer Rails.logger.info "SimplefinItem::Importer - last_synced_at: #{simplefin_item.last_synced_at.inspect}" Rails.logger.info "SimplefinItem::Importer - sync_start_date: #{simplefin_item.sync_start_date.inspect}" + # Clear stale error and reconciliation stats from previous syncs at the start of a full import + # This ensures the UI doesn't show outdated warnings from old sync runs + if sync.respond_to?(:sync_stats) + sync.update_columns(sync_stats: { + "cleared_at" => Time.current.iso8601, + "import_started" => true + }) + end + begin # Defensive guard: If last_synced_at is set but there are linked accounts # with no transactions captured yet (typical after a balances-only run), # force the first full run to use chunked history to backfill. - linked_accounts = simplefin_item.simplefin_accounts.joins(:account) + # + # Check for linked accounts via BOTH legacy FK (accounts.simplefin_account_id) AND + # the new AccountProvider system. An account is "linked" if either association exists. + linked_accounts = simplefin_item.simplefin_accounts.select { |sfa| sfa.current_account.present? } no_txns_yet = linked_accounts.any? && linked_accounts.all? { |sfa| sfa.raw_transactions_payload.blank? } if simplefin_item.last_synced_at.nil? || no_txns_yet # First sync (or balances-only pre-run) — use chunked approach to get full history - Rails.logger.info "SimplefinItem::Importer - Using chunked history import" + Rails.logger.info "SimplefinItem::Importer - Using CHUNKED HISTORY import (last_synced_at=#{simplefin_item.last_synced_at.inspect}, no_txns_yet=#{no_txns_yet})" import_with_chunked_history else # Regular sync - use single request with buffer - Rails.logger.info "SimplefinItem::Importer - Using regular sync" + Rails.logger.info "SimplefinItem::Importer - Using REGULAR SYNC (last_synced_at=#{simplefin_item.last_synced_at&.strftime('%Y-%m-%d %H:%M')})" import_regular_sync end rescue RateLimitedError => e @@ -105,9 +118,91 @@ class SimplefinItem::Importer # Only update balance for already-linked accounts (if any), to avoid creating duplicates in setup. if (acct = sfa.current_account) adapter = Account::ProviderImportAdapter.new(acct) + + # Normalize balances for SimpleFIN liabilities so immediate UI is correct after discovery + bal = to_decimal(account_data[:balance]) + avail = to_decimal(account_data[:"available-balance"]) + observed = bal.nonzero? ? bal : avail + + is_linked_liability = [ "CreditCard", "Loan" ].include?(acct.accountable_type) + inferred = begin + Simplefin::AccountTypeMapper.infer( + name: account_data[:name], + holdings: account_data[:holdings], + extra: account_data[:extra], + balance: bal, + available_balance: avail, + institution: account_data.dig(:org, :name) + ) + rescue + nil + end + is_mapper_liability = inferred && [ "CreditCard", "Loan" ].include?(inferred.accountable_type) + is_liability = is_linked_liability || is_mapper_liability + + normalized = observed + if is_liability + # Try the overpayment analyzer first (feature-flagged) + begin + result = SimplefinAccount::Liabilities::OverpaymentAnalyzer + .new(sfa, observed_balance: observed) + .call + + case result.classification + when :credit + normalized = -observed.abs + when :debt + normalized = observed.abs + else + # Fallback to existing normalization when unknown/disabled + begin + obs = { + reason: result.reason, + tx_count: result.metrics[:tx_count], + charges_total: result.metrics[:charges_total], + payments_total: result.metrics[:payments_total], + observed: observed.to_s("F") + }.compact + Rails.logger.info("SimpleFIN overpayment heuristic (balances-only): unknown; falling back #{obs.inspect}") + rescue + # no-op + end + both_present = bal.nonzero? && avail.nonzero? + if both_present && same_sign?(bal, avail) + if bal.positive? && avail.positive? + normalized = -observed.abs + elsif bal.negative? && avail.negative? + normalized = observed.abs + end + else + normalized = -observed + end + end + rescue NameError + # Analyzer missing; use legacy path + both_present = bal.nonzero? && avail.nonzero? + if both_present && same_sign?(bal, avail) + if bal.positive? && avail.positive? + normalized = -observed.abs + elsif bal.negative? && avail.negative? + normalized = observed.abs + end + else + normalized = -observed + end + end + end + + cash = if acct.accountable_type == "Investment" + # Leave investment cash to investment calculators in full run + normalized + else + normalized + end + adapter.update_balance( - balance: account_data[:balance], - cash_balance: account_data[:"available-balance"], + balance: normalized, + cash_balance: cash, source: "simplefin" ) end @@ -146,6 +241,16 @@ class SimplefinItem::Importer return end + # Skip zero balance detection for liability accounts (CreditCard, Loan) where + # 0 balance with no holdings is normal (paid off card/loan) + account_type = simplefin_account.current_account&.accountable_type + return if %w[CreditCard Loan].include?(account_type) + + # Only count each account once per sync run to avoid false positives during + # chunked imports (which process the same account multiple times) + zero_balance_seen_keys << key if zeroish_balance && no_holdings + return if zero_balance_seen_keys.count(key) > 1 + if zeroish_balance && no_holdings stats["zero_runs"][key] = stats["zero_runs"][key].to_i + 1 # Cap to avoid unbounded growth @@ -161,6 +266,11 @@ class SimplefinItem::Importer end end + # Track accounts that have been flagged for zero balance in this sync run + def zero_balance_seen_keys + @zero_balance_seen_keys ||= [] + end + # Track seen error fingerprints during a single importer run to avoid double counting def seen_errors @seen_errors ||= Set.new @@ -319,6 +429,7 @@ class SimplefinItem::Importer # Step 2: Fetch transactions/holdings using the regular window. start_date = determine_sync_start_date + Rails.logger.info "SimplefinItem::Importer - import_regular_sync: last_synced_at=#{simplefin_item.last_synced_at&.strftime('%Y-%m-%d %H:%M')} => start_date=#{start_date&.strftime('%Y-%m-%d')}" accounts_data = fetch_accounts_data(start_date: start_date, pending: true) return if accounts_data.nil? # Error already handled @@ -360,6 +471,7 @@ class SimplefinItem::Importer # # Returns nothing; side-effects are snapshot + account upserts. def perform_account_discovery + Rails.logger.info "SimplefinItem::Importer - perform_account_discovery START (no date params - transactions may be empty)" discovery_data = fetch_accounts_data(start_date: nil) discovered_count = discovery_data&.dig(:accounts)&.size.to_i Rails.logger.info "SimpleFin discovery (no params) returned #{discovered_count} accounts" @@ -390,9 +502,48 @@ class SimplefinItem::Importer persist_stats! end end + + # Clean up orphaned SimplefinAccount records whose account_id no longer exists upstream. + # This handles the case where a user deletes and re-adds an institution in SimpleFIN, + # which generates new account IDs. Without this cleanup, both old (stale) and new + # SimplefinAccount records would appear in the setup UI as duplicates. + upstream_account_ids = discovery_data[:accounts].map { |a| a[:id].to_s }.compact + prune_orphaned_simplefin_accounts(upstream_account_ids) end end + # Removes SimplefinAccount records that no longer exist upstream and are not linked to any Account. + # This prevents duplicate accounts from appearing in the setup UI after a user re-adds an + # institution in SimpleFIN (which generates new account IDs). + def prune_orphaned_simplefin_accounts(upstream_account_ids) + return if upstream_account_ids.blank? + + # Find SimplefinAccount records with account_ids NOT in the upstream set + # Eager-load associations to prevent N+1 queries when checking linkage + orphaned = simplefin_item.simplefin_accounts + .includes(:account, :account_provider) + .where.not(account_id: upstream_account_ids) + .where.not(account_id: nil) + + orphaned.each do |sfa| + # Only delete if not linked to any Account (via legacy FK or AccountProvider) + # Note: sfa.account checks the legacy FK on Account.simplefin_account_id + # sfa.account_provider checks the new AccountProvider join table + linked_via_legacy = sfa.account.present? + linked_via_provider = sfa.account_provider.present? + + if !linked_via_legacy && !linked_via_provider + Rails.logger.info "SimpleFin: Pruning orphaned SimplefinAccount id=#{sfa.id} account_id=#{sfa.account_id} (no longer exists upstream)" + stats["accounts_pruned"] = stats.fetch("accounts_pruned", 0) + 1 + sfa.destroy + else + Rails.logger.info "SimpleFin: Keeping stale SimplefinAccount id=#{sfa.id} account_id=#{sfa.account_id} (still linked to Account)" + end + end + + persist_stats! if stats["accounts_pruned"].to_i > 0 + end + # Fetches accounts (and optionally transactions/holdings) from SimpleFin. # # Params: @@ -496,6 +647,32 @@ class SimplefinItem::Importer transactions = account_data[:transactions] holdings = account_data[:holdings] + # Log detailed info for accounts with holdings (investment accounts) to debug missing transactions + # Note: SimpleFIN doesn't include a 'type' field, so we detect investment accounts by presence of holdings or name + acct_name = account_data[:name].to_s.downcase + has_holdings = holdings.is_a?(Array) && holdings.any? + is_investment = has_holdings || acct_name.include?("ira") || acct_name.include?("401k") || acct_name.include?("retirement") || acct_name.include?("brokerage") + + # Always log for all accounts to trace the import flow + Rails.logger.info "SimplefinItem::Importer#import_account - account_id=#{account_id} name='#{account_data[:name]}' txn_count=#{transactions&.count || 0} holdings_count=#{holdings&.count || 0}" + + if is_investment + Rails.logger.info "SimpleFIN Investment Account Debug - account_id=#{account_id} name='#{account_data[:name]}'" + Rails.logger.info " - API response keys: #{account_data.keys.inspect}" + Rails.logger.info " - transactions count: #{transactions&.count || 0}" + Rails.logger.info " - holdings count: #{holdings&.count || 0}" + Rails.logger.info " - existing raw_transactions_payload count: #{simplefin_account.raw_transactions_payload.to_a.count}" + + # Log transaction data + if transactions.is_a?(Array) && transactions.any? + Rails.logger.info " - Transaction IDs: #{transactions.map { |t| t[:id] || t["id"] }.inspect}" + else + Rails.logger.warn " - NO TRANSACTIONS in API response for investment account!" + # Log what the transactions field actually contains + Rails.logger.info " - transactions raw value: #{account_data[:transactions].inspect}" + end + end + # Update all attributes; only update transactions if present to avoid wiping prior data attrs = { name: account_data[:name], @@ -514,6 +691,8 @@ class SimplefinItem::Importer if transactions.is_a?(Array) && transactions.any? existing_transactions = simplefin_account.raw_transactions_payload.to_a + Rails.logger.info "SimplefinItem::Importer#import_account - Merging transactions for account_id=#{account_id}: #{existing_transactions.count} existing + #{transactions.count} new" + # Build a map of key => best_tx best_by_key = {} @@ -569,7 +748,22 @@ class SimplefinItem::Importer end end - attrs[:raw_transactions_payload] = best_by_key.values + merged_transactions = best_by_key.values + attrs[:raw_transactions_payload] = merged_transactions + + Rails.logger.info "SimplefinItem::Importer#import_account - Merged result for account_id=#{account_id}: #{merged_transactions.count} total transactions" + + # NOTE: Reconciliation disabled - it analyzes the SimpleFin API response + # which only contains ~90 days of history, creating misleading "gap" warnings + # that don't reflect actual database state. Re-enable if we improve it to + # compare against database transactions instead of just the API response. + # begin + # reconcile_transactions(simplefin_account, merged_transactions) + # rescue => e + # Rails.logger.warn("SimpleFin: reconciliation failed for sfa=#{simplefin_account.id || account_id}: #{e.class} - #{e.message}") + # end + else + Rails.logger.info "SimplefinItem::Importer#import_account - No transactions in API response for account_id=#{account_id} (transactions=#{transactions.inspect.first(100)})" end # Track whether incoming holdings are new/changed so we can materialize and refresh balances @@ -604,6 +798,11 @@ class SimplefinItem::Importer begin simplefin_account.save! + # Log final state after save for debugging + if is_investment + Rails.logger.info "SimplefinItem::Importer#import_account - SAVED account_id=#{account_id}: raw_transactions_payload now has #{simplefin_account.reload.raw_transactions_payload.to_a.count} transactions" + end + # Post-save side effects acct = simplefin_account.current_account if acct @@ -773,14 +972,193 @@ class SimplefinItem::Importer end def initial_sync_lookback_period - # Default to 7 days for initial sync. Providers that support deeper - # history will supply it via chunked fetches, and users can optionally - # set a custom `sync_start_date` to go further back. - 7 + # Default to 60 days for initial sync to capture recent investment + # transactions (dividends, contributions, etc.). Providers that support + # deeper history will supply it via chunked fetches, and users can + # optionally set a custom `sync_start_date` to go further back. + 60 end def sync_buffer_period - # Default to 7 days buffer for subsequent syncs - 7 + # Default to 30 days buffer for subsequent syncs + # Investment accounts often have infrequent transactions (dividends, etc.) + # that would be missed with a shorter window + 30 + end + + # Transaction reconciliation: detect potential data gaps or missing transactions + # This helps identify when SimpleFin may not be returning complete data + def reconcile_transactions(simplefin_account, new_transactions) + return if new_transactions.blank? + + account_id = simplefin_account.account_id + existing_transactions = simplefin_account.raw_transactions_payload.to_a + reconciliation = { account_id: account_id, issues: [] } + + # 1. Check for unexpected transaction count drops + # If we previously had more transactions and now have fewer (after merge), + # something may have been removed upstream + if existing_transactions.any? + existing_count = existing_transactions.size + new_count = new_transactions.size + + # After merging, we should have at least as many as before + # A significant drop (>10%) could indicate data loss + if new_count < existing_count + drop_pct = ((existing_count - new_count).to_f / existing_count * 100).round(1) + if drop_pct > 10 + reconciliation[:issues] << { + type: "transaction_count_drop", + message: "Transaction count dropped from #{existing_count} to #{new_count} (#{drop_pct}% decrease)", + severity: drop_pct > 25 ? "high" : "medium" + } + end + end + end + + # 2. Detect gaps in transaction history + # Look for periods with no transactions that seem unusual + gaps = detect_transaction_gaps(new_transactions) + if gaps.any? + reconciliation[:issues] += gaps.map do |gap| + { + type: "transaction_gap", + message: "No transactions between #{gap[:start_date]} and #{gap[:end_date]} (#{gap[:days]} days)", + severity: gap[:days] > 30 ? "high" : "medium", + gap_start: gap[:start_date], + gap_end: gap[:end_date], + gap_days: gap[:days] + } + end + end + + # 3. Check for stale data (most recent transaction is old) + latest_tx_date = extract_latest_transaction_date(new_transactions) + if latest_tx_date.present? + days_since_latest = (Date.current - latest_tx_date).to_i + if days_since_latest > 7 + reconciliation[:issues] << { + type: "stale_transactions", + message: "Most recent transaction is #{days_since_latest} days old", + severity: days_since_latest > 14 ? "high" : "medium", + latest_date: latest_tx_date.to_s, + days_stale: days_since_latest + } + end + end + + # 4. Check for duplicate transaction IDs (data integrity issue) + duplicate_ids = find_duplicate_transaction_ids(new_transactions) + if duplicate_ids.any? + reconciliation[:issues] << { + type: "duplicate_ids", + message: "Found #{duplicate_ids.size} duplicate transaction ID(s)", + severity: "low", + duplicate_count: duplicate_ids.size + } + end + + # Record reconciliation results in stats + if reconciliation[:issues].any? + stats["reconciliation"] ||= {} + stats["reconciliation"][account_id] = reconciliation + + # Count issues by severity + high_severity = reconciliation[:issues].count { |i| i[:severity] == "high" } + medium_severity = reconciliation[:issues].count { |i| i[:severity] == "medium" } + + if high_severity > 0 + stats["reconciliation_warnings"] = stats.fetch("reconciliation_warnings", 0) + high_severity + Rails.logger.warn("SimpleFin reconciliation: #{high_severity} high-severity issue(s) for account #{account_id}") + + ActiveSupport::Notifications.instrument( + "simplefin.reconciliation_warning", + item_id: simplefin_item.id, + account_id: account_id, + issues: reconciliation[:issues] + ) + end + + if medium_severity > 0 + stats["reconciliation_notices"] = stats.fetch("reconciliation_notices", 0) + medium_severity + end + + persist_stats! + end + + reconciliation + end + + # Detect gaps in transaction history (periods with no activity) + def detect_transaction_gaps(transactions) + return [] if transactions.blank? || transactions.size < 2 + + # Extract and sort transaction dates + dates = transactions.map do |tx| + t = tx.with_indifferent_access + posted = t[:posted] + next nil if posted.blank? || posted.to_i <= 0 + Time.at(posted.to_i).to_date + end.compact.uniq.sort + + return [] if dates.size < 2 + + gaps = [] + min_gap_days = 14 # Only report gaps of 2+ weeks + + dates.each_cons(2) do |earlier, later| + gap_days = (later - earlier).to_i + if gap_days >= min_gap_days + gaps << { + start_date: earlier.to_s, + end_date: later.to_s, + days: gap_days + } + end + end + + # Limit to top 3 largest gaps to avoid noise + gaps.sort_by { |g| -g[:days] }.first(3) + end + + # Extract the most recent transaction date + def extract_latest_transaction_date(transactions) + return nil if transactions.blank? + + latest_timestamp = transactions.map do |tx| + t = tx.with_indifferent_access + posted = t[:posted] + posted.to_i if posted.present? && posted.to_i > 0 + end.compact.max + + latest_timestamp ? Time.at(latest_timestamp).to_date : nil + end + + # Find duplicate transaction IDs + def find_duplicate_transaction_ids(transactions) + return [] if transactions.blank? + + ids = transactions.map do |tx| + t = tx.with_indifferent_access + t[:id] || t[:fitid] + end.compact + + ids.group_by(&:itself).select { |_, v| v.size > 1 }.keys + end + + # --- Simple helpers for numeric handling in normalization --- + def to_decimal(value) + return BigDecimal("0") if value.nil? + case value + when BigDecimal then value + when String then BigDecimal(value) rescue BigDecimal("0") + when Numeric then BigDecimal(value.to_s) + else + BigDecimal("0") + end + end + + def same_sign?(a, b) + (a.positive? && b.positive?) || (a.negative? && b.negative?) end end diff --git a/app/models/simplefin_item/syncer.rb b/app/models/simplefin_item/syncer.rb index b00d40028..28bc6816a 100644 --- a/app/models/simplefin_item/syncer.rb +++ b/app/models/simplefin_item/syncer.rb @@ -10,7 +10,15 @@ class SimplefinItem::Syncer # can review and manually link accounts first. This mirrors the historical flow # users expect: initial 7-day balances snapshot, then full chunked history after linking. begin - if simplefin_item.simplefin_accounts.joins(:account).count == 0 + # Check for linked accounts via BOTH legacy FK (accounts.simplefin_account_id) AND + # the new AccountProvider system. An account is "linked" if either association exists. + linked_via_legacy = simplefin_item.simplefin_accounts.joins(:account).count + linked_via_provider = simplefin_item.simplefin_accounts.joins(:account_provider).count + total_linked = simplefin_item.simplefin_accounts.select { |sfa| sfa.current_account.present? }.count + + Rails.logger.info("SimplefinItem::Syncer - linked check: legacy=#{linked_via_legacy}, provider=#{linked_via_provider}, total=#{total_linked}") + + if total_linked == 0 sync.update!(status_text: "Discovering accounts (balances only)...") if sync.respond_to?(:status_text) # Pre-mark the sync as balances_only for runtime only (no persistence) begin @@ -52,8 +60,9 @@ class SimplefinItem::Syncer finalize_setup_counts(sync) # Process transactions/holdings only for linked accounts - linked_accounts = simplefin_item.simplefin_accounts.joins(:account) - if linked_accounts.any? + # Check both legacy FK and AccountProvider associations + linked_simplefin_accounts = simplefin_item.simplefin_accounts.select { |sfa| sfa.current_account.present? } + if linked_simplefin_accounts.any? sync.update!(status_text: "Processing transactions and holdings...") if sync.respond_to?(:status_text) simplefin_item.process_accounts @@ -77,7 +86,11 @@ class SimplefinItem::Syncer def finalize_setup_counts(sync) sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text) total_accounts = simplefin_item.simplefin_accounts.count - linked_accounts = simplefin_item.simplefin_accounts.joins(:account) + + # Count linked accounts using both legacy FK and AccountProvider associations + linked_count = simplefin_item.simplefin_accounts.count { |sfa| sfa.current_account.present? } + + # Unlinked = no legacy FK AND no AccountProvider unlinked_accounts = simplefin_item.simplefin_accounts .left_joins(:account, :account_provider) .where(accounts: { id: nil }, account_providers: { id: nil }) @@ -93,7 +106,7 @@ class SimplefinItem::Syncer existing = (sync.sync_stats || {}) setup_stats = { "total_accounts" => total_accounts, - "linked_accounts" => linked_accounts.count, + "linked_accounts" => linked_count, "unlinked_accounts" => unlinked_accounts.count } sync.update!(sync_stats: existing.merge(setup_stats)) @@ -185,7 +198,8 @@ class SimplefinItem::Syncer window_start = sync.created_at || 30.minutes.ago window_end = Time.current - account_ids = simplefin_item.simplefin_accounts.joins(:account).pluck("accounts.id") + # Get account IDs via BOTH legacy FK and AccountProvider to ensure we capture all linked accounts + account_ids = simplefin_item.simplefin_accounts.filter_map { |sfa| sfa.current_account&.id } return {} if account_ids.empty? tx_scope = Entry.where(account_id: account_ids, source: "simplefin", entryable_type: "Transaction") @@ -193,14 +207,16 @@ class SimplefinItem::Syncer tx_updated = tx_scope.where(updated_at: window_start..window_end).where.not(created_at: window_start..window_end).count tx_seen = tx_imported + tx_updated - holdings_scope = Holding.where(account_id: account_ids) - holdings_processed = holdings_scope.where(created_at: window_start..window_end).count + # Count holdings from raw_holdings_payload (what the sync found) rather than + # the database. Holdings are applied asynchronously via SimplefinHoldingsApplyJob, + # so database counts would always be 0 at this point. + holdings_found = simplefin_item.simplefin_accounts.sum { |sfa| Array(sfa.raw_holdings_payload).size } { "tx_imported" => tx_imported, "tx_updated" => tx_updated, "tx_seen" => tx_seen, - "holdings_processed" => holdings_processed, + "holdings_found" => holdings_found, "window_start" => window_start, "window_end" => window_end } diff --git a/app/models/trade.rb b/app/models/trade.rb index f7be46827..b9233f9db 100644 --- a/app/models/trade.rb +++ b/app/models/trade.rb @@ -4,10 +4,20 @@ class Trade < ApplicationRecord monetize :price belongs_to :security + belongs_to :category, optional: true validates :qty, presence: true validates :price, :currency, presence: true + # Trade types for categorization + def buy? + qty.positive? + end + + def sell? + qty.negative? + end + class << self def build_name(type, qty, ticker) prefix = type == "buy" ? "Buy" : "Sell" diff --git a/app/models/trade/create_form.rb b/app/models/trade/create_form.rb index b07cc87d8..a6973df72 100644 --- a/app/models/trade/create_form.rb +++ b/app/models/trade/create_form.rb @@ -41,7 +41,8 @@ class Trade::CreateForm qty: signed_qty, price: price, currency: currency, - security: security + security: security, + category: investment_category_for(type) ) ) @@ -53,6 +54,14 @@ class Trade::CreateForm trade_entry end + def investment_category_for(trade_type) + # Buy trades are categorized as "Savings & Investments" (expense) + # Sell trades are left uncategorized for now + return nil unless trade_type == "buy" + + account.family.categories.find_by(name: "Savings & Investments") + end + def create_interest_income signed_amount = amount.to_d * -1 diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index 67fac05dc..40387dfbf 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -21,6 +21,7 @@ class TradeImport < Import qty: row.qty, currency: row.currency.presence || mapped_account.currency, price: row.price, + category: investment_category_for(row.qty, mapped_account.family), entry: Entry.new( account: mapped_account, date: row.date_iso, @@ -53,7 +54,7 @@ class TradeImport < Import end def dry_run - mappings = { transactions: rows.count } + mappings = { transactions: rows_count } mappings.merge( accounts: Import::AccountMapping.for_import(self).creational.count @@ -76,6 +77,14 @@ class TradeImport < Import end private + def investment_category_for(qty, family) + # Buy trades (positive qty) are categorized as "Savings & Investments" + # Sell trades are left uncategorized - users will be prompted to categorize + return nil unless qty.to_d.positive? + + family.categories.find_by(name: "Savings & Investments") + end + def find_or_create_security(ticker: nil, exchange_operating_mic: nil) return nil unless ticker.present? diff --git a/app/models/transaction.rb b/app/models/transaction.rb index 368fa6453..dd8eb3064 100644 --- a/app/models/transaction.rb +++ b/app/models/transaction.rb @@ -9,6 +9,8 @@ class Transaction < ApplicationRecord accepts_nested_attributes_for :taggings, allow_destroy: true + after_save :clear_merchant_unlinked_association, if: :merchant_id_previously_changed? + enum :kind, { standard: "standard", # A regular transaction, included in budget analytics funds_movement: "funds_movement", # Movement of funds between accounts, excluded from budget analytics @@ -39,4 +41,14 @@ class Transaction < ApplicationRecord rescue false end + + private + def clear_merchant_unlinked_association + return unless merchant_id.present? && merchant.is_a?(ProviderMerchant) + + family = entry&.account&.family + return unless family + + FamilyMerchantAssociation.where(family: family, merchant: merchant).delete_all + end end diff --git a/app/models/transaction/transferable.rb b/app/models/transaction/transferable.rb index cea7978ba..098087610 100644 --- a/app/models/transaction/transferable.rb +++ b/app/models/transaction/transferable.rb @@ -14,11 +14,11 @@ module Transaction::Transferable transfer_as_inflow || transfer_as_outflow end - def transfer_match_candidates + def transfer_match_candidates(date_window: 30) candidates_scope = if self.entry.amount.negative? - family_matches_scope.where("inflow_candidates.entryable_id = ?", self.id) + family_matches_scope(date_window: date_window).where("inflow_candidates.entryable_id = ?", self.id) else - family_matches_scope.where("outflow_candidates.entryable_id = ?", self.id) + family_matches_scope(date_window: date_window).where("outflow_candidates.entryable_id = ?", self.id) end candidates_scope.map do |match| @@ -30,7 +30,7 @@ module Transaction::Transferable end private - def family_matches_scope - self.entry.account.family.transfer_match_candidates + def family_matches_scope(date_window:) + self.entry.account.family.transfer_match_candidates(date_window: date_window) end end diff --git a/app/models/transfer.rb b/app/models/transfer.rb index f11867665..93de0a068 100644 --- a/app/models/transfer.rb +++ b/app/models/transfer.rb @@ -133,6 +133,7 @@ class Transfer < ApplicationRecord return unless inflow_transaction&.entry && outflow_transaction&.entry date_diff = (inflow_transaction.entry.date - outflow_transaction.entry.date).abs - errors.add(:base, "Must be within 4 days") if date_diff > 4 + max_days = status == "confirmed" ? 30 : 4 + errors.add(:base, "Must be within #{max_days} days") if date_diff > max_days end end diff --git a/app/models/user.rb b/app/models/user.rb index 8f1b6e2b7..569cafd2b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,5 +1,7 @@ class User < ApplicationRecord - has_secure_password + # Allow nil password for SSO-only users (JIT provisioning). + # Custom validation ensures password is present for non-SSO registration. + has_secure_password validations: false belongs_to :family belongs_to :last_viewed_chat, class_name: "Chat", optional: true @@ -17,6 +19,11 @@ class User < ApplicationRecord validate :ensure_valid_profile_image validates :default_period, inclusion: { in: Period::PERIODS.keys } validates :default_account_order, inclusion: { in: AccountOrder::ORDERS.keys } + + # Password is required on create unless the user is being created via SSO JIT. + # SSO JIT users have password_digest = nil and authenticate via OIDC only. + validates :password, presence: true, on: :create, unless: :skip_password_validation? + validates :password, length: { minimum: 8 }, allow_nil: true normalizes :email, with: ->(email) { email.strip.downcase } normalizes :unconfirmed_email, with: ->(email) { email&.strip&.downcase } @@ -104,6 +111,20 @@ class User < ApplicationRecord ai_enabled && ai_available? end + # SSO-only users have OIDC identities but no local password. + # They cannot use password reset or local login. + def sso_only? + password_digest.nil? && oidc_identities.exists? + end + + # Check if user has a local password set (can authenticate locally) + def has_local_password? + password_digest.present? + end + + # Attribute to skip password validation during SSO JIT provisioning + attr_accessor :skip_password_validation + # Deactivation validate :can_deactivate, if: -> { active_changed? && !active } after_update_commit :purge_later, if: -> { saved_change_to_active?(from: true, to: false) } @@ -258,6 +279,10 @@ class User < ApplicationRecord end private + def skip_password_validation? + skip_password_validation == true + end + def default_dashboard_section_order %w[cashflow_sankey outflows_donut net_worth_chart balance_sheet] end diff --git a/app/views/accounts/_logo.html.erb b/app/views/accounts/_logo.html.erb index 52be912f8..2d609cf29 100644 --- a/app/views/accounts/_logo.html.erb +++ b/app/views/accounts/_logo.html.erb @@ -9,6 +9,8 @@ <% if account.institution_domain.present? && Setting.brand_fetch_client_id.present? %> <%= image_tag "https://cdn.brandfetch.io/#{account.institution_domain}/icon/fallback/lettermark/w/40/h/40?c=#{Setting.brand_fetch_client_id}", class: "shrink-0 rounded-full #{size_classes[size]}" %> +<% elsif account.logo_url.present? %> + <%= image_tag account.logo_url, class: "shrink-0 rounded-full #{size_classes[size]}", loading: "lazy" %> <% elsif account.logo.attached? %> <%= image_tag account.logo, class: "shrink-0 rounded-full #{size_classes[size]}" %> <% else %> diff --git a/app/views/accounts/index.html.erb b/app/views/accounts/index.html.erb index f9447c314..b9686c517 100644 --- a/app/views/accounts/index.html.erb +++ b/app/views/accounts/index.html.erb @@ -21,7 +21,7 @@ -<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? %> +<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? %> <%= render "empty" %> <% else %>
@@ -41,6 +41,10 @@ <%= render @enable_banking_items.sort_by(&:created_at) %> <% end %> + <% if @coinstats_items.any? %> + <%= render @coinstats_items.sort_by(&:created_at) %> + <% end %> + <% if @manual_accounts.any? %>
<%= render "accounts/index/manual_accounts", accounts: @manual_accounts %> diff --git a/app/views/api/v1/imports/index.json.jbuilder b/app/views/api/v1/imports/index.json.jbuilder new file mode 100644 index 000000000..eff1c6414 --- /dev/null +++ b/app/views/api/v1/imports/index.json.jbuilder @@ -0,0 +1,21 @@ +json.data do + json.array! @imports do |import| + json.id import.id + json.type import.type + json.status import.status + json.created_at import.created_at + json.updated_at import.updated_at + json.account_id import.account_id + json.rows_count import.rows_count + json.error import.error if import.error.present? + end +end + +json.meta do + json.current_page @pagy.page + json.next_page @pagy.next + json.prev_page @pagy.prev + json.total_pages @pagy.pages + json.total_count @pagy.count + json.per_page @per_page +end diff --git a/app/views/api/v1/imports/show.json.jbuilder b/app/views/api/v1/imports/show.json.jbuilder new file mode 100644 index 000000000..18509062e --- /dev/null +++ b/app/views/api/v1/imports/show.json.jbuilder @@ -0,0 +1,30 @@ +json.data do + json.id @import.id + json.type @import.type + json.status @import.status + json.created_at @import.created_at + json.updated_at @import.updated_at + json.account_id @import.account_id + json.error @import.error if @import.error.present? + + json.configuration do + json.date_col_label @import.date_col_label + json.amount_col_label @import.amount_col_label + json.name_col_label @import.name_col_label + json.category_col_label @import.category_col_label + json.tags_col_label @import.tags_col_label + json.notes_col_label @import.notes_col_label + json.account_col_label @import.account_col_label + json.date_format @import.date_format + json.number_format @import.number_format + json.signage_convention @import.signage_convention + end + + json.stats do + json.rows_count @import.rows_count + json.valid_rows_count @import.rows.select(&:valid?).count if @import.rows.loaded? + end + + # Only show a subset of rows for preview if needed, or link to a separate rows endpoint + # json.sample_rows @import.rows.limit(5) +end diff --git a/app/views/coinstats_items/_coinstats_item.html.erb b/app/views/coinstats_items/_coinstats_item.html.erb new file mode 100644 index 000000000..55736702e --- /dev/null +++ b/app/views/coinstats_items/_coinstats_item.html.erb @@ -0,0 +1,105 @@ +<%# locals: (coinstats_item:) %> + +<%= tag.div id: dom_id(coinstats_item) do %> +
+ +
+ <%= icon "chevron-right", class: "group-open:transform group-open:rotate-90" %> + +
+
+ <%= tag.p coinstats_item.institution_display_name.first.upcase, class: "text-success text-xs font-medium" %> +
+
+ +
+
+ <%= tag.p coinstats_item.institution_display_name, class: "font-medium text-primary" %> + <% if coinstats_item.scheduled_for_deletion? %> +

<%= t(".deletion_in_progress") %>

+ <% end %> +
+

<%= t(".provider_name") %>

+ <% if coinstats_item.syncing? %> +
+ <%= icon "loader", size: "sm", class: "animate-spin" %> + <%= tag.span t(".syncing") %> +
+ <% elsif coinstats_item.requires_update? %> +
+ <%= icon "alert-triangle", size: "sm", color: "warning" %> + <%= tag.span t(".reconnect") %> +
+ <% else %> +

+ <% if coinstats_item.last_synced_at %> + <% if coinstats_item.sync_status_summary %> + <%= t(".status_with_summary", timestamp: time_ago_in_words(coinstats_item.last_synced_at), summary: coinstats_item.sync_status_summary) %> + <% else %> + <%= t(".status", timestamp: time_ago_in_words(coinstats_item.last_synced_at)) %> + <% end %> + <% else %> + <%= t(".status_never") %> + <% end %> +

+ <% end %> +
+
+ +
+ <% if coinstats_item.requires_update? %> + <%= render DS::Link.new( + text: t(".update_api_key"), + icon: "refresh-cw", + variant: "secondary", + href: settings_providers_path, + frame: "_top" + ) %> + <% elsif Rails.env.development? %> + <%= icon( + "refresh-cw", + as_button: true, + href: sync_coinstats_item_path(coinstats_item) + ) %> + <% end %> + + <%= render DS::Menu.new do |menu| %> + <% menu.with_item( + variant: "button", + text: t(".delete"), + icon: "trash-2", + href: coinstats_item_path(coinstats_item), + method: :delete, + confirm: CustomConfirm.for_resource_deletion(coinstats_item.institution_display_name, high_severity: true) + ) %> + <% end %> +
+
+ + <% unless coinstats_item.scheduled_for_deletion? %> +
+ <% if coinstats_item.accounts.any? %> + <%= render "accounts/index/account_groups", accounts: coinstats_item.accounts %> + <% end %> + + <%# Sync summary (collapsible) - using shared ProviderSyncSummary component %> + <% stats = if defined?(@coinstats_sync_stats_map) && @coinstats_sync_stats_map + @coinstats_sync_stats_map[coinstats_item.id] || {} + else + coinstats_item.syncs.ordered.first&.sync_stats || {} + end %> + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: coinstats_item + ) %> + + <% if coinstats_item.accounts.empty? %> +
+

<%= t(".no_wallets_title") %>

+

<%= t(".no_wallets_message") %>

+
+ <% end %> +
+ <% end %> +
+<% end %> diff --git a/app/views/coinstats_items/new.html.erb b/app/views/coinstats_items/new.html.erb new file mode 100644 index 000000000..c49890232 --- /dev/null +++ b/app/views/coinstats_items/new.html.erb @@ -0,0 +1,77 @@ +<%= turbo_frame_tag "modal" do %> + <%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title")) %> + + <% dialog.with_body do %> + <% error_msg = local_assigns[:error_message] || @error_message %> + <% if error_msg.present? %> +
+

<%= error_msg %>

+
+ <% end %> + + <% items = local_assigns[:coinstats_items] || @coinstats_items || Current.family.coinstats_items.where.not(api_key: nil) %> + <% if items&.any? %> + <% selected_item = items.first %> + <% blockchains = local_assigns[:blockchains] || @blockchains || [] %> + <% address_value = local_assigns[:address] || @address %> + <% blockchain_value = local_assigns[:blockchain] || @blockchain %> + <%= styled_form_with url: link_wallet_coinstats_items_path, + method: :post, + data: { turbo: true }, + class: "space-y-3" do |form| %> + <%= form.hidden_field :coinstats_item_id, value: selected_item.id %> + + <%= form.text_field :address, + label: t(".address_label"), + placeholder: t(".address_placeholder"), + value: address_value %> + + <% if blockchains.present? %> + <%= form.select :blockchain, + options_for_select(blockchains, blockchain_value), + { include_blank: t(".blockchain_select_blank") }, + label: t(".blockchain_label"), + class: "w-full rounded-md border border-primary px-3 py-2 text-sm bg-container-inset text-primary" %> + <% else %> + <%= form.text_field :blockchain, + label: t(".blockchain_label"), + placeholder: t(".blockchain_placeholder"), + value: blockchain_value %> + <% end %> + +
+ <%= form.submit t(".link"), + class: "inline-flex items-center justify-center rounded-lg px-4 py-2 text-sm font-medium text-inverse bg-inverse hover:bg-inverse-hover focus:outline-none focus:ring-2 focus:ring-gray-900 focus:ring-offset-2 transition-colors" %> +
+ <% end %> + <% else %> +
+
+ <%= icon("alert-circle", class: "text-warning w-5 h-5 shrink-0 mt-0.5") %> +
+

<%= t(".not_configured_title") %>

+

<%= t(".not_configured_message") %>

+
+
+ +
+
    +
  1. <%= t(".not_configured_step1_html").html_safe %>
  2. +
  3. <%= t(".not_configured_step2_html").html_safe %>
  4. +
  5. <%= t(".not_configured_step3_html").html_safe %>
  6. +
+
+ +
+ <%= link_to settings_providers_path, + class: "w-full inline-flex items-center justify-center rounded-lg font-medium whitespace-nowrap rounded-lg hidden md:inline-flex px-3 py-2 text-sm text-inverse bg-inverse hover:bg-inverse-hover disabled:bg-gray-500 theme-dark:disabled:bg-gray-400", + data: { turbo: false } do %> + <%= t(".go_to_settings") %> + <% end %> +
+
+ <% end %> + <% end %> + <% end %> +<% end %> diff --git a/app/views/enable_banking_items/_enable_banking_item.html.erb b/app/views/enable_banking_items/_enable_banking_item.html.erb index 4c13e8860..f82ea7b27 100644 --- a/app/views/enable_banking_items/_enable_banking_item.html.erb +++ b/app/views/enable_banking_items/_enable_banking_item.html.erb @@ -85,6 +85,17 @@ <%= render "accounts/index/account_groups", accounts: enable_banking_item.accounts %> <% end %> + <%# Sync summary (collapsible) - using shared ProviderSyncSummary component %> + <% stats = if defined?(@enable_banking_sync_stats_map) && @enable_banking_sync_stats_map + @enable_banking_sync_stats_map[enable_banking_item.id] || {} + else + enable_banking_item.syncs.ordered.first&.sync_stats || {} + end %> + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: enable_banking_item + ) %> + <% if enable_banking_item.unlinked_accounts_count > 0 %>

Setup needed

diff --git a/app/views/enable_banking_items/new.html.erb b/app/views/enable_banking_items/new.html.erb index 26bc308f6..f7e99cf3d 100644 --- a/app/views/enable_banking_items/new.html.erb +++ b/app/views/enable_banking_items/new.html.erb @@ -96,7 +96,7 @@

Setup Steps:

    -
  1. Go to Settings → Bank Sync Providers
  2. +
  3. Go to Settings → Providers
  4. Find the Enable Banking section
  5. Enter your Enable Banking credentials
  6. Return here to link your accounts
  7. diff --git a/app/views/family_exports/new.html.erb b/app/views/family_exports/new.html.erb index a23477368..7c8f3ac98 100644 --- a/app/views/family_exports/new.html.erb +++ b/app/views/family_exports/new.html.erb @@ -33,8 +33,8 @@ <%= form_with url: family_exports_path, method: :post, class: "space-y-4" do |form| %>
    - <%= link_to "Cancel", "#", class: "flex-1 text-center px-4 py-2 border border-primary rounded-lg hover:bg-surface-hover", data: { action: "click->modal#close" } %> - <%= form.submit "Export data", class: "flex-1 bg-inverse fg-inverse rounded-lg px-4 py-2 cursor-pointer" %> + <%= link_to "Cancel", "#", class: "flex-1 text-center px-4 py-2 text-primary bg-surface border border-primary rounded-lg hover:bg-surface-hover", data: { action: "click->modal#close" } %> + <%= form.submit "Export data", class: "flex-1 bg-inverse fg-inverse rounded-lg px-4 py-2 cursor-pointer hover:bg-inverse-hover" %>
    <% end %>
diff --git a/app/views/family_merchants/_form.html.erb b/app/views/family_merchants/_form.html.erb index f0680ab54..49363d7ea 100644 --- a/app/views/family_merchants/_form.html.erb +++ b/app/views/family_merchants/_form.html.erb @@ -1,7 +1,7 @@ <%# locals: (family_merchant:) %>
- <%= styled_form_with model: family_merchant, class: "space-y-4" do |f| %> + <%= styled_form_with model: family_merchant, url: family_merchant.persisted? ? family_merchant_path(family_merchant) : family_merchants_path, class: "space-y-4" do |f| %>
<% if family_merchant.errors.any? %> <%= render "shared/form_errors", model: family_merchant %> diff --git a/app/views/family_merchants/_provider_merchant.html.erb b/app/views/family_merchants/_provider_merchant.html.erb index 1d222d5ea..c2cb0e434 100644 --- a/app/views/family_merchants/_provider_merchant.html.erb +++ b/app/views/family_merchants/_provider_merchant.html.erb @@ -21,4 +21,21 @@ <%= provider_merchant.source&.titleize %> + + <%= render DS::Menu.new do |menu| %> + <% menu.with_item(variant: "link", text: t(".edit"), href: edit_family_merchant_path(provider_merchant), icon: "pencil", data: { turbo_frame: "modal" }) %> + <% menu.with_item( + variant: "button", + text: t(".remove"), + href: family_merchant_path(provider_merchant), + icon: "trash-2", + method: :delete, + confirm: CustomConfirm.new( + destructive: true, + title: t(".remove_confirm_title"), + body: t(".remove_confirm_body", name: provider_merchant.name), + btn_text: t(".remove") + )) %> + <% end %> + diff --git a/app/views/family_merchants/index.html.erb b/app/views/family_merchants/index.html.erb index df047726f..0550739ef 100644 --- a/app/views/family_merchants/index.html.erb +++ b/app/views/family_merchants/index.html.erb @@ -1,12 +1,20 @@
-

Merchants

+

<%= t(".title") %>

- <%= render DS::Link.new( - text: "New merchant", - variant: "primary", - href: new_family_merchant_path, - frame: :modal - ) %> +
+ <%= render DS::Link.new( + text: t(".merge"), + variant: "outline", + href: merge_family_merchants_path, + frame: :modal + ) %> + <%= render DS::Link.new( + text: t(".new"), + variant: "primary", + href: new_family_merchant_path, + frame: :modal + ) %> +
@@ -59,7 +67,7 @@
<%= icon "info", class: "w-5 h-5 text-link mt-0.5 flex-shrink-0" %> -

<%= t(".provider_read_only") %>

+

<%= t(".provider_info") %>

@@ -71,6 +79,7 @@ <%= t(".table.merchant") %> <%= t(".table.source") %> + <%= t(".table.actions") %> @@ -85,4 +94,38 @@
<% end %>
+ + <% if @unlinked_merchants.any? %> +
+
+

<%= t(".unlinked_title") %>

+ · +

<%= @unlinked_merchants.count %>

+
+ +
+
+ <%= icon "info", class: "w-5 h-5 text-subdued mt-0.5 flex-shrink-0" %> +

<%= t(".unlinked_info") %>

+
+
+ +
+
+ + + + + + + + + + <%= render partial: "family_merchants/provider_merchant", collection: @unlinked_merchants %> + +
<%= t(".table.merchant") %><%= t(".table.source") %><%= t(".table.actions") %>
+
+
+
+ <% end %>
diff --git a/app/views/family_merchants/merge.html.erb b/app/views/family_merchants/merge.html.erb new file mode 100644 index 000000000..77d6ba84f --- /dev/null +++ b/app/views/family_merchants/merge.html.erb @@ -0,0 +1,33 @@ +<%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title"), subtitle: t(".description")) %> + <% dialog.with_body do %> + <%= styled_form_with url: perform_merge_family_merchants_path, method: :post, class: "space-y-4" do |f| %> + <%= f.collection_select :target_id, + @merchants, + :id, :name, + { prompt: t(".select_target"), label: t(".target_label") }, + { required: true } %> + +
+

<%= t(".sources_label") %>

+
+ <% @merchants.each do |merchant| %> + + <% end %> +
+

<%= t(".sources_hint") %>

+
+ + <%= render DS::Button.new( + text: t(".submit"), + full_width: true + ) %> + <% end %> + <% end %> +<% end %> diff --git a/app/views/holdings/_holding.html.erb b/app/views/holdings/_holding.html.erb index 7daa151ff..4865af8ee 100644 --- a/app/views/holdings/_holding.html.erb +++ b/app/views/holdings/_holding.html.erb @@ -3,8 +3,8 @@ <%= turbo_frame_tag dom_id(holding) do %>
- <% if Setting.brand_fetch_client_id.present? %> - <%= image_tag "https://cdn.brandfetch.io/#{holding.ticker}/icon/fallback/lettermark/w/40/h/40?c=#{Setting.brand_fetch_client_id}", class: "w-9 h-9 rounded-full", loading: "lazy" %> + <% if holding.security.brandfetch_icon_url.present? %> + <%= image_tag holding.security.brandfetch_icon_url, class: "w-9 h-9 rounded-full", loading: "lazy" %> <% elsif holding.security.logo_url.present? %> <%= image_tag holding.security.logo_url, class: "w-9 h-9 rounded-full", loading: "lazy" %> <% else %> diff --git a/app/views/holdings/show.html.erb b/app/views/holdings/show.html.erb index 9e84a43f0..32918b49e 100644 --- a/app/views/holdings/show.html.erb +++ b/app/views/holdings/show.html.erb @@ -6,8 +6,8 @@ <%= tag.p @holding.ticker, class: "text-sm text-secondary" %>
- <% if Setting.brand_fetch_client_id.present? %> - <%= image_tag "https://cdn.brandfetch.io/#{@holding.ticker}/icon/fallback/lettermark/w/40/h/40?c=#{Setting.brand_fetch_client_id}", loading: "lazy", class: "w-9 h-9 rounded-full" %> + <% if @holding.security.brandfetch_icon_url.present? %> + <%= image_tag @holding.security.brandfetch_icon_url, loading: "lazy", class: "w-9 h-9 rounded-full" %> <% elsif @holding.security.logo_url.present? %> <%= image_tag @holding.security.logo_url, loading: "lazy", class: "w-9 h-9 rounded-full" %> <% else %> diff --git a/app/views/import/uploads/show.html.erb b/app/views/import/uploads/show.html.erb index 1115382ec..7227ff352 100644 --- a/app/views/import/uploads/show.html.erb +++ b/app/views/import/uploads/show.html.erb @@ -4,7 +4,10 @@ <%= content_for :previous_path, imports_path %> -
+
+ + <%= render "imports/drag_drop_overlay", title: "Drop CSV to upload", subtitle: "Your file will be uploaded automatically" %> +

<%= t(".title") %>

@@ -18,7 +21,7 @@ <% end %> <% tabs.with_panel(tab_id: "csv-upload") do %> - <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2" do |form| %> + <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2", data: { drag_and_drop_import_target: "form" } do |form| %> <%= form.select :col_sep, Import::SEPARATORS, label: true %> <% if @import.type == "TransactionImport" || @import.type == "TradeImport" %> @@ -41,7 +44,7 @@

- <%= form.file_field :csv_file, class: "hidden", "data-auto-submit-form-target": "auto", "data-file-upload-target": "input" %> + <%= form.file_field :csv_file, class: "hidden", "data-auto-submit-form-target": "auto", "data-file-upload-target": "input", "data-drag-and-drop-import-target": "input" %>
diff --git a/app/views/imports/_drag_drop_overlay.html.erb b/app/views/imports/_drag_drop_overlay.html.erb new file mode 100644 index 000000000..504920a86 --- /dev/null +++ b/app/views/imports/_drag_drop_overlay.html.erb @@ -0,0 +1,7 @@ + \ No newline at end of file diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 5741ff241..3f19d50ed 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -19,7 +19,8 @@ data-app-layout-user-id-value="<%= Current.user.id %>"> <%# MOBILE - Top nav %> -