diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0859eca33..02a5a640e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,9 @@ jobs: - name: Scan for security vulnerabilities in Ruby dependencies run: bin/brakeman --no-pager + - name: Validate preview deploy workflow hardening + run: ruby bin/preview_deploy_security_check.rb + scan_js: runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/workflows/label-not-gittensor.yml b/.github/workflows/label-not-gittensor.yml deleted file mode 100644 index 7b846e8f7..000000000 --- a/.github/workflows/label-not-gittensor.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Label non-Gittensor PRs - -on: - pull_request_target: - types: - - opened - - reopened - -permissions: - pull-requests: write - -jobs: - label-pr: - runs-on: ubuntu-latest - steps: - - name: Add not-gittensor label for matched authors - uses: actions/github-script@v7 - env: - GITTENSOR_USERS: ${{ vars.GITTENSOR_USERS || '[]' }} - GITTENSOR_EXCEPTIONS: ${{ vars.GITTENSOR_EXCEPTIONS || '[]' }} - TARGET_LABEL: not-gittensor - with: - script: | - const parseList = (raw, name) => { - try { - const parsed = JSON.parse(raw || '[]'); - if (!Array.isArray(parsed)) { - core.setFailed(`${name} must be a JSON array.`); - return []; - } - return parsed.map((value) => String(value).toLowerCase()); - } catch (error) { - core.setFailed(`Failed to parse ${name}: ${error.message}`); - return []; - } - }; - - const author = context.payload.pull_request.user.login.toLowerCase(); - const users = new Set(parseList(process.env.GITTENSOR_USERS, 'GITTENSOR_USERS')); - const exceptions = new Set(parseList(process.env.GITTENSOR_EXCEPTIONS, 'GITTENSOR_EXCEPTIONS')); - - if (users.has(author) || exceptions.has(author)) { - core.info(`No label needed for @${author}.`); - return; - } - - const existingLabels = context.payload.pull_request.labels.map((label) => label.name); - if (existingLabels.includes(process.env.TARGET_LABEL)) { - core.info(`Label ${process.env.TARGET_LABEL} already present.`); - return; - } - - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - labels: [process.env.TARGET_LABEL], - }); - - core.info(`Added ${process.env.TARGET_LABEL} to PR #${context.payload.pull_request.number}.`); diff --git a/.github/workflows/preview-deploy.yml b/.github/workflows/preview-deploy.yml index 3074503de..aba03b987 100644 --- a/.github/workflows/preview-deploy.yml +++ b/.github/workflows/preview-deploy.yml @@ -16,6 +16,11 @@ jobs: name: Deploy to Cloudflare Containers runs-on: ubuntu-latest timeout-minutes: 15 + concurrency: + group: preview-deploy-${{ github.event.pull_request.number }} + cancel-in-progress: true + environment: + name: preview permissions: actions: read contents: read @@ -32,10 +37,10 @@ jobs: steps: - name: Wait for PR CI to pass - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 with: script: | - const headSha = context.payload.pull_request.head.sha; + const headSha = process.env.HEAD_SHA; const timeoutMs = 10 * 60 * 1000; const pollMs = 15 * 1000; const startedAt = Date.now(); @@ -73,68 +78,62 @@ jobs: core.setFailed(`Timed out waiting for Pull Request workflow for ${headSha}. Last state: ${lastState}`); - - name: Checkout code - uses: actions/checkout@v5 + - name: Checkout PR code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + path: pr + persist-credentials: false + + - name: Checkout trusted preview tooling + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: trusted + persist-credentials: false + sparse-checkout: | + workers/preview - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" - - name: Install Wrangler dependencies - working-directory: workers/preview - run: npm install - - - name: Configure preview files for this PR - working-directory: workers/preview - run: | - sed -i "s/\${PR_NUMBER}/${PR_NUMBER}/g" wrangler.toml - sed -i "s/\${PR_NUMBER}/${PR_NUMBER}/g" src/index.ts - cat wrangler.toml - - - name: Delete existing preview container app before redeploy - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - working-directory: workers/preview + - name: Prepare trusted preview deploy workspace run: | set -euo pipefail - CONTAINER_NAME="sure-preview-${PR_NUMBER}-railscontainer" - echo "Looking for stale preview container app: $CONTAINER_NAME" - CONTAINER_ID=$(npx wrangler containers list --json | jq -r --arg NAME "$CONTAINER_NAME" ' - map(select((.name // .application_name // .app_name // "") == $NAME)) - | first - | (.id // .container_id // .application_id // empty) - ') + preview_dir="$RUNNER_TEMP/sure-preview-worker" + rm -rf "$preview_dir" + mkdir -p "$preview_dir" - if [ -n "$CONTAINER_ID" ]; then - echo "Deleting stale preview container app $CONTAINER_NAME ($CONTAINER_ID)" - npx wrangler containers delete "$CONTAINER_ID" - else - echo "No stale preview container app found; continuing" - fi + cp trusted/workers/preview/package.json "$preview_dir/package.json" + cp trusted/workers/preview/package-lock.json "$preview_dir/package-lock.json" + cp trusted/workers/preview/tsconfig.json "$preview_dir/tsconfig.json" + cp trusted/workers/preview/wrangler.toml "$preview_dir/wrangler.toml" + cp -R pr/workers/preview/src "$preview_dir/src" - - name: Delete existing preview Worker before redeploy - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - working-directory: workers/preview - run: | - WORKER_NAME="sure-preview-${PR_NUMBER}" - echo "Ensuring fresh preview deployment for $WORKER_NAME" - npx wrangler delete --name "$WORKER_NAME" --force || echo "Existing preview not found; continuing" + sed -i "s/\${PR_NUMBER}/${PR_NUMBER}/g" "$preview_dir/wrangler.toml" + sed -i "s/\${PR_NUMBER}/${PR_NUMBER}/g" "$preview_dir/src/index.ts" + sed -i \ + "s#image = \"../../Dockerfile.preview\"#image = \"${GITHUB_WORKSPACE}/pr/Dockerfile.preview\"#" \ + "$preview_dir/wrangler.toml" + + cat "$preview_dir/wrangler.toml" + cd "$preview_dir" + npm ci --ignore-scripts --no-audit --no-fund - name: Create GitHub Deployment id: deployment - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 with: script: | + const prNumber = process.env.PR_NUMBER; + const headSha = process.env.HEAD_SHA; const deployment = await github.rest.repos.createDeployment({ owner: context.repo.owner, repo: context.repo.repo, - ref: context.payload.pull_request.head.sha, - environment: `preview-pr-${process.env.PR_NUMBER}`, + ref: headSha, + environment: `preview-pr-${prNumber}`, auto_merge: false, required_contexts: [], description: 'PR Preview Deployment' @@ -144,13 +143,15 @@ jobs: - name: Deploy to Cloudflare Containers id: deploy - working-directory: workers/preview env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} CLOUDFLARE_WORKERS_SUBDOMAIN: ${{ secrets.CLOUDFLARE_WORKERS_SUBDOMAIN }} run: | - npx wrangler deploy --var "PR_NUMBER:${PR_NUMBER}" + set -euo pipefail + + cd "$RUNNER_TEMP/sure-preview-worker" + ./node_modules/.bin/wrangler deploy --config wrangler.toml --var "PR_NUMBER:${PR_NUMBER}" # Get the deployment URL PREVIEW_URL="https://sure-preview-${PR_NUMBER}.${CLOUDFLARE_WORKERS_SUBDOMAIN}.workers.dev" @@ -165,22 +166,26 @@ jobs: - name: Update Deployment Status if: always() && steps.deployment.outputs.result - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + DEPLOYMENT_ID: ${{ steps.deployment.outputs.result }} + PREVIEW_URL: ${{ steps.deploy.outputs.preview_url }} with: script: | const state = '${{ job.status }}' === 'success' ? 'success' : 'failure'; + const previewUrl = process.env.PREVIEW_URL || undefined; await github.rest.repos.createDeploymentStatus({ owner: context.repo.owner, repo: context.repo.repo, - deployment_id: ${{ steps.deployment.outputs.result }}, + deployment_id: Number(process.env.DEPLOYMENT_ID), state: state, - environment_url: state === 'success' ? '${{ steps.deploy.outputs.preview_url }}' : undefined, + environment_url: state === 'success' ? previewUrl : undefined, description: state === 'success' ? 'Preview deployed successfully' : 'Preview deployment failed' }); - name: Comment on PR if: success() - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 env: PREVIEW_URL: ${{ steps.deploy.outputs.preview_url }} with: @@ -229,9 +234,8 @@ jobs: } - name: Store cleanup metadata if: success() - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: preview-cleanup-pr-${{ env.PR_NUMBER }} - path: | - workers/preview/wrangler.toml + path: ${{ runner.temp }}/sure-preview-worker/wrangler.toml retention-days: 2 diff --git a/app/components/UI/account/chart.html.erb b/app/components/UI/account/chart.html.erb index 91fb0b69c..c14627575 100644 --- a/app/components/UI/account/chart.html.erb +++ b/app/components/UI/account/chart.html.erb @@ -4,7 +4,7 @@
<%= tag.p title, class: "text-sm font-medium text-secondary" %> - <% if account.investment? %> + <% if account.supports_trades? %> <%= render "investments/value_tooltip", balance: account.balance_money, holdings: holdings_value_money, cash: account.cash_balance_money %> <% end %>
@@ -19,7 +19,7 @@ <%= form_with url: account_path(account), method: :get, data: { controller: "auto-submit-form" } do |form| %>
- <% if account.investment? %> + <% if account.supports_trades? %> <%= form.select :chart_view, [[t(".views.total_value"), "balance"], [t(".views.holdings"), "holdings_balance"], [t(".views.cash"), "cash_balance"]], { selected: view }, diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 2becdd3b3..be8f2adf2 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -24,6 +24,7 @@ class AccountsController < ApplicationController @ibkr_items = visible_provider_items(family.ibkr_items.ordered.includes(:syncs, :ibkr_accounts)) @indexa_capital_items = visible_provider_items(family.indexa_capital_items.ordered.includes(:syncs, :indexa_capital_accounts)) @sophtron_items = visible_provider_items(family.sophtron_items.ordered.includes(:syncs, :sophtron_accounts)) + @binance_items = visible_provider_items(family.binance_items.ordered.includes(:binance_accounts, :accounts, :syncs)) # Build sync stats maps for all providers build_sync_stats_maps @@ -397,5 +398,20 @@ class AccountsController < ApplicationController latest_sync = item.syncs.ordered.first @indexa_capital_sync_stats_map[item.id] = latest_sync&.sync_stats || {} end + + # Binance sync stats + @binance_sync_stats_map = {} + @binance_unlinked_count_map = {} + @binance_items.each do |item| + latest_sync = item.syncs.ordered.first + @binance_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + + # Count unlinked accounts + count = item.binance_accounts + .left_joins(:account_provider) + .where(account_providers: { id: nil }) + .count + @binance_unlinked_count_map[item.id] = count + end end end diff --git a/app/controllers/binance_items_controller.rb b/app/controllers/binance_items_controller.rb index 7f3471e58..292d77cc8 100644 --- a/app/controllers/binance_items_controller.rb +++ b/app/controllers/binance_items_controller.rb @@ -211,7 +211,23 @@ class BinanceItemsController < ApplicationController end def complete_account_setup - selected_accounts = Array(params[:selected_accounts]).reject(&:blank?) + setup_params = complete_account_setup_params + + if setup_params[:sync_start_date].present? + parsed_date = begin + Date.parse(setup_params[:sync_start_date].to_s) + rescue ArgumentError + nil + end + + if parsed_date.present? && parsed_date <= Date.current + @binance_item.update!(sync_start_date: parsed_date) + else + flash.now[:alert] = "Sync start date must be a valid date in the past." + end + end + + selected_accounts = Array(setup_params[:selected_accounts]).reject(&:blank?) created_accounts = [] selected_accounts.each do |binance_account_id| @@ -284,4 +300,8 @@ class BinanceItemsController < ApplicationController def binance_item_params params.require(:binance_item).permit(:name, :sync_start_date, :api_key, :api_secret) end + + def complete_account_setup_params + params.permit(:sync_start_date, selected_accounts: []) + end end diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb index a38e5166c..2a9aa6402 100644 --- a/app/controllers/categories_controller.rb +++ b/app/controllers/categories_controller.rb @@ -14,6 +14,12 @@ class CategoriesController < ApplicationController set_categories end + def merge + @categories = Current.family.categories.alphabetically + + render layout: turbo_frame_request? ? false : "settings" + end + def create @category = Current.family.categories.new(category_params) @@ -67,6 +73,29 @@ class CategoriesController < ApplicationController redirect_back_or_to categories_path, notice: t(".success") end + def perform_merge + permitted_params = category_merge_params + + if permitted_params[:target_id].present? && Array(permitted_params[:source_ids]).include?(permitted_params[:target_id]) + return redirect_to merge_categories_path, alert: t(".target_selected_as_source") + end + + target = Current.family.categories.find_by(id: permitted_params[:target_id]) + return redirect_to merge_categories_path, alert: t(".target_not_found") unless target + + sources = Current.family.categories.where(id: permitted_params[:source_ids]) + return redirect_to merge_categories_path, alert: t(".invalid_categories") unless sources.any? + + merger = Category::Merger.new(family: Current.family, target_category: target, source_categories: sources) + return redirect_to merge_categories_path, alert: t(".no_categories_selected") unless merger.merge! + + redirect_to categories_path, notice: t(".success", count: merger.merged_count) + rescue Category::Merger::UnauthorizedCategoryError => e + redirect_to merge_categories_path, alert: e.message + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotDestroyed => e + redirect_to merge_categories_path, alert: record_error_message(e) + end + private def set_category @category = Current.family.categories.find(params[:id]) @@ -89,4 +118,13 @@ class CategoriesController < ApplicationController def category_params params.require(:category).permit(:name, :color, :parent_id, :lucide_icon) end + + def category_merge_params + params.permit(:target_id, source_ids: []) + end + + def record_error_message(error) + record = error.respond_to?(:record) ? error.record : nil + record&.errors&.full_messages&.to_sentence.presence || error.message + end end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 94526c310..194a55dd3 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -2,6 +2,7 @@ class ImportsController < ApplicationController include SettingsHelper before_action :set_import, only: %i[show update publish destroy revert apply_template] + before_action :require_statement_import_permission!, only: %i[update publish destroy revert apply_template] def update # Handle both pdf_import[account_id] and import[account_id] param formats @@ -13,7 +14,9 @@ class ImportsController < ApplicationController redirect_back_or_to import_path(@import), alert: t("imports.update.invalid_account", default: "Account not found.") return end - @import.update!(account: account) + return if @import.account_statement.present? && !require_account_permission!(account) + + @import.is_a?(PdfImport) ? @import.assign_account!(account) : @import.update!(account: account) end redirect_to import_path(@import), notice: t("imports.update.account_saved", default: "Account saved.") @@ -134,24 +137,32 @@ class ImportsController < ApplicationController private def set_import - @import = Current.family.imports.includes(:account).find(params[:id]) + @import = Current.family.imports.includes(:account, :account_statement).find(params[:id]) + raise ActiveRecord::RecordNotFound if @import.account_statement.present? && !@import.account_statement.viewable_by?(Current.user) end def import_params params.require(:import).permit(:import_file) end + def require_statement_import_permission! + return if @import.account_statement.blank? || @import.account_statement.manageable_by?(Current.user) + + redirect_target = @import.account || @import.account_statement + redirect_back_or_to redirect_target, alert: t("accounts.not_authorized") + end + def create_pdf_import(file) - if file.size > Import::MAX_PDF_SIZE - redirect_to new_import_path, alert: t("imports.create.pdf_too_large", max_size: Import::MAX_PDF_SIZE / 1.megabyte) - return - end + return redirect_to new_import_path, alert: t("accounts.not_authorized") unless AccountStatement.statement_manager?(Current.user) + return redirect_to new_import_path, alert: t("imports.create.pdf_too_large", max_size: Import::MAX_PDF_SIZE / 1.megabyte) if file.size > Import::MAX_PDF_SIZE - pdf_import = Current.family.imports.create!(type: "PdfImport") - pdf_import.pdf_file.attach(file) + pdf_import = PdfImport.create_from_upload!(family: Current.family, file: file, user: Current.user) pdf_import.process_with_ai_later - redirect_to import_path(pdf_import), notice: t("imports.create.pdf_processing") + rescue AccountStatement::DuplicateUploadError + redirect_to new_import_path, alert: t("imports.create.duplicate_pdf_unavailable") + rescue AccountStatement::InvalidUploadError + redirect_to new_import_path, alert: t("imports.create.invalid_pdf") end def create_document_import(file) diff --git a/app/controllers/invitations_controller.rb b/app/controllers/invitations_controller.rb index ba98b9e68..f11965a0a 100644 --- a/app/controllers/invitations_controller.rb +++ b/app/controllers/invitations_controller.rb @@ -17,7 +17,9 @@ class InvitationsController < ApplicationController if @invitation.save normalized_email = @invitation.email.to_s.strip.downcase existing_user = User.find_by(email: normalized_email) - if existing_user && @invitation.accept_for(existing_user) + if existing_user && @invitation.would_orphan_owned_accounts?(existing_user) + flash[:alert] = t(".existing_user_has_family_data") + elsif existing_user && @invitation.accept_for(existing_user) flash[:notice] = t(".existing_user_added") elsif existing_user flash[:alert] = t(".failure") diff --git a/app/controllers/plaid_items_controller.rb b/app/controllers/plaid_items_controller.rb index 08de7e0b4..39a0e7226 100644 --- a/app/controllers/plaid_items_controller.rb +++ b/app/controllers/plaid_items_controller.rb @@ -1,4 +1,6 @@ class PlaidItemsController < ApplicationController + include StreamExtensions + before_action :set_plaid_item, only: %i[edit destroy sync] before_action :require_admin!, only: %i[new create select_existing_account link_existing_account edit destroy sync] @@ -12,6 +14,8 @@ class PlaidItemsController < ApplicationController accountable_type: params[:accountable_type] || "Depository", region: region ) + rescue Plaid::ApiError => e + handle_link_token_error(e) end def edit @@ -21,6 +25,8 @@ class PlaidItemsController < ApplicationController webhooks_url: webhooks_url, redirect_url: accounts_url, ) + rescue Plaid::ApiError => e + handle_link_token_error(e) end def create @@ -104,6 +110,58 @@ class PlaidItemsController < ApplicationController plaid_item_params.dig(:metadata, :institution, :name) end + # When `link_token/create` (or the update equivalent) raises, surface a + # friendly alert to the user instead of letting the modal frame render + # blank. Plaid configuration/product-access errors are the common case for + # self-hosted users — without this, the Link modal simply never opens and + # the only signal lives in server logs. + def handle_link_token_error(error) + error_body = safe_parse_plaid_error(error) + error_code = error_body["error_code"].to_s + + Rails.logger.warn( + "Plaid link_token request failed: #{error_code} - #{error_body['error_message']}" + ) + Sentry.capture_exception(error) if defined?(Sentry) + + alert = friendly_link_token_alert(error_code, error_body["error_message"]) + + respond_to do |format| + format.html { redirect_to accounts_path, alert: alert } + format.turbo_stream { stream_redirect_to(accounts_path, alert: alert) } + end + end + + def safe_parse_plaid_error(error) + JSON.parse(error.response_body.to_s) + rescue JSON::ParserError + {} + end + + # Plaid surfaces its own actionable copy on configuration / product-access + # failures (e.g. "Your account is not enabled for the following products + # [...]. To request access, visit dashboard.plaid.com..."). Those messages + # are safe to show verbatim — they describe a Plaid-side config issue, + # not user data. For everything else we fall back to a generic message + # and rely on the log + Sentry trail. + SHOWABLE_PLAID_ERROR_CODES = %w[ + INVALID_PRODUCT + PRODUCTS_NOT_SUPPORTED + NO_PRODUCTS_PERMISSION + ADDITION_LIMIT + INVALID_INSTITUTION + INSTITUTION_NOT_ENABLED_IN_REGION + INSTITUTION_NOT_SUPPORTED + ].freeze + + def friendly_link_token_alert(error_code, error_message) + if SHOWABLE_PLAID_ERROR_CODES.include?(error_code) && error_message.present? + t("plaid_items.errors.link_token_with_message", message: error_message) + else + t("plaid_items.errors.link_token_generic") + end + end + def plaid_us_webhooks_url return webhooks_plaid_url if Rails.env.production? diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb index 26a03f23d..9c5bd46a5 100644 --- a/app/controllers/reports_controller.rb +++ b/app/controllers/reports_controller.rb @@ -471,6 +471,7 @@ class ReportsController < ApplicationController has_investments: true, portfolio_value: investment_statement.portfolio_value_money, unrealized_trend: investment_statement.unrealized_gains_trend, + period_return_trend: investment_statement.period_return_trend(period: @period), period_contributions: period_totals.contributions, period_withdrawals: period_totals.withdrawals, top_holdings: investment_statement.top_holdings(limit: 5), diff --git a/app/controllers/settings/profiles_controller.rb b/app/controllers/settings/profiles_controller.rb index e1f3b5db4..c08341eba 100644 --- a/app/controllers/settings/profiles_controller.rb +++ b/app/controllers/settings/profiles_controller.rb @@ -26,6 +26,12 @@ class Settings::ProfilesController < ApplicationController return end + if @user.owned_accounts.where.not(family_id: Current.family.id).exists? + flash[:alert] = t(".member_owns_other_family_data") + redirect_to settings_profile_path + return + end + if @user.destroy # Also destroy the invitation associated with this user for this family Current.family.invitations.find_by(email: @user.email)&.destroy diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb index 66677a6c1..3e5c73e3a 100644 --- a/app/helpers/languages_helper.rb +++ b/app/helpers/languages_helper.rb @@ -168,7 +168,8 @@ module LanguagesHelper "zh-CN", # Chinese (Simplified) "zh-TW", # Chinese (Traditional) "nl", # Dutch - "hu" # Hungarian + "hu", # Hungarian + "vi" # Vietnamese ].freeze COUNTRY_MAPPING = { diff --git a/app/javascript/controllers/chat_controller.js b/app/javascript/controllers/chat_controller.js index 7e067309c..25020f35e 100644 --- a/app/javascript/controllers/chat_controller.js +++ b/app/javascript/controllers/chat_controller.js @@ -1,10 +1,11 @@ import { Controller } from "@hotwired/stimulus"; export default class extends Controller { - static targets = ["messages", "form", "input"]; + static targets = ["messages", "form", "input", "submit"]; connect() { this.#configureAutoScroll(); + this.#updateSubmitState(); } disconnect() { @@ -22,10 +23,13 @@ export default class extends Controller { input.style.height = `${Math.min(input.scrollHeight, lineHeight * maxLines)}px`; input.style.overflowY = input.scrollHeight > lineHeight * maxLines ? "auto" : "hidden"; + + this.#updateSubmitState(); } submitSampleQuestion(e) { this.inputTarget.value = e.target.dataset.chatQuestionParam; + this.#updateSubmitState(); setTimeout(() => { this.formTarget.requestSubmit(); @@ -36,10 +40,21 @@ export default class extends Controller { handleInputKeyDown(e) { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); - this.formTarget.requestSubmit(); + if (this.#hasContent()) { + this.formTarget.requestSubmit(); + } } } + #hasContent() { + return this.inputTarget.value.trim().length > 0; + } + + #updateSubmitState() { + if (!this.hasSubmitTarget) return; + this.submitTarget.disabled = !this.#hasContent(); + } + #configureAutoScroll() { this.messagesObserver = new MutationObserver((_mutations) => { if (this.hasMessagesTarget) { diff --git a/app/javascript/controllers/plaid_controller.js b/app/javascript/controllers/plaid_controller.js index e24031f77..735c6f420 100644 --- a/app/javascript/controllers/plaid_controller.js +++ b/app/javascript/controllers/plaid_controller.js @@ -128,9 +128,32 @@ export default class extends Controller { }; handleExit = (err, metadata) => { - // If there was an error during update mode, refresh the page to show latest status - if (err && metadata.status === "requires_credentials") { + // If there was an error during update mode, refresh the page to show + // latest status. Guard `metadata` (Plaid can fire onExit with it + // undefined when Link aborts very early) and gate the redirect on + // `isUpdateValue` so first-time link failures don't bounce the user + // away from whatever page they were on. + if ( + err && + metadata && + metadata.status === "requires_credentials" && + this.isUpdateValue + ) { window.location.href = "/accounts"; + return; + } + + // Promote Plaid's own error payload to the console so a silent modal + // close still leaves a breadcrumb (issue #1792). Plaid Link's own UI + // is responsible for showing a message inside the modal when this + // fires; backend link-token failures are handled server-side via the + // PlaidItemsController rescue + flash. + if (err?.error_code) { + console.error( + "Plaid Link exited with error", + err.error_code, + err.display_message || err.error_message + ); } }; diff --git a/app/javascript/controllers/sankey_chart_controller.js b/app/javascript/controllers/sankey_chart_controller.js index 1af8a25a8..cb24edeb9 100644 --- a/app/javascript/controllers/sankey_chart_controller.js +++ b/app/javascript/controllers/sankey_chart_controller.js @@ -511,7 +511,7 @@ export default class extends Controller { .append("div") .attr( "class", - "bg-gray-700 text-white text-sm p-2 rounded pointer-events-none absolute z-50 top-0", + "bg-container text-primary text-sm font-sans p-2 border border-secondary rounded-lg pointer-events-none absolute z-50 top-0 privacy-sensitive", ) .style("opacity", 0) .style("pointer-events", "none"); diff --git a/app/jobs/identify_recurring_transactions_job.rb b/app/jobs/identify_recurring_transactions_job.rb index 9bb2c4156..d6ca2dffa 100644 --- a/app/jobs/identify_recurring_transactions_job.rb +++ b/app/jobs/identify_recurring_transactions_job.rb @@ -42,21 +42,13 @@ class IdentifyRecurringTransactionsJob < ApplicationJob "recurring_transaction_identify:#{family_id}" end + # Debounce gate: delegate to `Sync.any_incomplete_for?`, which polls every + # `Syncable` provider association on `Family` via reflection. The previous + # hand-rolled list covered only 5 of the 14 `*_items` associations on + # `Family`, so a Coinbase/Mercury/Brex/etc. sync in flight silently + # bypassed this gate and let the identifier run against a partial dataset. def family_has_incomplete_syncs?(family) - # Check family's own syncs - return true if family.syncs.incomplete.exists? - - # Check all provider items' syncs - return true if family.plaid_items.joins(:syncs).merge(Sync.incomplete).exists? if family.respond_to?(:plaid_items) - return true if family.simplefin_items.joins(:syncs).merge(Sync.incomplete).exists? if family.respond_to?(:simplefin_items) - return true if family.lunchflow_items.joins(:syncs).merge(Sync.incomplete).exists? if family.respond_to?(:lunchflow_items) - return true if family.enable_banking_items.joins(:syncs).merge(Sync.incomplete).exists? if family.respond_to?(:enable_banking_items) - return true if family.sophtron_items.joins(:syncs).merge(Sync.incomplete).exists? if family.respond_to?(:sophtron_items) - - # Check accounts' syncs - return true if family.accounts.joins(:syncs).merge(Sync.incomplete).exists? - - false + Sync.any_incomplete_for?(family) end def with_advisory_lock(family_id) diff --git a/app/jobs/process_pdf_job.rb b/app/jobs/process_pdf_job.rb index 1ed48cf49..c577806dc 100644 --- a/app/jobs/process_pdf_job.rb +++ b/app/jobs/process_pdf_job.rb @@ -3,9 +3,9 @@ class ProcessPdfJob < ApplicationJob def perform(pdf_import) return unless pdf_import.is_a?(PdfImport) - return unless pdf_import.pdf_uploaded? + return reset_processing_claim(pdf_import) unless pdf_import.pdf_uploaded? return if pdf_import.status == "complete" - return if pdf_import.ai_processed? && (!pdf_import.statement_with_transactions? || pdf_import.rows_count > 0) + return reset_processing_claim(pdf_import) if pdf_import.ai_processed? && (!pdf_import.statement_with_transactions? || pdf_import.rows_count > 0) pdf_import.update!(status: :importing) @@ -62,12 +62,11 @@ class ProcessPdfJob < ApplicationJob end def upload_to_vector_store(pdf_import, document_type:) - filename = pdf_import.pdf_file.filename.to_s file_content = pdf_import.pdf_file_content family_document = pdf_import.family.upload_document( file_content: file_content, - filename: filename, + filename: pdf_import.pdf_filename, metadata: { "type" => document_type } ) @@ -85,4 +84,8 @@ class ProcessPdfJob < ApplicationJob def statement_with_transactions?(document_type) document_type.in?(%w[bank_statement credit_card_statement]) end + + def reset_processing_claim(pdf_import) + pdf_import.with_lock { pdf_import.update!(status: :pending) if pdf_import.importing? && pdf_import.updated_at <= 30.minutes.ago } + end end diff --git a/app/models/account.rb b/app/models/account.rb index 458f50bbe..48a64b796 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -266,7 +266,9 @@ class Account < ApplicationRecord end def create_from_binance_account(binance_account) - create_from_crypto_exchange_account(binance_account, family: binance_account.binance_item.family) + account = create_from_crypto_exchange_account(binance_account, family: binance_account.binance_item.family) + account.set_opening_anchor_balance(balance: 0) + account end def create_from_ibkr_account(ibkr_account) @@ -289,6 +291,7 @@ class Account < ApplicationRecord } } + # Capture the created account in a variable create_and_sync(attributes, skip_initial_sync: true) end diff --git a/app/models/account/provider_import_adapter.rb b/app/models/account/provider_import_adapter.rb index d307e81bf..f5bfe202f 100644 --- a/app/models/account/provider_import_adapter.rb +++ b/app/models/account/provider_import_adapter.rb @@ -44,12 +44,37 @@ class Account::ProviderImportAdapter raise ArgumentError, "Entry with external_id '#{external_id}' already exists with different entryable type: #{entry.entryable_type}" end + # Determine early whether the incoming transaction is pending — needed by both + # the protection check (pending→booked bypass) and the auto-claim path below. + incoming_pending = false + if extra.is_a?(Hash) + pending_extra = extra.with_indifferent_access + incoming_pending = + ActiveModel::Type::Boolean.new.cast(pending_extra.dig("simplefin", "pending")) || + ActiveModel::Type::Boolean.new.cast(pending_extra.dig("plaid", "pending")) || + ActiveModel::Type::Boolean.new.cast(pending_extra.dig("lunchflow", "pending")) || + ActiveModel::Type::Boolean.new.cast(pending_extra.dig("enable_banking", "pending")) + end + # === PROTECTION CHECK: Skip entries that should not be overwritten === # Check persisted Transaction entries for protection flags before making changes. # This prevents sync from overwriting user edits, CSV imports, or excluded entries. if entry.persisted? skip_reason = determine_skip_reason(entry) if skip_reason + # Pending→booked bypass for user_modified entries: clear the stale pending flag + # when the provider delivers a booked version of the same transaction. + # Some ASPSPs (e.g. Revolut Italy via Enable Banking) reuse the same transaction_id + # for pending and booked, so the entry is found by external_id rather than going + # through the auto-claim path. Without this, a user who categorised a pending entry + # (setting user_modified=true) would see the pending badge stuck forever. + # Excluded and import_locked entries are intentionally left untouched. + if skip_reason == "user_modified" && !incoming_pending && entry.entryable.is_a?(Transaction) + entry_is_pending = Transaction::PENDING_PROVIDERS.any? { |p| entry.transaction.extra&.dig(p, "pending") } + if entry_is_pending + entry.transaction.update!(extra: clear_pending_flags_from_extra(entry.transaction.extra)) + end + end record_skip(entry, skip_reason) return entry end @@ -76,17 +101,6 @@ class Account::ProviderImportAdapter end end - # If still a new entry and this is a POSTED transaction, check for matching pending transactions - incoming_pending = false - if extra.is_a?(Hash) - pending_extra = extra.with_indifferent_access - incoming_pending = - ActiveModel::Type::Boolean.new.cast(pending_extra.dig("simplefin", "pending")) || - ActiveModel::Type::Boolean.new.cast(pending_extra.dig("plaid", "pending")) || - ActiveModel::Type::Boolean.new.cast(pending_extra.dig("lunchflow", "pending")) || - ActiveModel::Type::Boolean.new.cast(pending_extra.dig("enable_banking", "pending")) - end - if entry.new_record? && !incoming_pending pending_match = nil @@ -119,12 +133,7 @@ class Account::ProviderImportAdapter # exclude it from re-import (preventing the old pending from being recreated on the # next sync when the stored raw payload still contains the pending transaction data). if entry.entryable.is_a?(Transaction) - ex = (entry.transaction.extra || {}).deep_dup - Transaction::PENDING_PROVIDERS.each do |provider| - next unless ex.key?(provider) - ex[provider].delete("pending") - ex.delete(provider) if ex[provider].empty? - end + ex = clear_pending_flags_from_extra(entry.transaction.extra) if old_pending_external_id.present? existing_claims = Array.wrap(ex["auto_claimed_pending_ids"]) ex["auto_claimed_pending_ids"] = (existing_claims + [ old_pending_external_id ]).uniq @@ -134,6 +143,18 @@ class Account::ProviderImportAdapter end end + # Pending→booked for same-external-id providers (non-protected path). + # For ASPSPs like Revolut Italy that reuse the same transaction_id for pending and + # booked, the auto-claim path above is skipped (entry.persisted? from the start). + # If extra is nil (no FX, no MCC) the deep-merge block later is skipped too, so we + # must clear the stale pending flag here before the final save. + # (The auto-claim path already clears it in-memory, so this is a no-op there.) + if !incoming_pending && entry.entryable.is_a?(Transaction) + if Transaction::PENDING_PROVIDERS.any? { |p| entry.transaction.extra&.dig(p, "pending") } + entry.transaction.extra = clear_pending_flags_from_extra(entry.transaction.extra) + end + end + # Track if this is a new posted transaction (for fuzzy suggestion after save) is_new_posted = entry.new_record? && !incoming_pending @@ -977,11 +998,25 @@ class Account::ProviderImportAdapter } end - # Memoized per adapter instance (which is per-account). Membership in - # goal_accounts is stable across a sync batch. - def account_linked_to_any_goal? - return @account_linked_to_any_goal if defined?(@account_linked_to_any_goal) + private - @account_linked_to_any_goal = account.goal_accounts.exists? - end + # Memoized per adapter instance (which is per-account). Membership in + # goal_accounts is stable across a sync batch. + def account_linked_to_any_goal? + return @account_linked_to_any_goal if defined?(@account_linked_to_any_goal) + + @account_linked_to_any_goal = account.goal_accounts.exists? + end + + def clear_pending_flags_from_extra(extra) + ex = (extra || {}).deep_dup + ex = {} unless ex.is_a?(Hash) + Transaction::PENDING_PROVIDERS.each do |provider| + next unless ex.key?(provider) + next unless ex[provider].is_a?(Hash) + ex[provider].delete("pending") + ex.delete(provider) if ex[provider].empty? + end + ex + end end diff --git a/app/models/account_statement.rb b/app/models/account_statement.rb index 07cc80c86..5240093fd 100644 --- a/app/models/account_statement.rb +++ b/app/models/account_statement.rb @@ -33,6 +33,7 @@ class AccountStatement < ApplicationRecord belongs_to :account, optional: true belongs_to :suggested_account, class_name: "Account", optional: true + has_many :pdf_imports, -> { where(type: "PdfImport").ordered }, class_name: "PdfImport", dependent: :restrict_with_error has_one_attached :original_file, dependent: :purge_later enum :source, { manual_upload: "manual_upload" }, validate: true, default: "manual_upload" @@ -360,6 +361,10 @@ class AccountStatement < ApplicationRecord content_type.in?(ALLOWED_EXTENSION_CONTENT_TYPES[".xlsx"]) end + def latest_reusable_pdf_import + pdf_imports.where.not(status: :failed).order(created_at: :desc).first + end + private def reconciliation_check(key:, statement_amount:, ledger_amount:) diff --git a/app/models/assistant.rb b/app/models/assistant.rb index 0a779b680..583737761 100644 --- a/app/models/assistant.rb +++ b/app/models/assistant.rb @@ -27,6 +27,7 @@ module Assistant Function::GetHoldings, Function::GetBalanceSheet, Function::GetIncomeStatement, + Function::GetBudget, Function::ImportBankStatement, Function::SearchFamilyFiles, Function::CreateGoal diff --git a/app/models/assistant/function/get_budget.rb b/app/models/assistant/function/get_budget.rb new file mode 100644 index 000000000..3cf95115b --- /dev/null +++ b/app/models/assistant/function/get_budget.rb @@ -0,0 +1,200 @@ +class Assistant::Function::GetBudget < Assistant::Function + include ActiveSupport::NumberHelper + + MAX_PRIOR_MONTHS = 11 + + class << self + def name + "get_budget" + end + + def description + <<~INSTRUCTIONS + Use this to see how the user is tracking against their monthly budget — total + budgeted vs spent and a parent/subcategory breakdown matching the budget UI. + + This is great for answering questions like: + - How am I tracking against my budget this month? + - Which categories am I over budget on? + - How does this month's spending compare to the last few months? + + Parameters: + - `month` (optional): "YYYY-MM" or "MMM-YYYY". Defaults to the current month. + - `prior_months` (optional): integer 0..#{MAX_PRIOR_MONTHS}. Number of months + preceding the target month to include for trend comparison. Default 0. + + Example (current month only): + + ``` + get_budget({}) + ``` + + Example (current month plus last 2 months): + + ``` + get_budget({ month: "#{Date.current.strftime('%Y-%m')}", prior_months: 2 }) + ``` + INSTRUCTIONS + end + end + + def strict_mode? + false + end + + def params_schema + build_schema( + properties: { + month: { + type: "string", + description: "Target month in YYYY-MM or MMM-YYYY format. Defaults to the current month." + }, + prior_months: { + type: "integer", + description: "Number of months before the target month to also return for trend comparison.", + minimum: 0, + maximum: MAX_PRIOR_MONTHS + } + } + ) + end + + def call(params = {}) + target_start = resolve_month_start(params["month"]) + prior = [ params["prior_months"].to_i, 0 ].max + prior = [ prior, MAX_PRIOR_MONTHS ].min + + month_starts = (0..prior).map { |offset| shift_months(target_start, -offset) }.reverse + requested = month_starts.count { |start_date| Budget.budget_date_valid?(start_date, family: family) } + + months = month_starts.filter_map do |start_date| + next unless Budget.budget_date_valid?(start_date, family: family) + build_month_payload(start_date, bootstrap: start_date == target_start) + end + + result = { + currency: family.currency, + months: months + } + unavailable = requested - months.length + result[:months_unavailable] = unavailable if unavailable > 0 + result + end + + private + def build_month_payload(start_date, bootstrap:) + budget = if bootstrap + Budget.find_or_bootstrap(family, start_date: start_date, user: user) + else + budget_start, budget_end = Budget.period_for(start_date, family: family) + family.budgets.find_by(start_date: budget_start, end_date: budget_end) + end + return nil unless budget + + groups = BudgetCategory::Group.for(budget.budget_categories) + + { + month: budget.to_param, + period: { + start_date: budget.start_date, + end_date: budget.end_date + }, + is_current: budget.current?, + initialized: budget.initialized?, + totals: { + budgeted_spending: format_money(budget.budgeted_spending), + allocated_spending: format_money(budget.allocated_spending), + available_to_allocate: format_money(budget.available_to_allocate), + actual_spending: format_money(budget.actual_spending), + available_to_spend: format_money(budget.available_to_spend), + percent_of_budget_spent: format_percent(budget.initialized? ? budget.percent_of_budget_spent : 0), + overage_percent: format_percent(budget.overage_percent) + }, + income: { + expected_income: format_money(budget.expected_income), + actual_income: format_money(budget.actual_income), + remaining_expected_income: format_money((budget.expected_income || 0) - budget.actual_income) + }, + categories: groups.map { |group| serialize_group(group, include_daily_suggestion: budget.current?) } + } + end + + def serialize_group(group, include_daily_suggestion:) + parent = group.budget_category + serialize_category(parent, include_daily_suggestion: include_daily_suggestion).merge( + color: parent.category.color, + subcategories: group.budget_subcategories.map do |sub| + serialize_category(sub, include_daily_suggestion: include_daily_suggestion).merge( + inherits_parent_budget: sub.inherits_parent_budget? + ) + end + ) + end + + def serialize_category(bc, include_daily_suggestion:) + payload = { + name: bc.name, + budgeted: format_money(bc.display_budgeted_spending), + actual: format_money(bc.actual_spending), + available: format_money(bc.available_to_spend), + percent_spent: format_percent(bc.percent_of_budget_spent || 0), + status: category_status(bc) + } + + if include_daily_suggestion + suggestion = bc.suggested_daily_spending + payload[:suggested_daily_spending] = suggestion[:amount].format if suggestion + end + + payload + end + + def category_status(bc) + return "over_budget" if bc.over_budget_with_budget? + return "unbudgeted" if bc.unbudgeted_with_spending? + return "near_limit" if bc.budgeted? && bc.near_limit? + return "on_track" if bc.on_track? + "no_activity" + end + + def resolve_month_start(raw) + base = parse_month(raw) + return (base || Date.current).beginning_of_month unless family.uses_custom_month_start? + + # Match Budget.param_to_date for explicit slugs so the input round-trips with the response. + base ? Date.new(base.year, base.month, family.month_start_day) : family.custom_month_start_for(Date.current) + end + + def parse_month(raw) + return nil if raw.blank? + + # Date.strptime ignores trailing characters, so guard with strict anchors first. + fmt = case raw + when /\A\d{4}-\d{2}\z/ then "%Y-%m" + when /\A[A-Za-z]{3}-\d{4}\z/ then "%b-%Y" + end + + raise Assistant::Error, "Invalid month: #{raw}. Use YYYY-MM or MMM-YYYY." if fmt.nil? + + Date.strptime(raw, fmt) + rescue ArgumentError + raise Assistant::Error, "Invalid month: #{raw}. Use YYYY-MM or MMM-YYYY." + end + + def shift_months(date, n) + shifted = date >> n + if family.uses_custom_month_start? + family.custom_month_start_for(shifted) + else + shifted.beginning_of_month + end + end + + def format_money(value) + Money.new(value || 0, family.currency).format + end + + def format_percent(value) + number_to_percentage(value || 0, precision: 1) + end +end diff --git a/app/models/balance/reverse_calculator.rb b/app/models/balance/reverse_calculator.rb index 39b530178..ba970fdcd 100644 --- a/app/models/balance/reverse_calculator.rb +++ b/app/models/balance/reverse_calculator.rb @@ -25,20 +25,20 @@ class Balance::ReverseCalculator < Balance::BaseCalculator start_non_cash_balance = end_non_cash_balance market_value_change = 0 elsif valuation && valuation.entryable.reconciliation? - # Reconciliation waypoint: reset to the known API-reported balance. - # These waypoints are created by CurrentBalanceManager when it preserves - # a stale current_anchor as a reconciliation before replacing it. - # We derive both cash and non-cash from the total to ensure the split - # reflects the account's cash ratio on that date. + # Reconciliation waypoint: hard-reset the END-of-day balance to the + # API-reported value, neutralizing any drift accumulated from missing + # transactions between here and the next anchor. The START is still + # derived from this day's own flows, so a same-day transaction is + # attributed exactly once (and not added on top of the waypoint). end_cash_balance = derive_cash_balance_on_date_from_total( total_balance: valuation.amount, date: date ) end_non_cash_balance = valuation.amount - end_cash_balance - start_cash_balance = end_cash_balance - start_non_cash_balance = end_non_cash_balance - market_value_change = 0 + start_cash_balance = derive_start_cash_balance(end_cash_balance: end_cash_balance, date: date) + start_non_cash_balance = derive_start_non_cash_balance(end_non_cash_balance: end_non_cash_balance, date: date) + market_value_change = market_value_change_on_date(date, flows) else start_cash_balance = derive_start_cash_balance(end_cash_balance: end_cash_balance, date: date) start_non_cash_balance = derive_start_non_cash_balance(end_non_cash_balance: end_non_cash_balance, date: date) diff --git a/app/models/binance_account/processor.rb b/app/models/binance_account/processor.rb index 540383ec1..21306f275 100644 --- a/app/models/binance_account/processor.rb +++ b/app/models/binance_account/processor.rb @@ -61,47 +61,83 @@ class BinanceAccount::Processor provider = binance_account.binance_item&.binance_provider return unless provider + # 1. Initialize data from existing payload + existing_spot = binance_account.raw_transactions_payload&.dig("spot") || {} + existing_futures = binance_account.raw_transactions_payload&.dig("futures") || {} + existing_p2p = binance_account.raw_transactions_payload&.dig("p2p") || [] + + # 2. Fetch P2P Trades (This now runs even if you have no spot assets) + new_p2p = fetch_new_p2p_trades(provider, existing_p2p) + + # 3. Handle Spot & Futures symbols symbols = extract_trade_symbols - return if symbols.empty? - - existing_spot = binance_account.raw_transactions_payload&.dig("spot") || {} new_trades_by_symbol = {} + new_futures_by_symbol = {} - symbols.each do |symbol| - TRADE_QUOTE_CURRENCIES.each do |quote| - pair = "#{symbol}#{quote}" - begin - new_trades = fetch_new_trades(provider, pair, existing_spot[pair]) - new_trades_by_symbol[pair] = new_trades if new_trades.present? - rescue Provider::Binance::InvalidSymbolError => e - # Pair doesn't exist on Binance for this quote currency — expected, skip silently - Rails.logger.debug "BinanceAccount::Processor - skipping #{pair}: #{e.message}" + # Only attempt to loop if we actually have symbols (e.g., BTC, ETH) + if symbols.any? + symbols.each do |symbol| + TRADE_QUOTE_CURRENCIES.each do |quote| + pair = "#{symbol}#{quote}" + begin + new_trades = fetch_new_trades(provider, pair, existing_spot[pair], :spot) + new_trades_by_symbol[pair] = new_trades if new_trades.present? + rescue Provider::Binance::InvalidSymbolError => e + Rails.logger.debug "BinanceAccount::Processor - skipping spot #{pair}: #{e.message}" + end + + begin + new_futures = fetch_new_trades(provider, pair, existing_futures[pair], :futures) + new_futures_by_symbol[pair] = new_futures if new_futures.present? + rescue Provider::Binance::InvalidSymbolError => e + Rails.logger.debug "BinanceAccount::Processor - skipping futures #{pair}: #{e.message}" + end end - # ApiError, AuthenticationError and RateLimitError propagate so the sync is marked failed end end - merged_spot = existing_spot.merge(new_trades_by_symbol) { |_pair, old, new_t| old + new_t } + # 4. Process New Records into Database Entries FIRST + # We process these into the DB first. If they fail or raise an error, + # the method halts before updating the raw_transactions_payload cache, + # ensuring a retry happens on the next sync execution. + process_trades(new_trades_by_symbol, :spot) if new_trades_by_symbol.any? + process_trades(new_futures_by_symbol, :futures) if new_futures_by_symbol.any? + process_p2p_trades(new_p2p) if new_p2p.any? + + # 5. Merge Results ONLY after successful DB insertion + merged_spot = existing_spot.merge(new_trades_by_symbol) { |_pair, old, new_t| old + new_t } + merged_futures = existing_futures.merge(new_futures_by_symbol) { |_pair, old, new_t| old + new_t } + merged_p2p = existing_p2p + new_p2p + + # 6. Update the Account Payload LAST (Safe Caching Boundary) binance_account.update!(raw_transactions_payload: { "spot" => merged_spot, + "futures" => merged_futures, + "p2p" => merged_p2p, "fetched_at" => Time.current.iso8601 }) - - process_trades(new_trades_by_symbol) end # Fetches only trades newer than what is already cached for the given pair. # On the first sync (no cached trades) fetches the most recent page. # On subsequent syncs starts from max_cached_id + 1 and paginates forward. - def fetch_new_trades(provider, pair, cached_trades) + def fetch_new_trades(provider, pair, cached_trades, market_type) limit = 1000 max_cached_id = cached_trades&.map { |t| t["id"].to_i }&.max from_id = max_cached_id ? max_cached_id + 1 : nil + start_time = nil + unless max_cached_id + start_time = binance_account.binance_item&.sync_start_date&.to_time&.to_i&.*(1000) + end all_new = [] loop do - page = provider.get_spot_trades(pair, limit: limit, from_id: from_id) + page = if market_type == :spot + provider.get_spot_trades(pair, limit: limit, from_id: from_id, startTime: start_time) + else + provider.get_futures_trades(pair, limit: limit, from_id: from_id, startTime: start_time) + end break if page.blank? all_new.concat(page) @@ -113,6 +149,47 @@ class BinanceAccount::Processor all_new end + def fetch_new_p2p_trades(provider, cached_p2p) + # Binance P2P history endpoint only supports max 30-day windows. + # If no cache exists, we fetch back to sync_start_date (or default 30 days). + # If cache exists, we fetch from the last cached trade timestamp. + max_cached_timestamp = cached_p2p&.map { |t| t["createTime"].to_i }&.max + + start_time = if max_cached_timestamp + max_cached_timestamp + elsif binance_account.binance_item&.sync_start_date + binance_account.binance_item.sync_start_date.to_time.to_i * 1000 + else + (Time.current - 30.days).to_i * 1000 + end + + all_new = [] + current_start = start_time + + loop do + current_end = [ current_start + 30.days.to_i * 1000, Time.current.to_i * 1000 ].min + + page = provider.get_all_p2p_trades(start_timestamp: current_start, end_timestamp: current_end) + + # We might fetch overlapping trades if they share the exact timestamp, filter by unique orderNumber + if page.present? + cached_order_numbers = cached_p2p&.map { |t| t["orderNumber"] } || [] + new_order_numbers = all_new.map { |t| t["orderNumber"] } + + unique_page = page.reject do |t| + cached_order_numbers.include?(t["orderNumber"]) || new_order_numbers.include?(t["orderNumber"]) + end + + all_new.concat(unique_page) + end + + break if current_end >= Time.current.to_i * 1000 + current_start = current_end + 1 + end + + all_new + end + def extract_trade_symbols stablecoins = BinanceAccount::STABLECOINS quote_re = /(#{TRADE_QUOTE_CURRENCIES.join("|")})$/ @@ -122,21 +199,24 @@ class BinanceAccount::Processor current = assets.map { |a| a["symbol"] || a[:symbol] }.compact # Base symbols from previously fetched pairs (recovers sold-out assets) - prev_pairs = binance_account.raw_transactions_payload&.dig("spot")&.keys || [] + prev_spot = binance_account.raw_transactions_payload&.dig("spot")&.keys || [] + prev_futures = binance_account.raw_transactions_payload&.dig("futures")&.keys || [] + prev_pairs = (prev_spot + prev_futures).uniq previous = prev_pairs.map { |pair| pair.gsub(quote_re, "") } (current + previous).uniq.compact.reject { |s| s.blank? || stablecoins.include?(s) } end - def process_trades(trades_by_symbol) + def process_trades(trades_by_symbol, market_type) trades_by_symbol.each do |pair, trades| - trades.each { |trade| process_spot_trade(trade, pair) } + trades.each { |trade| process_trade(trade, pair, market_type) } end rescue StandardError => e Rails.logger.error "BinanceAccount::Processor - trade processing failed: #{e.message}" + raise end - def process_spot_trade(trade, pair) + def process_trade(trade, pair, market_type) account = binance_account.current_account return unless account @@ -149,7 +229,8 @@ class BinanceAccount::Processor return unless security - external_id = "binance_spot_#{pair}_#{trade["id"]}" + prefix = market_type == :spot ? "spot" : "futures" + external_id = "binance_#{prefix}_#{pair}_#{trade["id"]}" return if account.entries.exists?(external_id: external_id) date = Time.zone.at(trade["time"].to_i / 1000).to_date @@ -170,7 +251,7 @@ class BinanceAccount::Processor amount_usd = amount_usd_raw.round(2) commission = commission_in_usd(trade, base_symbol, price_usd, date: date) - is_buyer = trade["isBuyer"] + is_buyer = trade.key?("isBuyer") ? trade["isBuyer"] : trade["buyer"] if is_buyer account.entries.create!( @@ -209,23 +290,38 @@ class BinanceAccount::Processor end rescue StandardError => e Rails.logger.error "BinanceAccount::Processor - failed to process trade #{trade["id"]}: #{e.message}" + raise end # Converts an amount denominated in quote_symbol to USD. - # Stablecoins are treated as 1:1; others use historical price when date is given, - # falling back to current USDT spot price. + # Stablecoins are treated as 1:1. + # For fiat/crypto assets, tries Binance historical price first, falls back to internal ExchangeRate. def quote_to_usd(amount, quote_symbol, date: nil) return amount if BinanceAccount::STABLECOINS.include?(quote_symbol) + return amount if quote_symbol.to_s.upcase == "USD" provider = binance_account.binance_item&.binance_provider - return nil unless provider - spot = nil - spot = provider.get_historical_price("#{quote_symbol}USDT", date) if date.present? && provider.respond_to?(:get_historical_price) - spot ||= provider.get_spot_price("#{quote_symbol}USDT") - return nil if spot.nil? + if provider + spot = nil + begin + spot = provider.get_historical_price("#{quote_symbol}USDT", date) if date.present? && provider.respond_to?(:get_historical_price) + spot ||= provider.get_spot_price("#{quote_symbol}USDT") + rescue Provider::Binance::InvalidSymbolError + # Fall through to ExchangeRate lookup + end + return (amount * spot.to_d).round(8) if spot.present? + end - (amount * spot.to_d).round(8) + # Fallback to internal app ExchangeRate provider (crucial for P2P fiat currencies like TZS, NGN) + fallback_rate = ExchangeRate.find_or_fetch_rate(from: quote_symbol, to: "USD", date: date || Date.current, cache: true) + if fallback_rate.present? + # Extract the numeric rate from the returned object (or use it directly if it's already a number) + rate_val = fallback_rate.respond_to?(:rate) ? fallback_rate.rate : fallback_rate + return (amount * rate_val.to_d).round(8) + end + + nil rescue StandardError => e Rails.logger.warn "BinanceAccount::Processor - could not convert #{quote_symbol} to USD: #{e.message}" nil @@ -233,6 +329,117 @@ class BinanceAccount::Processor # Converts the trade commission to USD. # commissionAsset can be: a stablecoin (≈ 1 USD), the base asset, or something else (e.g. BNB). + def process_p2p_trades(trades) + account = binance_account.current_account + return unless account + + Rails.logger.info "BinanceAccount::Processor - found #{trades.size} P2P trades to process" + + trades.each do |trade| + external_id = "binance_p2p_#{trade["orderNumber"]}" + funding_external_id = "#{external_id}_funding" + + # Deduplicate by checking for either the Trade or Funding leg in a single query + if account.entries.where(external_id: [ external_id, funding_external_id ]).exists? + Rails.logger.info "BinanceAccount::Processor - skipping P2P trade #{trade["orderNumber"]}: already exists in DB" + next + end + + date = Time.zone.at(trade["createTime"].to_i / 1000).to_date + trade_type = trade["tradeType"] # BUY or SELL + + begin + # Grab the exact Fiat and Crypto truth straight from the payload + fiat_currency = trade["fiat"] + fiat_amount = trade["totalPrice"].to_d + fiat_price = trade["unitPrice"].to_d + + crypto_asset = trade["asset"] + gross_crypto = trade["amount"].to_d + net_crypto = (trade["takerAmount"] || gross_crypto).to_d + crypto_fee = (trade["takerCommission"] || 0).to_d + + ticker = "CRYPTO:#{crypto_asset}" + security = BinanceAccount::SecurityResolver.resolve(ticker, crypto_asset) + + unless security + Rails.logger.warn "BinanceAccount::Processor - skipping P2P trade #{trade["orderNumber"]}: could not resolve security for #{crypto_asset}" + next + end + + # Convert the crypto fee (if any) to its fiat equivalent using the trade's exact unit price + fiat_fee = (crypto_fee * fiat_price).round(2) + + # 3. AI Fix: Wrap the double-entry in a transaction block to guarantee ledger integrity + account.transaction do + if trade_type == "BUY" + # BUY LOGIC: User sent Fiat from their bank, received Crypto + account.entries.create!( + date: date, + name: "P2P Payment (#{fiat_currency})", + amount: -fiat_amount, # Fiat leaving the system + currency: fiat_currency, + external_id: funding_external_id, + source: "binance", + entryable: Transaction.new + ) + + account.entries.create!( + date: date, + name: "P2P Buy #{gross_crypto.round(8)} #{crypto_asset}", + amount: fiat_amount, # Fiat value entering as Crypto (Cost Basis) + currency: fiat_currency, + external_id: external_id, + source: "binance", + entryable: Trade.new( + security: security, + qty: net_crypto, + price: fiat_price, + currency: fiat_currency, + fee: fiat_fee, + investment_activity_label: "Buy" + ) + ) + else + # SELL LOGIC: User liquidated Crypto, received Fiat to their bank + account.entries.create!( + date: date, + name: "P2P Sell #{gross_crypto.round(8)} #{crypto_asset}", + amount: -fiat_amount, # Fiat value of Crypto leaving + currency: fiat_currency, + external_id: external_id, + source: "binance", + entryable: Trade.new( + security: security, + qty: -net_crypto, + price: fiat_price, + currency: fiat_currency, + fee: fiat_fee, + investment_activity_label: "Sell" + ) + ) + + account.entries.create!( + date: date, + name: "P2P Receipt (#{fiat_currency})", + amount: fiat_amount, # Fiat entering the system + currency: fiat_currency, + external_id: funding_external_id, + source: "binance", + entryable: Transaction.new + ) + end + end + rescue => e + Rails.logger.error "BINANCE P2P SYNC CRASHED for Order #{trade["orderNumber"]}: #{e.message}" + raise + end + end + rescue StandardError => e + Rails.logger.error "BinanceAccount::Processor - P2P trade processing failed: #{e.message}" + raise + end + def commission_in_usd(trade, base_symbol, trade_price, date: nil) raw = trade["commission"].to_d commission_asset = trade["commissionAsset"].to_s.upcase diff --git a/app/models/binance_item/futures_importer.rb b/app/models/binance_item/futures_importer.rb new file mode 100644 index 000000000..8371d377d --- /dev/null +++ b/app/models/binance_item/futures_importer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Pulls USDⓈ-M futures account data (balance and positions). +# Returns normalized asset list with source tag "futures". +class BinanceItem::FuturesImporter + attr_reader :binance_item, :provider + + def initialize(binance_item, provider:) + @binance_item = binance_item + @provider = provider + end + + # @return [Hash] { assets: [...], raw: , source: "futures" } + def import + raw = provider.get_futures_account + + # Binance Futures returns a slightly different format than spot + # assets are in raw["assets"], positions in raw["positions"] + + assets = [] + + # Process base assets (e.g. USDT, BUSD balances) + Array(raw["assets"]).each do |asset| + wallet_balance = asset["walletBalance"].to_d + unrealized_profit = asset["unrealizedProfit"].to_d + + # Total equity is wallet balance + unrealized PNL + total = wallet_balance + unrealized_profit + + next if total.zero? + + assets << { + symbol: asset["asset"], + free: asset["availableBalance"] || wallet_balance.to_s, + locked: (wallet_balance - (asset["availableBalance"] || wallet_balance.to_s).to_d).to_s, + total: total.to_s + } + end + + { assets: assets, raw: raw, source: "futures" } + rescue => e + Rails.logger.error "BinanceItem::FuturesImporter #{binance_item.id} - #{e.message}" + { assets: [], raw: nil, source: "futures", error: e.message } + end +end diff --git a/app/models/binance_item/importer.rb b/app/models/binance_item/importer.rb index 7d499db70..1987022d2 100644 --- a/app/models/binance_item/importer.rb +++ b/app/models/binance_item/importer.rb @@ -15,8 +15,9 @@ class BinanceItem::Importer spot_result = BinanceItem::SpotImporter.new(binance_item, provider: binance_provider).import margin_result = BinanceItem::MarginImporter.new(binance_item, provider: binance_provider).import earn_result = BinanceItem::EarnImporter.new(binance_item, provider: binance_provider).import + futures_result = BinanceItem::FuturesImporter.new(binance_item, provider: binance_provider).import - all_assets = tagged_assets(spot_result) + tagged_assets(margin_result) + tagged_assets(earn_result) + all_assets = tagged_assets(spot_result) + tagged_assets(margin_result) + tagged_assets(earn_result) + tagged_assets(futures_result) return { success: true, assets_imported: 0, total_usd: 0 } if all_assets.empty? @@ -27,13 +28,15 @@ class BinanceItem::Importer total_usd: total_usd, spot_raw: spot_result[:raw], margin_raw: margin_result[:raw], - earn_raw: earn_result[:raw] + earn_raw: earn_result[:raw], + futures_raw: futures_result[:raw] ) binance_item.upsert_binance_snapshot!({ "spot" => spot_result[:raw], "margin" => margin_result[:raw], "earn" => earn_result[:raw], + "futures" => futures_result[:raw], "imported_at" => Time.current.iso8601 }) @@ -68,7 +71,7 @@ class BinanceItem::Importer 0 end - def upsert_binance_account(all_assets:, total_usd:, spot_raw:, margin_raw:, earn_raw:) + def upsert_binance_account(all_assets:, total_usd:, spot_raw:, margin_raw:, earn_raw:, futures_raw:) ba = binance_item.binance_accounts.find_or_initialize_by(account_type: "combined") ba.assign_attributes( @@ -80,6 +83,7 @@ class BinanceItem::Importer "spot" => spot_raw, "margin" => margin_raw, "earn" => earn_raw, + "futures" => futures_raw, "assets" => all_assets.map(&:stringify_keys), "fetched_at" => Time.current.iso8601 } @@ -90,7 +94,7 @@ class BinanceItem::Importer end def build_institution_metadata(all_assets) - %w[spot margin earn].each_with_object({}) do |source, hash| + %w[spot margin earn futures].each_with_object({}) do |source, hash| source_assets = all_assets.select { |a| a[:source] == source } hash[source] = { "asset_count" => source_assets.size, diff --git a/app/models/budget.rb b/app/models/budget.rb index f08f1e0d0..7e673b3fa 100644 --- a/app/models/budget.rb +++ b/app/models/budget.rb @@ -31,27 +31,24 @@ class Budget < ApplicationRecord end def budget_date_valid?(date, family:) - budget_start = if family.uses_custom_month_start? - family.custom_month_start_for(date) - else - date.beginning_of_month - end - + budget_start, _ = period_for(date, family: family) budget_start >= oldest_valid_budget_date(family) && budget_start <= latest_valid_budget_start_date(family) end + def period_for(date, family:) + if family.uses_custom_month_start? + [ family.custom_month_start_for(date), family.custom_month_end_for(date) ] + else + [ date.beginning_of_month, date.end_of_month ] + end + end + def find_or_bootstrap(family, start_date:, user: nil) return nil unless budget_date_valid?(start_date, family: family) Budget.transaction do - if family.uses_custom_month_start? - budget_start = family.custom_month_start_for(start_date) - budget_end = family.custom_month_end_for(start_date) - else - budget_start = start_date.beginning_of_month - budget_end = start_date.end_of_month - end + budget_start, budget_end = period_for(start_date, family: family) budget = Budget.find_or_create_by!( family: family, diff --git a/app/models/budget_category.rb b/app/models/budget_category.rb index 312db666a..ec4930623 100644 --- a/app/models/budget_category.rb +++ b/app/models/budget_category.rb @@ -198,11 +198,9 @@ class BudgetCategory < ApplicationRecord # Returns hash with suggested daily spending info or nil if not applicable def suggested_daily_spending return nil unless available_to_spend > 0 + return nil unless budget.current? - budget_date = budget.start_date - return nil unless budget_date.month == Date.current.month && budget_date.year == Date.current.year - - days_remaining = (budget_date.end_of_month - Date.current).to_i + 1 + days_remaining = (budget.end_date - Date.current).to_i + 1 return nil unless days_remaining > 0 { diff --git a/app/models/category/merger.rb b/app/models/category/merger.rb new file mode 100644 index 000000000..13176a14e --- /dev/null +++ b/app/models/category/merger.rb @@ -0,0 +1,91 @@ +class Category::Merger + class UnauthorizedCategoryError < StandardError; end + + attr_reader :family, :target_category, :source_categories, :merged_count + + def initialize(family:, target_category:, source_categories:) + @family = family + @target_category = target_category + @merged_count = 0 + + validate_category_belongs_to_family!(target_category, "Target category") + + sources = Array(source_categories) + sources.each { |category| validate_category_belongs_to_family!(category, "Source category '#{category.name}'") } + + @source_categories = sources.reject { |category| category.id == target_category.id } + validate_hierarchy! + validate_reparenting! + end + + def merge! + return false if source_categories.empty? + + Category.transaction { merge_sources! } + true + end + + private + def merge_sources! + source_categories.each do |source| + family.transactions.where(category_id: source.id).update_all(category_id: target_category.id) + merge_budget_categories(source) + family.categories.where(parent_id: source.id).where.not(id: target_category.id).update_all(parent_id: target_category.id) + family.categories.find(source.id).destroy! + @merged_count += 1 + end + end + + def validate_category_belongs_to_family!(category, label) + return if category&.family_id == family.id + + raise UnauthorizedCategoryError, "#{label} does not belong to this family" + end + + def validate_hierarchy! + target_ancestor_ids = ancestor_ids_for(target_category) + return unless source_categories.any? { |source| target_ancestor_ids.include?(source.id) } + + raise UnauthorizedCategoryError, "A parent category cannot be merged into its own subcategory" + end + + def validate_reparenting! + return if target_category.parent_id.blank? + return unless source_categories.any? { |source| family.categories.exists?(parent_id: source.id) } + + raise UnauthorizedCategoryError, "Cannot merge a category with subcategories into a subcategory" + end + + def ancestor_ids_for(category) + ids = [] + seen_ids = Set.new + current = category + + while current&.parent_id.present? && seen_ids.exclude?(current.parent_id) + ids << current.parent_id + seen_ids << current.parent_id + current = family.categories.find_by(id: current.parent_id) + end + + ids + end + + def merge_budget_categories(source) + family.budget_categories.where(category_id: source.id).find_each do |source_budget_category| + target_budget_category = family.budget_categories.find_by( + budget_id: source_budget_category.budget_id, + category_id: target_category.id + ) + + if target_budget_category + target_budget_category.update!( + budgeted_spending: (target_budget_category.budgeted_spending || 0).to_d + + (source_budget_category.budgeted_spending || 0).to_d + ) + source_budget_category.destroy! + else + source_budget_category.update!(category_id: target_category.id) + end + end + end +end diff --git a/app/models/holding/materializer.rb b/app/models/holding/materializer.rb index 5832e2033..689a43ad1 100644 --- a/app/models/holding/materializer.rb +++ b/app/models/holding/materializer.rb @@ -88,8 +88,27 @@ class Holding::Materializer "cost_basis_source" => reconciled[:cost_basis_source] ) else - # No cost_basis to set, or existing is better - don't touch cost_basis fields - holdings_to_upsert_without_cost << base_attrs + # No new calculated value — fall back to the most recent provider + # cost_basis for this security on or before the holding date. + # Calculated/manual values outrank a provider carry-forward. + existing_source = existing&.cost_basis_source + preserve_existing = existing&.cost_basis.present? && %w[calculated manual].include?(existing_source) + + if preserve_existing + holdings_to_upsert_without_cost << base_attrs + else + carried = carry_forward_provider_cost_basis(holding) + + if carried && (existing&.cost_basis != carried || existing_source != "provider") + holdings_to_upsert_with_cost << base_attrs.merge( + "cost_basis" => carried, + "cost_basis_source" => "provider" + ) + else + # No cost_basis to set, or existing is better - don't touch cost_basis fields + holdings_to_upsert_without_cost << base_attrs + end + end end end @@ -165,6 +184,50 @@ class Holding::Materializer [ holding.account_id || account.id, holding.security_id, holding.date, holding.currency ] end + # Returns the most recent provider-supplied cost_basis for the given holding's + # security on or before its date, converted to the holding's currency. + # Used to backfill calculated rows past the provider's last snapshot so + # reports keep showing trend data. + # + # Provider and calculated rows can be denominated in different currencies + # (e.g., IBKR reports USD holdings while the reverse calculator converts to + # the account's base currency). When they differ, the cost_basis is converted + # at the snapshot date — the same convention ReverseCalculator uses for trade + # prices — so the result is consistent with trade-derived cost_basis values. + def carry_forward_provider_cost_basis(holding) + snapshots = provider_cost_basis_snapshots[holding.security_id] + return nil if snapshots.blank? + + result = nil + snapshots.each do |snap_date, cost_basis, snap_currency| + break if snap_date > holding.date + result = [ cost_basis, snap_currency, snap_date ] + end + return nil unless result + + cost_basis, snap_currency, snap_date = result + return cost_basis if snap_currency == holding.currency + + Money.new(cost_basis, snap_currency).exchange_to(holding.currency, date: snap_date).amount + rescue Money::ConversionError + nil + end + + def provider_cost_basis_snapshots + @provider_cost_basis_snapshots ||= begin + ids = @holdings.map(&:security_id).uniq + account.holdings + .where.not(account_provider_id: nil) + .where.not(cost_basis: nil) + .where(security_id: ids) + .order(:date) # ascending required: carry_forward_provider_cost_basis scans and breaks on snap_date > holding.date + .pluck(:security_id, :currency, :date, :cost_basis) + .each_with_object(Hash.new { |h, k| h[k] = [] }) do |(security_id, currency, date, cost_basis), memo| + memo[security_id] << [ date, cost_basis, currency ] + end + end + end + def purge_stale_holdings portfolio_security_ids = account.trades.distinct.pluck(:security_id) diff --git a/app/models/import.rb b/app/models/import.rb index 71887faed..d70a19eb2 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -40,6 +40,7 @@ class Import < ApplicationRecord belongs_to :family belongs_to :account, optional: true + belongs_to :account_statement, optional: true before_validation :set_default_number_format before_validation :ensure_utf8_encoding diff --git a/app/models/investment_statement.rb b/app/models/investment_statement.rb index e4f79fee1..bb7a836a9 100644 --- a/app/models/investment_statement.rb +++ b/app/models/investment_statement.rb @@ -156,6 +156,77 @@ class InvestmentStatement ) end + def period_return_trend(period: Period.current_month) + currency = family.currency + account_ids = investment_account_ids + return nil if account_ids.empty? + + absolute_return = ActiveRecord::Base.connection.select_value( + ActiveRecord::Base.sanitize_sql_array([ + <<~SQL.squish, + SELECT COALESCE(SUM(b.net_market_flows * COALESCE(er.rate, 1)), 0) + FROM balances b + JOIN accounts a ON a.id = b.account_id + LEFT JOIN exchange_rates er ON ( + er.date = b.date + AND er.from_currency = b.currency + AND er.to_currency = :currency + ) + WHERE a.id IN (:account_ids) + AND a.family_id = :family_id + AND a.status IN ('draft', 'active') + AND b.date BETWEEN :start_date AND :end_date + SQL + { + currency: currency, + account_ids: account_ids, + family_id: family.id, + start_date: period.date_range.begin, + end_date: period.date_range.end + } + ]) + ).to_d + + period_start = period.date_range.begin + + # Single query for all accounts' most recent pre-period balance (strict < to avoid + # double-counting the first day's net_market_flows in both the denominator and absolute_return). + # FX conversion is done in SQL (matching absolute_return) so balance rows whose currency + # differs from the account's current currency (e.g. after a currency change) are still picked up. + start_value = ActiveRecord::Base.connection.select_value( + ActiveRecord::Base.sanitize_sql_array([ + <<~SQL.squish, + SELECT COALESCE(SUM(b.end_balance * COALESCE(er.rate, 1)), 0) + FROM accounts a + INNER JOIN balances b ON b.account_id = a.id + LEFT JOIN exchange_rates er ON ( + er.date = :period_start + AND er.from_currency = b.currency + AND er.to_currency = :currency + ) + INNER JOIN ( + SELECT b2.account_id, MAX(b2.date) AS max_date + FROM balances b2 + WHERE b2.account_id IN (:account_ids) + AND b2.date < :period_start + GROUP BY b2.account_id + ) latest ON latest.account_id = b.account_id AND b.date = latest.max_date + WHERE a.id IN (:account_ids) + AND a.family_id = :family_id + AND a.status IN ('draft', 'active') + SQL + { account_ids: account_ids, period_start: period_start, family_id: family.id, currency: currency } + ]) + ).to_d + + return nil if start_value.zero? + + Trend.new( + current: Money.new(start_value + absolute_return, currency), + previous: Money.new(start_value, currency) + ) + end + # Day change across portfolio, summed in family currency def day_change changes = current_holdings.to_a.filter_map do |h| diff --git a/app/models/invitation.rb b/app/models/invitation.rb index 99ecb0da7..a7b391288 100644 --- a/app/models/invitation.rb +++ b/app/models/invitation.rb @@ -32,6 +32,7 @@ class Invitation < ApplicationRecord return false if user.blank? return false unless pending? return false unless emails_match?(user) + return false if would_orphan_owned_accounts?(user) transaction do user.update!(family_id: family_id, role: role.to_s) @@ -41,6 +42,14 @@ class Invitation < ApplicationRecord true end + def would_orphan_owned_accounts?(user) + return false if user.blank? + return false if user.family_id.blank? + return false if user.family_id == family_id + + user.owned_accounts.where.not(family_id: family_id).exists? + end + private def emails_match?(user) diff --git a/app/models/pdf_import.rb b/app/models/pdf_import.rb index f610e6b0f..1e1f494ef 100644 --- a/app/models/pdf_import.rb +++ b/app/models/pdf_import.rb @@ -2,6 +2,32 @@ class PdfImport < Import has_one_attached :pdf_file, dependent: :purge_later validates :document_type, inclusion: { in: DOCUMENT_TYPES }, allow_nil: true + validate :account_statement_matches_import + + class << self + def create_from_upload!(family:, file:, user:) + statement = AccountStatement.create_from_prepared_upload!( + family: family, + account: nil, + prepared_upload: AccountStatement.prepare_upload!(file) + ) + + create_from_statement!(statement: statement) + rescue AccountStatement::DuplicateUploadError => e + raise unless e.statement.manageable_by?(user) + + create_from_statement!(statement: e.statement) + end + + def create_from_statement!(statement:) + reusable_import = statement.latest_reusable_pdf_import + return reusable_import if reusable_import && + reusable_import.account_id == statement.account_id && + reusable_import.date_format == statement.family.date_format + + create!(family: statement.family, account: statement.account, account_statement: statement, date_format: statement.family.date_format, status: :pending) + end + end def import! raise "Account required for PDF import" unless account.present? @@ -31,8 +57,18 @@ class PdfImport < Import end end + def assign_account!(account) + transaction do + update!(account: account) + if (statement = account_statement) + statement.lock! + statement.link_to_account!(account) if statement.account_id != account.id + end + end + end + def pdf_uploaded? - pdf_file.attached? + statement_backed? || pdf_file.attached? end def ai_processed? @@ -40,7 +76,16 @@ class PdfImport < Import end def process_with_ai_later - ProcessPdfJob.perform_later(self) + return false unless with_lock { pending? && !ai_processed? && rows_count.zero? && pdf_uploaded? && update!(status: :importing) } + + begin + ProcessPdfJob.perform_later(self) + true + rescue StandardError => e + Rails.logger.error("Failed to enqueue PDF processing for import #{id}: #{e.class.name} - #{e.message}") + reload.with_lock { update!(status: :pending) } + false + end end def process_with_ai @@ -172,9 +217,20 @@ class PdfImport < Import end def pdf_file_content - return nil unless pdf_file.attached? + return @pdf_file_content if defined?(@pdf_file_content) + return @pdf_file_content = account_statement.original_file.download if statement_backed? - pdf_file.download + @pdf_file_content = pdf_file.download if pdf_file.attached? + end + + def pdf_filename + return account_statement.filename if statement_backed? + + pdf_file.filename.to_s if pdf_file.attached? + end + + def statement_backed? + account_statement&.original_file&.attached? end def required_column_keys @@ -199,4 +255,10 @@ class PdfImport < Import rescue ArgumentError date_str.to_s end + + def account_statement_matches_import + return if account_statement.blank? || (account_statement.family_id == family_id && account_statement.pdf?) + + errors.add(:account_statement, :invalid) + end end diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index 58e39bae2..deef9039e 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -45,15 +45,24 @@ class PlaidItem < ApplicationRecord access_token: access_token ) rescue Plaid::ApiError => e - error_body = JSON.parse(e.response_body) - - if error_body["error_code"] == "ITEM_NOT_FOUND" - # Mark the connection as invalid but don't auto-delete - update!(status: :requires_update) + error_body = begin + JSON.parse(e.response_body.to_s) + rescue JSON::ParserError + {} end - Sentry.capture_exception(e) - nil + if error_body["error_code"] == "ITEM_NOT_FOUND" + # Mark the connection as invalid but don't auto-delete. The caller + # gets nil so the calling controller can decide what to render. + update!(status: :requires_update) + Sentry.capture_exception(e) if defined?(Sentry) + nil + else + # Re-raise so the controller can surface a friendly alert to the user + # (issue #1792). Swallowing here previously left the Plaid modal frame + # blank with no actionable signal. + raise + end end def destroy_later diff --git a/app/models/provider/binance.rb b/app/models/provider/binance.rb index d1b8c018c..683b660fc 100644 --- a/app/models/provider/binance.rb +++ b/app/models/provider/binance.rb @@ -13,6 +13,7 @@ class Provider::Binance # Pipelock incorrectly interprets the '@' in Ruby instance variables as a password delimiter # in an URL (e.g. https://user:password@host). SPOT_BASE_URL = "https://api.binance.com".freeze + FUTURES_BASE_URL = "https://fapi.binance.com".freeze base_uri SPOT_BASE_URL default_options.merge!({ timeout: 30 }.merge(httparty_ssl_options)) @@ -87,14 +88,63 @@ class Provider::Binance signed_get("/api/v3/myTrades", extra_params: params) end + # USDⓈ-M Futures account — requires signed request + def get_futures_account + signed_get("/fapi/v2/account", base_url: FUTURES_BASE_URL) + end + + # Futures trade history for a single symbol + def get_futures_trades(symbol, limit: 1000, from_id: nil) + params = { "symbol" => symbol, "limit" => limit.to_s } + params["fromId"] = from_id.to_s if from_id + signed_get("/fapi/v1/userTrades", extra_params: params, base_url: FUTURES_BASE_URL) + end + + # P2P trade history — requires signed request + # Pass start_timestamp to fetch only recent trades (max 30 days window) + def get_p2p_trades(start_timestamp: nil, end_timestamp: nil) + params = { "tradeType" => "BUY" } # default to BUY, will loop in processor for SELL + params["startTimestamp"] = start_timestamp.to_s if start_timestamp + params["endTimestamp"] = end_timestamp.to_s if end_timestamp + signed_get("/sapi/v1/c2c/orderMatch/listUserOrderHistory", extra_params: params) + end + + # Internal helper to handle both buy and sell types since API requires specific tradeType or gets default BUY + def get_all_p2p_trades(start_timestamp: nil, end_timestamp: nil) + %w[BUY SELL].flat_map do |trade_type| + page = 1 + rows = 100 + data = [] + loop do + result = signed_get( + "/sapi/v1/c2c/orderMatch/listUserOrderHistory", + extra_params: { + "tradeType" => trade_type, + "startTimestamp" => start_timestamp&.to_s, + "endTimestamp" => end_timestamp&.to_s, + "page" => page.to_s, + "rows" => rows.to_s + }.compact + ) + batch = result.is_a?(Hash) ? Array(result["data"]) : [] + data.concat(batch) + break if batch.size < rows + page += 1 + end + data + end + end + private - def signed_get(path, extra_params: {}) + def signed_get(path, extra_params: {}, base_url: SPOT_BASE_URL) params = timestamp_params.merge(extra_params) query_string = URI.encode_www_form(params.sort) + full_url = "#{base_url}#{path}" + response = self.class.get( - path, + full_url, query: "#{query_string}&signature=#{sign(query_string)}", headers: auth_headers ) diff --git a/app/models/sync.rb b/app/models/sync.rb index e2ec1f28a..83cb439ce 100644 --- a/app/models/sync.rb +++ b/app/models/sync.rb @@ -71,6 +71,14 @@ class Sync < ApplicationRecord query end + # True iff the family has any pending/syncing Sync — across its own row, + # its accounts, and every Syncable provider `*_items` association. Built + # on `for_family` so new provider integrations are picked up automatically + # via `family_syncable_associations` reflection (no hand-rolled list). + def any_incomplete_for?(family) + for_family(family).incomplete.exists? + end + private def account_syncable_ids(family, resource_owner) (resource_owner ? resource_owner.accessible_accounts : family.accounts) diff --git a/app/views/accounts/index.html.erb b/app/views/accounts/index.html.erb index 38ba4d888..e8f4bee52 100644 --- a/app/views/accounts/index.html.erb +++ b/app/views/accounts/index.html.erb @@ -17,7 +17,7 @@ ) %> <% end %> -<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @brex_items.empty? && @ibkr_items.empty? && @snaptrade_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? %> +<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @brex_items.empty? && @ibkr_items.empty? && @snaptrade_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? && @binance_items.empty? %> <%= render "empty" %> <% else %>
@@ -57,6 +57,10 @@ <%= render @coinbase_items.sort_by(&:created_at) %> <% end %> + <% if @binance_items.any? %> + <%= render @binance_items.sort_by(&:created_at) %> + <% end %> + <% if @snaptrade_items.any? %> <%= render @snaptrade_items.sort_by(&:created_at) %> <% end %> diff --git a/app/views/binance_items/_binance_item.html.erb b/app/views/binance_items/_binance_item.html.erb index 5a1374fc7..0fd356f3d 100644 --- a/app/views/binance_items/_binance_item.html.erb +++ b/app/views/binance_items/_binance_item.html.erb @@ -67,7 +67,7 @@ <% end %> <%= render DS::Menu.new do |menu| %> - <% if unlinked_count.to_i > 0 %> + <% if binance_item.unlinked_accounts_count > 0 %> <% menu.with_item( variant: "link", text: t(".import_accounts_menu"), @@ -110,10 +110,10 @@ provider_item: binance_item ) %> - <% if unlinked_count.to_i > 0 && binance_item.accounts.empty? %> -
+ <% if binance_item.unlinked_accounts_count > 0 %> +

<%= t(".setup_needed") %>

-

<%= t(".setup_description") %>

+

<%= t(".setup_description") %>

<%= render DS::Link.new( text: t(".setup_action"), icon: "plus", diff --git a/app/views/binance_items/setup_accounts.html.erb b/app/views/binance_items/setup_accounts.html.erb index 4b7ab0af5..323f2fb70 100644 --- a/app/views/binance_items/setup_accounts.html.erb +++ b/app/views/binance_items/setup_accounts.html.erb @@ -33,6 +33,18 @@
+
+

<%= t(".historical_import") %>

+
+ <%= form.label :sync_start_date, t("settings.providers.binance_panel.sync_start_date_label"), class: "label" %> + <%= form.date_field :sync_start_date, + value: @binance_item.sync_start_date || (Date.current - 1.year), + max: Date.current, + class: "input" %> +

<%= t("settings.providers.binance_panel.sync_start_date_help") %>

+
+
+ <% if @binance_accounts.empty? %>

<%= t(".no_accounts") %>

@@ -69,7 +81,7 @@ <%= binance_account.currency %>

-
+

<%= number_with_delimiter(binance_account.current_balance || 0, delimiter: ",") %>

diff --git a/app/views/categories/_badge.html.erb b/app/views/categories/_badge.html.erb index 5570f2c70..5ab8d6c1d 100644 --- a/app/views/categories/_badge.html.erb +++ b/app/views/categories/_badge.html.erb @@ -1,7 +1,7 @@ <%# locals: (category:) %> <% category ||= Category.uncategorized %> -
+
<% end %> + <%= render DS::Link.new( + text: t(".merge"), + variant: "outline", + icon: "combine", + href: merge_categories_path, + frame: :modal + ) %> + <%= render DS::Link.new( text: t(".new"), variant: "primary", diff --git a/app/views/categories/merge.html.erb b/app/views/categories/merge.html.erb new file mode 100644 index 000000000..6d8e83300 --- /dev/null +++ b/app/views/categories/merge.html.erb @@ -0,0 +1,30 @@ +<%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title"), subtitle: t(".description")) %> + <% dialog.with_body do %> + <%= styled_form_with url: perform_merge_categories_path, method: :post, class: "space-y-4" do |f| %> + <%= f.collection_select :target_id, + @categories, + :id, :name_with_parent, + { prompt: t(".select_target"), label: t(".target_label") }, + { required: true } %> + +
+

<%= t(".sources_label") %>

+
+ <% @categories.each do |category| %> + + <% end %> +
+

<%= t(".sources_hint") %>

+
+ + <%= render DS::Button.new( + text: t(".submit"), + full_width: true + ) %> + <% end %> + <% end %> +<% end %> diff --git a/app/views/enable_banking_items/select_bank.html.erb b/app/views/enable_banking_items/select_bank.html.erb index 095ae6738..73010080d 100644 --- a/app/views/enable_banking_items/select_bank.html.erb +++ b/app/views/enable_banking_items/select_bank.html.erb @@ -45,9 +45,7 @@

<%= aspsp[:name] %>

<% if aspsp[:beta] %> - - <%= t(".beta_label", default: "Beta") %> - + <%= render DS::Pill.new(label: t(".beta_label", default: "Beta"), tone: :warning, marker: false) %> <% end %>
<% if aspsp[:bic].present? %> @@ -70,9 +68,12 @@ <% end %>
- <%= link_to t(".cancel", default: "Cancel"), settings_providers_path, - class: "inline-flex items-center gap-1 px-3 py-2 text-sm font-medium rounded-lg text-primary button-bg-secondary hover:button-bg-secondary-hover", - data: { turbo_frame: "_top", action: "DS--dialog#close" } %> + <%= render DS::Link.new( + text: t(".cancel", default: "Cancel"), + variant: :secondary, + href: settings_providers_path, + data: { turbo_frame: "_top", action: "DS--dialog#close" } + ) %>
<% end %> diff --git a/app/views/imports/_pdf_import.html.erb b/app/views/imports/_pdf_import.html.erb index 068d55da9..4d82ce324 100644 --- a/app/views/imports/_pdf_import.html.erb +++ b/app/views/imports/_pdf_import.html.erb @@ -14,16 +14,25 @@
+ <% if import.account_statement.present? %> +
+

<%= t("imports.pdf_import.source_statement") %>

+

+ <%= link_to import.account_statement.filename, account_statement_path(import.account_statement), class: "text-primary hover:text-primary-hover" %> +

+
+ <% end %> +

<%= t("imports.pdf_import.document_type_label") %>

-

+

<%= t("imports.document_types.#{import.document_type}", default: import.document_type&.humanize || t("imports.pdf_import.unknown_document_type", default: "Unknown")) %>

<%= t("imports.pdf_import.transactions_extracted", default: "Transactions Extracted") %>

-

+

<%= t("imports.pdf_import.transactions_extracted_count", count: import.rows_count, default: "%{count} transactions") %>

@@ -38,7 +47,7 @@

<%= t("imports.pdf_import.select_account_hint", default: "Choose which account to import these transactions into.") %>

<% end %> <% else %> -

+

<%= t("imports.pdf_import.no_accounts", default: "No accounts available. Please create an account first.") %>

<%= render DS::Link.new(text: t("imports.pdf_import.create_account", default: "Create Account"), href: new_account_path(return_to: import_path(import)), variant: "primary", full_width: true, frame: :modal) %> @@ -106,16 +115,25 @@
+ <% if import.account_statement.present? %> +
+

<%= t("imports.pdf_import.source_statement") %>

+

+ <%= link_to import.account_statement.filename, account_statement_path(import.account_statement), class: "text-primary hover:text-primary-hover" %> +

+
+ <% end %> +

<%= t("imports.pdf_import.document_type_label") %>

-

+

<%= t("imports.document_types.#{import.document_type}", default: import.document_type&.humanize || t("imports.pdf_import.unknown_document_type", default: "Unknown")) %>

<%= t("imports.pdf_import.summary_label") %>

-

+

<%= import.ai_summary %>

diff --git a/app/views/messages/_chat_form.html.erb b/app/views/messages/_chat_form.html.erb index 1fbc57c82..92d401e25 100644 --- a/app/views/messages/_chat_form.html.erb +++ b/app/views/messages/_chat_form.html.erb @@ -23,7 +23,8 @@ <% end %>
- <%= icon("arrow-up", as_button: true, type: "submit") %> + <%= icon("arrow-up", as_button: true, type: "submit", + data: { chat_target: "submit" }) %>
<% end %> diff --git a/app/views/reports/_investment_performance.html.erb b/app/views/reports/_investment_performance.html.erb index 23a922ebc..96f9fc8e6 100644 --- a/app/views/reports/_investment_performance.html.erb +++ b/app/views/reports/_investment_performance.html.erb @@ -3,7 +3,7 @@ <% if investment_metrics[:has_investments] %>
<%# Investment Summary Cards %> -
+
<%# Portfolio Value Card %>
@@ -31,6 +31,22 @@ <% end %>
+ <%# Period Return Card %> +
+
+ <%= icon("bar-chart-2", size: "sm") %> + <%= t("reports.investment_performance.period_return") %> +
+ <% if investment_metrics[:period_return_trend] %> +

+ <%= format_money(Money.new(investment_metrics[:period_return_trend].value, Current.family.currency)) %> + (<%= investment_metrics[:period_return_trend].percent_formatted %>) +

+ <% else %> +

<%= t("reports.investment_performance.no_data") %>

+ <% end %> +
+ <%# Period Contributions Card %>
diff --git a/app/views/reports/print.html.erb b/app/views/reports/print.html.erb index 1de2a4cd2..d92698ab4 100644 --- a/app/views/reports/print.html.erb +++ b/app/views/reports/print.html.erb @@ -210,6 +210,17 @@
<% end %> + <% if @investment_metrics[:period_return_trend] %> +
+ <%= t("reports.print.investments.period_return") %> + + <%= @investment_metrics[:period_return_trend].value >= 0 ? "+" : "" %><%= format_money(Money.new(@investment_metrics[:period_return_trend].value, Current.family.currency)) %> + + + <%= @investment_metrics[:period_return_trend].percent_formatted %> + +
+ <% end %>
<%= t("reports.print.investments.contributions") %> <%= format_money(@investment_metrics[:period_contributions]) %> diff --git a/app/views/settings/_settings_nav.html.erb b/app/views/settings/_settings_nav.html.erb index f9e5e975b..71cf9ff42 100644 --- a/app/views/settings/_settings_nav.html.erb +++ b/app/views/settings/_settings_nav.html.erb @@ -9,7 +9,7 @@ nav_sections = [ { label: t(".appearance_label"), path: settings_appearance_path, icon: "palette" }, { label: t(".profile_label"), path: settings_profile_path, icon: "circle-user" }, { label: t(".security_label"), path: settings_security_path, icon: "shield-check" }, - { label: t(".payment_label"), path: settings_payment_path, icon: "circle-dollar-sign", if: !self_hosted? && Current.family.can_manage_subscription? } + { label: t(".payment_label"), path: settings_payment_path, icon: "circle-dollar-sign", if: !self_hosted? && Current.family&.can_manage_subscription? } ] }, { diff --git a/app/views/settings/providers/_binance_panel.html.erb b/app/views/settings/providers/_binance_panel.html.erb index 578573188..1b329f160 100644 --- a/app/views/settings/providers/_binance_panel.html.erb +++ b/app/views/settings/providers/_binance_panel.html.erb @@ -54,6 +54,15 @@
+ <% if item.unlinked_accounts_count > 0 %> + <%= render DS::Link.new( + text: t("binance_items.binance_item.setup_action"), + icon: "plus", + variant: "primary", + href: setup_accounts_binance_item_path(item), + frame: :modal + ) %> + <% end %> <%= button_to sync_binance_item_path(item), method: :post, class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-secondary hover:text-primary border border-secondary rounded-lg hover:border-primary", diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index 8bccd8b4e..9aff39d03 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -180,8 +180,8 @@