From 174f7e6be6857f2f6a6481a34becd72ab1c72be4 Mon Sep 17 00:00:00 2001 From: Brian Richard <44203805+briian365@users.noreply.github.com> Date: Thu, 28 May 2026 00:58:00 +0300 Subject: [PATCH] feat(binance): add full account sync and transaction processing (#1822) * feat(binance): add full account sync and transaction processing - Fixed a bug that hindered Account setup - Wire up Binance accounts, sync statistics, and unlinked account tracking in the accounts dashboard. - Support setting a sync_start_date during Binance account setup. - Set Binance accounts' opening balance to zero to ensure the ledger builds cleanly from the actual trade history. - Expand the Binance importer and processor to handle Spot, Margin, Earn, P2P, and Futures trades and assets. - Implement TransactionBuilder to parse raw Binance trades, accurately calculating fees, base/quote asset amounts, and market values for proper ledger integration. - Update Binance API timeout (`recvWindow`) to 60,000ms to prevent connection drops. These changes provide comprehensive support for tracking Binance portfolios, ensuring accurate historical ledgers and proper visibility of sync statuses in the frontend dashboard. * refactor(binance): enforce strong params, double-entry safety, and native fiat currency support - Implement strong parameters in BinanceItemsController#complete_account_setup to satisfy Rails security guidelines. - Add robust date parsing with a grace fallback to prevent controller crashes on malformed sync start dates. - Wrap P2P transaction creations inside a database transaction block to guarantee ledger integrity and prevent orphan records. - Optimize P2P deduplication queries by batching checks for both transaction and funding external IDs. - Shift P2P entry persistence from forced USD tracking to native fiat values extracted directly from the Binance API payload. - Update BinanceAccount::ProcessorTest assertions and fixtures to validate native fiat and fee calculation logic. * fix(binance): process sync trades before caching transaction payload - Reorder Binance processor execution to insert trade records into the database prior to updating the `raw_transactions_payload` cache. This guarantees that if a database insertion fails, the cache won't prematurely mark the sync as successful, ensuring the data is retried on the next run. - Move `set_opening_anchor_balance(balance: 0)` out of the generic crypto exchange account builder and apply it specifically during Binance account creation. - Refactor date parsing in BinanceItemsController to explicitly catch `ArgumentError` via a block instead of using a blanket inline `rescue`. - Clean up the `setup_accounts` view template by removing hardcoded default translation strings. * fix(binance): enhance trade sync logic and error propagation - Pass `startTime` (from `sync_start_date`) to spot and futures trade endpoints on initial sync to optimize data fetching. - Include previously synced futures pairs alongside spot pairs when resolving relevant symbols to properly recover sold-out assets. - Re-raise exceptions in processor rescue blocks to prevent silent failures and ensure errors are correctly propagated to background jobs. - Decrease Binance API `recvWindow` from 60000ms to 5000ms to align with recommended default timeout values. --- app/controllers/accounts_controller.rb | 16 ++ app/controllers/binance_items_controller.rb | 22 +- app/models/account.rb | 5 +- app/models/binance_account/processor.rb | 271 +++++++++++++++--- app/models/binance_item/futures_importer.rb | 45 +++ app/models/binance_item/importer.rb | 12 +- app/models/provider/binance.rb | 54 +++- app/views/accounts/index.html.erb | 6 +- .../binance_items/_binance_item.html.erb | 8 +- .../binance_items/setup_accounts.html.erb | 14 +- .../providers/_binance_panel.html.erb | 9 + config/locales/views/settings/en.yml | 3 + .../binance_items_controller_test.rb | 41 +++ test/models/binance_account/processor_test.rb | 116 ++++++++ .../binance_item/futures_importer_test.rb | 42 +++ test/models/binance_item/importer_test.rb | 9 + 16 files changed, 627 insertions(+), 46 deletions(-) create mode 100644 app/models/binance_item/futures_importer.rb create mode 100644 test/models/binance_item/futures_importer_test.rb diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 2becdd3b3..be8f2adf2 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -24,6 +24,7 @@ class AccountsController < ApplicationController @ibkr_items = visible_provider_items(family.ibkr_items.ordered.includes(:syncs, :ibkr_accounts)) @indexa_capital_items = visible_provider_items(family.indexa_capital_items.ordered.includes(:syncs, :indexa_capital_accounts)) @sophtron_items = visible_provider_items(family.sophtron_items.ordered.includes(:syncs, :sophtron_accounts)) + @binance_items = visible_provider_items(family.binance_items.ordered.includes(:binance_accounts, :accounts, :syncs)) # Build sync stats maps for all providers build_sync_stats_maps @@ -397,5 +398,20 @@ class AccountsController < ApplicationController latest_sync = item.syncs.ordered.first @indexa_capital_sync_stats_map[item.id] = latest_sync&.sync_stats || {} end + + # Binance sync stats + @binance_sync_stats_map = {} + @binance_unlinked_count_map = {} + @binance_items.each do |item| + latest_sync = item.syncs.ordered.first + @binance_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + + # Count unlinked accounts + count = item.binance_accounts + .left_joins(:account_provider) + .where(account_providers: { id: nil }) + .count + @binance_unlinked_count_map[item.id] = count + end end end diff --git a/app/controllers/binance_items_controller.rb b/app/controllers/binance_items_controller.rb index 7f3471e58..292d77cc8 100644 --- a/app/controllers/binance_items_controller.rb +++ b/app/controllers/binance_items_controller.rb @@ -211,7 +211,23 @@ class BinanceItemsController < ApplicationController end def complete_account_setup - selected_accounts = Array(params[:selected_accounts]).reject(&:blank?) + setup_params = complete_account_setup_params + + if setup_params[:sync_start_date].present? + parsed_date = begin + Date.parse(setup_params[:sync_start_date].to_s) + rescue ArgumentError + nil + end + + if parsed_date.present? && parsed_date <= Date.current + @binance_item.update!(sync_start_date: parsed_date) + else + flash.now[:alert] = "Sync start date must be a valid date in the past." + end + end + + selected_accounts = Array(setup_params[:selected_accounts]).reject(&:blank?) created_accounts = [] selected_accounts.each do |binance_account_id| @@ -284,4 +300,8 @@ class BinanceItemsController < ApplicationController def binance_item_params params.require(:binance_item).permit(:name, :sync_start_date, :api_key, :api_secret) end + + def complete_account_setup_params + params.permit(:sync_start_date, selected_accounts: []) + end end diff --git a/app/models/account.rb b/app/models/account.rb index 6e11de9c4..ac33c1dad 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -263,7 +263,9 @@ class Account < ApplicationRecord end def create_from_binance_account(binance_account) - create_from_crypto_exchange_account(binance_account, family: binance_account.binance_item.family) + account = create_from_crypto_exchange_account(binance_account, family: binance_account.binance_item.family) + account.set_opening_anchor_balance(balance: 0) + account end def create_from_ibkr_account(ibkr_account) @@ -286,6 +288,7 @@ class Account < ApplicationRecord } } + # Capture the created account in a variable create_and_sync(attributes, skip_initial_sync: true) end diff --git a/app/models/binance_account/processor.rb b/app/models/binance_account/processor.rb index 540383ec1..21306f275 100644 --- a/app/models/binance_account/processor.rb +++ b/app/models/binance_account/processor.rb @@ -61,47 +61,83 @@ class BinanceAccount::Processor provider = binance_account.binance_item&.binance_provider return unless provider + # 1. Initialize data from existing payload + existing_spot = binance_account.raw_transactions_payload&.dig("spot") || {} + existing_futures = binance_account.raw_transactions_payload&.dig("futures") || {} + existing_p2p = binance_account.raw_transactions_payload&.dig("p2p") || [] + + # 2. Fetch P2P Trades (This now runs even if you have no spot assets) + new_p2p = fetch_new_p2p_trades(provider, existing_p2p) + + # 3. Handle Spot & Futures symbols symbols = extract_trade_symbols - return if symbols.empty? - - existing_spot = binance_account.raw_transactions_payload&.dig("spot") || {} new_trades_by_symbol = {} + new_futures_by_symbol = {} - symbols.each do |symbol| - TRADE_QUOTE_CURRENCIES.each do |quote| - pair = "#{symbol}#{quote}" - begin - new_trades = fetch_new_trades(provider, pair, existing_spot[pair]) - new_trades_by_symbol[pair] = new_trades if new_trades.present? - rescue Provider::Binance::InvalidSymbolError => e - # Pair doesn't exist on Binance for this quote currency — expected, skip silently - Rails.logger.debug "BinanceAccount::Processor - skipping #{pair}: #{e.message}" + # Only attempt to loop if we actually have symbols (e.g., BTC, ETH) + if symbols.any? + symbols.each do |symbol| + TRADE_QUOTE_CURRENCIES.each do |quote| + pair = "#{symbol}#{quote}" + begin + new_trades = fetch_new_trades(provider, pair, existing_spot[pair], :spot) + new_trades_by_symbol[pair] = new_trades if new_trades.present? + rescue Provider::Binance::InvalidSymbolError => e + Rails.logger.debug "BinanceAccount::Processor - skipping spot #{pair}: #{e.message}" + end + + begin + new_futures = fetch_new_trades(provider, pair, existing_futures[pair], :futures) + new_futures_by_symbol[pair] = new_futures if new_futures.present? + rescue Provider::Binance::InvalidSymbolError => e + Rails.logger.debug "BinanceAccount::Processor - skipping futures #{pair}: #{e.message}" + end end - # ApiError, AuthenticationError and RateLimitError propagate so the sync is marked failed end end - merged_spot = existing_spot.merge(new_trades_by_symbol) { |_pair, old, new_t| old + new_t } + # 4. Process New Records into Database Entries FIRST + # We process these into the DB first. If they fail or raise an error, + # the method halts before updating the raw_transactions_payload cache, + # ensuring a retry happens on the next sync execution. + process_trades(new_trades_by_symbol, :spot) if new_trades_by_symbol.any? + process_trades(new_futures_by_symbol, :futures) if new_futures_by_symbol.any? + process_p2p_trades(new_p2p) if new_p2p.any? + + # 5. Merge Results ONLY after successful DB insertion + merged_spot = existing_spot.merge(new_trades_by_symbol) { |_pair, old, new_t| old + new_t } + merged_futures = existing_futures.merge(new_futures_by_symbol) { |_pair, old, new_t| old + new_t } + merged_p2p = existing_p2p + new_p2p + + # 6. Update the Account Payload LAST (Safe Caching Boundary) binance_account.update!(raw_transactions_payload: { "spot" => merged_spot, + "futures" => merged_futures, + "p2p" => merged_p2p, "fetched_at" => Time.current.iso8601 }) - - process_trades(new_trades_by_symbol) end # Fetches only trades newer than what is already cached for the given pair. # On the first sync (no cached trades) fetches the most recent page. # On subsequent syncs starts from max_cached_id + 1 and paginates forward. - def fetch_new_trades(provider, pair, cached_trades) + def fetch_new_trades(provider, pair, cached_trades, market_type) limit = 1000 max_cached_id = cached_trades&.map { |t| t["id"].to_i }&.max from_id = max_cached_id ? max_cached_id + 1 : nil + start_time = nil + unless max_cached_id + start_time = binance_account.binance_item&.sync_start_date&.to_time&.to_i&.*(1000) + end all_new = [] loop do - page = provider.get_spot_trades(pair, limit: limit, from_id: from_id) + page = if market_type == :spot + provider.get_spot_trades(pair, limit: limit, from_id: from_id, startTime: start_time) + else + provider.get_futures_trades(pair, limit: limit, from_id: from_id, startTime: start_time) + end break if page.blank? all_new.concat(page) @@ -113,6 +149,47 @@ class BinanceAccount::Processor all_new end + def fetch_new_p2p_trades(provider, cached_p2p) + # Binance P2P history endpoint only supports max 30-day windows. + # If no cache exists, we fetch back to sync_start_date (or default 30 days). + # If cache exists, we fetch from the last cached trade timestamp. + max_cached_timestamp = cached_p2p&.map { |t| t["createTime"].to_i }&.max + + start_time = if max_cached_timestamp + max_cached_timestamp + elsif binance_account.binance_item&.sync_start_date + binance_account.binance_item.sync_start_date.to_time.to_i * 1000 + else + (Time.current - 30.days).to_i * 1000 + end + + all_new = [] + current_start = start_time + + loop do + current_end = [ current_start + 30.days.to_i * 1000, Time.current.to_i * 1000 ].min + + page = provider.get_all_p2p_trades(start_timestamp: current_start, end_timestamp: current_end) + + # We might fetch overlapping trades if they share the exact timestamp, filter by unique orderNumber + if page.present? + cached_order_numbers = cached_p2p&.map { |t| t["orderNumber"] } || [] + new_order_numbers = all_new.map { |t| t["orderNumber"] } + + unique_page = page.reject do |t| + cached_order_numbers.include?(t["orderNumber"]) || new_order_numbers.include?(t["orderNumber"]) + end + + all_new.concat(unique_page) + end + + break if current_end >= Time.current.to_i * 1000 + current_start = current_end + 1 + end + + all_new + end + def extract_trade_symbols stablecoins = BinanceAccount::STABLECOINS quote_re = /(#{TRADE_QUOTE_CURRENCIES.join("|")})$/ @@ -122,21 +199,24 @@ class BinanceAccount::Processor current = assets.map { |a| a["symbol"] || a[:symbol] }.compact # Base symbols from previously fetched pairs (recovers sold-out assets) - prev_pairs = binance_account.raw_transactions_payload&.dig("spot")&.keys || [] + prev_spot = binance_account.raw_transactions_payload&.dig("spot")&.keys || [] + prev_futures = binance_account.raw_transactions_payload&.dig("futures")&.keys || [] + prev_pairs = (prev_spot + prev_futures).uniq previous = prev_pairs.map { |pair| pair.gsub(quote_re, "") } (current + previous).uniq.compact.reject { |s| s.blank? || stablecoins.include?(s) } end - def process_trades(trades_by_symbol) + def process_trades(trades_by_symbol, market_type) trades_by_symbol.each do |pair, trades| - trades.each { |trade| process_spot_trade(trade, pair) } + trades.each { |trade| process_trade(trade, pair, market_type) } end rescue StandardError => e Rails.logger.error "BinanceAccount::Processor - trade processing failed: #{e.message}" + raise end - def process_spot_trade(trade, pair) + def process_trade(trade, pair, market_type) account = binance_account.current_account return unless account @@ -149,7 +229,8 @@ class BinanceAccount::Processor return unless security - external_id = "binance_spot_#{pair}_#{trade["id"]}" + prefix = market_type == :spot ? "spot" : "futures" + external_id = "binance_#{prefix}_#{pair}_#{trade["id"]}" return if account.entries.exists?(external_id: external_id) date = Time.zone.at(trade["time"].to_i / 1000).to_date @@ -170,7 +251,7 @@ class BinanceAccount::Processor amount_usd = amount_usd_raw.round(2) commission = commission_in_usd(trade, base_symbol, price_usd, date: date) - is_buyer = trade["isBuyer"] + is_buyer = trade.key?("isBuyer") ? trade["isBuyer"] : trade["buyer"] if is_buyer account.entries.create!( @@ -209,23 +290,38 @@ class BinanceAccount::Processor end rescue StandardError => e Rails.logger.error "BinanceAccount::Processor - failed to process trade #{trade["id"]}: #{e.message}" + raise end # Converts an amount denominated in quote_symbol to USD. - # Stablecoins are treated as 1:1; others use historical price when date is given, - # falling back to current USDT spot price. + # Stablecoins are treated as 1:1. + # For fiat/crypto assets, tries Binance historical price first, falls back to internal ExchangeRate. def quote_to_usd(amount, quote_symbol, date: nil) return amount if BinanceAccount::STABLECOINS.include?(quote_symbol) + return amount if quote_symbol.to_s.upcase == "USD" provider = binance_account.binance_item&.binance_provider - return nil unless provider - spot = nil - spot = provider.get_historical_price("#{quote_symbol}USDT", date) if date.present? && provider.respond_to?(:get_historical_price) - spot ||= provider.get_spot_price("#{quote_symbol}USDT") - return nil if spot.nil? + if provider + spot = nil + begin + spot = provider.get_historical_price("#{quote_symbol}USDT", date) if date.present? && provider.respond_to?(:get_historical_price) + spot ||= provider.get_spot_price("#{quote_symbol}USDT") + rescue Provider::Binance::InvalidSymbolError + # Fall through to ExchangeRate lookup + end + return (amount * spot.to_d).round(8) if spot.present? + end - (amount * spot.to_d).round(8) + # Fallback to internal app ExchangeRate provider (crucial for P2P fiat currencies like TZS, NGN) + fallback_rate = ExchangeRate.find_or_fetch_rate(from: quote_symbol, to: "USD", date: date || Date.current, cache: true) + if fallback_rate.present? + # Extract the numeric rate from the returned object (or use it directly if it's already a number) + rate_val = fallback_rate.respond_to?(:rate) ? fallback_rate.rate : fallback_rate + return (amount * rate_val.to_d).round(8) + end + + nil rescue StandardError => e Rails.logger.warn "BinanceAccount::Processor - could not convert #{quote_symbol} to USD: #{e.message}" nil @@ -233,6 +329,117 @@ class BinanceAccount::Processor # Converts the trade commission to USD. # commissionAsset can be: a stablecoin (≈ 1 USD), the base asset, or something else (e.g. BNB). + def process_p2p_trades(trades) + account = binance_account.current_account + return unless account + + Rails.logger.info "BinanceAccount::Processor - found #{trades.size} P2P trades to process" + + trades.each do |trade| + external_id = "binance_p2p_#{trade["orderNumber"]}" + funding_external_id = "#{external_id}_funding" + + # Deduplicate by checking for either the Trade or Funding leg in a single query + if account.entries.where(external_id: [ external_id, funding_external_id ]).exists? + Rails.logger.info "BinanceAccount::Processor - skipping P2P trade #{trade["orderNumber"]}: already exists in DB" + next + end + + date = Time.zone.at(trade["createTime"].to_i / 1000).to_date + trade_type = trade["tradeType"] # BUY or SELL + + begin + # Grab the exact Fiat and Crypto truth straight from the payload + fiat_currency = trade["fiat"] + fiat_amount = trade["totalPrice"].to_d + fiat_price = trade["unitPrice"].to_d + + crypto_asset = trade["asset"] + gross_crypto = trade["amount"].to_d + net_crypto = (trade["takerAmount"] || gross_crypto).to_d + crypto_fee = (trade["takerCommission"] || 0).to_d + + ticker = "CRYPTO:#{crypto_asset}" + security = BinanceAccount::SecurityResolver.resolve(ticker, crypto_asset) + + unless security + Rails.logger.warn "BinanceAccount::Processor - skipping P2P trade #{trade["orderNumber"]}: could not resolve security for #{crypto_asset}" + next + end + + # Convert the crypto fee (if any) to its fiat equivalent using the trade's exact unit price + fiat_fee = (crypto_fee * fiat_price).round(2) + + # 3. AI Fix: Wrap the double-entry in a transaction block to guarantee ledger integrity + account.transaction do + if trade_type == "BUY" + # BUY LOGIC: User sent Fiat from their bank, received Crypto + account.entries.create!( + date: date, + name: "P2P Payment (#{fiat_currency})", + amount: -fiat_amount, # Fiat leaving the system + currency: fiat_currency, + external_id: funding_external_id, + source: "binance", + entryable: Transaction.new + ) + + account.entries.create!( + date: date, + name: "P2P Buy #{gross_crypto.round(8)} #{crypto_asset}", + amount: fiat_amount, # Fiat value entering as Crypto (Cost Basis) + currency: fiat_currency, + external_id: external_id, + source: "binance", + entryable: Trade.new( + security: security, + qty: net_crypto, + price: fiat_price, + currency: fiat_currency, + fee: fiat_fee, + investment_activity_label: "Buy" + ) + ) + else + # SELL LOGIC: User liquidated Crypto, received Fiat to their bank + account.entries.create!( + date: date, + name: "P2P Sell #{gross_crypto.round(8)} #{crypto_asset}", + amount: -fiat_amount, # Fiat value of Crypto leaving + currency: fiat_currency, + external_id: external_id, + source: "binance", + entryable: Trade.new( + security: security, + qty: -net_crypto, + price: fiat_price, + currency: fiat_currency, + fee: fiat_fee, + investment_activity_label: "Sell" + ) + ) + + account.entries.create!( + date: date, + name: "P2P Receipt (#{fiat_currency})", + amount: fiat_amount, # Fiat entering the system + currency: fiat_currency, + external_id: funding_external_id, + source: "binance", + entryable: Transaction.new + ) + end + end + rescue => e + Rails.logger.error "BINANCE P2P SYNC CRASHED for Order #{trade["orderNumber"]}: #{e.message}" + raise + end + end + rescue StandardError => e + Rails.logger.error "BinanceAccount::Processor - P2P trade processing failed: #{e.message}" + raise + end + def commission_in_usd(trade, base_symbol, trade_price, date: nil) raw = trade["commission"].to_d commission_asset = trade["commissionAsset"].to_s.upcase diff --git a/app/models/binance_item/futures_importer.rb b/app/models/binance_item/futures_importer.rb new file mode 100644 index 000000000..8371d377d --- /dev/null +++ b/app/models/binance_item/futures_importer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Pulls USDⓈ-M futures account data (balance and positions). +# Returns normalized asset list with source tag "futures". +class BinanceItem::FuturesImporter + attr_reader :binance_item, :provider + + def initialize(binance_item, provider:) + @binance_item = binance_item + @provider = provider + end + + # @return [Hash] { assets: [...], raw: , source: "futures" } + def import + raw = provider.get_futures_account + + # Binance Futures returns a slightly different format than spot + # assets are in raw["assets"], positions in raw["positions"] + + assets = [] + + # Process base assets (e.g. USDT, BUSD balances) + Array(raw["assets"]).each do |asset| + wallet_balance = asset["walletBalance"].to_d + unrealized_profit = asset["unrealizedProfit"].to_d + + # Total equity is wallet balance + unrealized PNL + total = wallet_balance + unrealized_profit + + next if total.zero? + + assets << { + symbol: asset["asset"], + free: asset["availableBalance"] || wallet_balance.to_s, + locked: (wallet_balance - (asset["availableBalance"] || wallet_balance.to_s).to_d).to_s, + total: total.to_s + } + end + + { assets: assets, raw: raw, source: "futures" } + rescue => e + Rails.logger.error "BinanceItem::FuturesImporter #{binance_item.id} - #{e.message}" + { assets: [], raw: nil, source: "futures", error: e.message } + end +end diff --git a/app/models/binance_item/importer.rb b/app/models/binance_item/importer.rb index 7d499db70..1987022d2 100644 --- a/app/models/binance_item/importer.rb +++ b/app/models/binance_item/importer.rb @@ -15,8 +15,9 @@ class BinanceItem::Importer spot_result = BinanceItem::SpotImporter.new(binance_item, provider: binance_provider).import margin_result = BinanceItem::MarginImporter.new(binance_item, provider: binance_provider).import earn_result = BinanceItem::EarnImporter.new(binance_item, provider: binance_provider).import + futures_result = BinanceItem::FuturesImporter.new(binance_item, provider: binance_provider).import - all_assets = tagged_assets(spot_result) + tagged_assets(margin_result) + tagged_assets(earn_result) + all_assets = tagged_assets(spot_result) + tagged_assets(margin_result) + tagged_assets(earn_result) + tagged_assets(futures_result) return { success: true, assets_imported: 0, total_usd: 0 } if all_assets.empty? @@ -27,13 +28,15 @@ class BinanceItem::Importer total_usd: total_usd, spot_raw: spot_result[:raw], margin_raw: margin_result[:raw], - earn_raw: earn_result[:raw] + earn_raw: earn_result[:raw], + futures_raw: futures_result[:raw] ) binance_item.upsert_binance_snapshot!({ "spot" => spot_result[:raw], "margin" => margin_result[:raw], "earn" => earn_result[:raw], + "futures" => futures_result[:raw], "imported_at" => Time.current.iso8601 }) @@ -68,7 +71,7 @@ class BinanceItem::Importer 0 end - def upsert_binance_account(all_assets:, total_usd:, spot_raw:, margin_raw:, earn_raw:) + def upsert_binance_account(all_assets:, total_usd:, spot_raw:, margin_raw:, earn_raw:, futures_raw:) ba = binance_item.binance_accounts.find_or_initialize_by(account_type: "combined") ba.assign_attributes( @@ -80,6 +83,7 @@ class BinanceItem::Importer "spot" => spot_raw, "margin" => margin_raw, "earn" => earn_raw, + "futures" => futures_raw, "assets" => all_assets.map(&:stringify_keys), "fetched_at" => Time.current.iso8601 } @@ -90,7 +94,7 @@ class BinanceItem::Importer end def build_institution_metadata(all_assets) - %w[spot margin earn].each_with_object({}) do |source, hash| + %w[spot margin earn futures].each_with_object({}) do |source, hash| source_assets = all_assets.select { |a| a[:source] == source } hash[source] = { "asset_count" => source_assets.size, diff --git a/app/models/provider/binance.rb b/app/models/provider/binance.rb index d1b8c018c..683b660fc 100644 --- a/app/models/provider/binance.rb +++ b/app/models/provider/binance.rb @@ -13,6 +13,7 @@ class Provider::Binance # Pipelock incorrectly interprets the '@' in Ruby instance variables as a password delimiter # in an URL (e.g. https://user:password@host). SPOT_BASE_URL = "https://api.binance.com".freeze + FUTURES_BASE_URL = "https://fapi.binance.com".freeze base_uri SPOT_BASE_URL default_options.merge!({ timeout: 30 }.merge(httparty_ssl_options)) @@ -87,14 +88,63 @@ class Provider::Binance signed_get("/api/v3/myTrades", extra_params: params) end + # USDⓈ-M Futures account — requires signed request + def get_futures_account + signed_get("/fapi/v2/account", base_url: FUTURES_BASE_URL) + end + + # Futures trade history for a single symbol + def get_futures_trades(symbol, limit: 1000, from_id: nil) + params = { "symbol" => symbol, "limit" => limit.to_s } + params["fromId"] = from_id.to_s if from_id + signed_get("/fapi/v1/userTrades", extra_params: params, base_url: FUTURES_BASE_URL) + end + + # P2P trade history — requires signed request + # Pass start_timestamp to fetch only recent trades (max 30 days window) + def get_p2p_trades(start_timestamp: nil, end_timestamp: nil) + params = { "tradeType" => "BUY" } # default to BUY, will loop in processor for SELL + params["startTimestamp"] = start_timestamp.to_s if start_timestamp + params["endTimestamp"] = end_timestamp.to_s if end_timestamp + signed_get("/sapi/v1/c2c/orderMatch/listUserOrderHistory", extra_params: params) + end + + # Internal helper to handle both buy and sell types since API requires specific tradeType or gets default BUY + def get_all_p2p_trades(start_timestamp: nil, end_timestamp: nil) + %w[BUY SELL].flat_map do |trade_type| + page = 1 + rows = 100 + data = [] + loop do + result = signed_get( + "/sapi/v1/c2c/orderMatch/listUserOrderHistory", + extra_params: { + "tradeType" => trade_type, + "startTimestamp" => start_timestamp&.to_s, + "endTimestamp" => end_timestamp&.to_s, + "page" => page.to_s, + "rows" => rows.to_s + }.compact + ) + batch = result.is_a?(Hash) ? Array(result["data"]) : [] + data.concat(batch) + break if batch.size < rows + page += 1 + end + data + end + end + private - def signed_get(path, extra_params: {}) + def signed_get(path, extra_params: {}, base_url: SPOT_BASE_URL) params = timestamp_params.merge(extra_params) query_string = URI.encode_www_form(params.sort) + full_url = "#{base_url}#{path}" + response = self.class.get( - path, + full_url, query: "#{query_string}&signature=#{sign(query_string)}", headers: auth_headers ) diff --git a/app/views/accounts/index.html.erb b/app/views/accounts/index.html.erb index 38ba4d888..e8f4bee52 100644 --- a/app/views/accounts/index.html.erb +++ b/app/views/accounts/index.html.erb @@ -17,7 +17,7 @@ ) %> <% end %> -<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @brex_items.empty? && @ibkr_items.empty? && @snaptrade_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? %> +<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @brex_items.empty? && @ibkr_items.empty? && @snaptrade_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? && @binance_items.empty? %> <%= render "empty" %> <% else %>
@@ -57,6 +57,10 @@ <%= render @coinbase_items.sort_by(&:created_at) %> <% end %> + <% if @binance_items.any? %> + <%= render @binance_items.sort_by(&:created_at) %> + <% end %> + <% if @snaptrade_items.any? %> <%= render @snaptrade_items.sort_by(&:created_at) %> <% end %> diff --git a/app/views/binance_items/_binance_item.html.erb b/app/views/binance_items/_binance_item.html.erb index 5a1374fc7..0fd356f3d 100644 --- a/app/views/binance_items/_binance_item.html.erb +++ b/app/views/binance_items/_binance_item.html.erb @@ -67,7 +67,7 @@ <% end %> <%= render DS::Menu.new do |menu| %> - <% if unlinked_count.to_i > 0 %> + <% if binance_item.unlinked_accounts_count > 0 %> <% menu.with_item( variant: "link", text: t(".import_accounts_menu"), @@ -110,10 +110,10 @@ provider_item: binance_item ) %> - <% if unlinked_count.to_i > 0 && binance_item.accounts.empty? %> -
+ <% if binance_item.unlinked_accounts_count > 0 %> +

<%= t(".setup_needed") %>

-

<%= t(".setup_description") %>

+

<%= t(".setup_description") %>

<%= render DS::Link.new( text: t(".setup_action"), icon: "plus", diff --git a/app/views/binance_items/setup_accounts.html.erb b/app/views/binance_items/setup_accounts.html.erb index 4b7ab0af5..323f2fb70 100644 --- a/app/views/binance_items/setup_accounts.html.erb +++ b/app/views/binance_items/setup_accounts.html.erb @@ -33,6 +33,18 @@
+
+

<%= t(".historical_import") %>

+
+ <%= form.label :sync_start_date, t("settings.providers.binance_panel.sync_start_date_label"), class: "label" %> + <%= form.date_field :sync_start_date, + value: @binance_item.sync_start_date || (Date.current - 1.year), + max: Date.current, + class: "input" %> +

<%= t("settings.providers.binance_panel.sync_start_date_help") %>

+
+
+ <% if @binance_accounts.empty? %>

<%= t(".no_accounts") %>

@@ -69,7 +81,7 @@ <%= binance_account.currency %>

-
+

<%= number_with_delimiter(binance_account.current_balance || 0, delimiter: ",") %>

diff --git a/app/views/settings/providers/_binance_panel.html.erb b/app/views/settings/providers/_binance_panel.html.erb index 578573188..1b329f160 100644 --- a/app/views/settings/providers/_binance_panel.html.erb +++ b/app/views/settings/providers/_binance_panel.html.erb @@ -54,6 +54,15 @@
+ <% if item.unlinked_accounts_count > 0 %> + <%= render DS::Link.new( + text: t("binance_items.binance_item.setup_action"), + icon: "plus", + variant: "primary", + href: setup_accounts_binance_item_path(item), + frame: :modal + ) %> + <% end %> <%= button_to sync_binance_item_path(item), method: :post, class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-secondary hover:text-primary border border-secondary rounded-lg hover:border-primary", diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index 1762c9d37..f167d34c2 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -375,6 +375,9 @@ en: connect_button: Connect Binance syncing: Syncing... sync: Sync + historical_import: "Historical Import Settings" + sync_start_date_label: "Import data from" + sync_start_date_help: "Select how far back to fetch historical trades." disconnect_confirm: "Are you sure you want to disconnect Binance?" kraken_panel: step1_html: 'Go to Kraken API settings' diff --git a/test/controllers/binance_items_controller_test.rb b/test/controllers/binance_items_controller_test.rb index 91b13d60b..451705428 100644 --- a/test/controllers/binance_items_controller_test.rb +++ b/test/controllers/binance_items_controller_test.rb @@ -55,6 +55,47 @@ class BinanceItemsControllerTest < ActionDispatch::IntegrationTest assert_equal "Crypto", binance_account.current_account.accountable_type end + test "complete_account_setup updates sync_start_date when provided with a valid past date" do + binance_account = @binance_item.binance_accounts.create!( + name: "Spot Portfolio", + account_type: "spot", + currency: "USD", + current_balance: 1000.0 + ) + + past_date = (Date.current - 7.days).to_s + + post complete_account_setup_binance_item_url(@binance_item), params: { + selected_accounts: [ binance_account.id ], + sync_start_date: past_date + } + + assert_response :redirect + @binance_item.reload + assert_equal Date.parse(past_date), @binance_item.sync_start_date + end + + test "complete_account_setup rejects a future sync_start_date and sets flash alert" do + binance_account = @binance_item.binance_accounts.create!( + name: "Spot Portfolio", + account_type: "spot", + currency: "USD", + current_balance: 1000.0 + ) + + future_date = (Date.current + 2.days).to_s + original_sync_date = @binance_item.sync_start_date + + post complete_account_setup_binance_item_url(@binance_item), params: { + selected_accounts: [ binance_account.id ], + sync_start_date: future_date + } + + @binance_item.reload + assert_nil @binance_item.sync_start_date + assert_equal "Sync start date must be a valid date in the past.", flash[:alert] + end + test "complete_account_setup with no selection shows message" do @binance_item.binance_accounts.create!( name: "Spot Portfolio", diff --git a/test/models/binance_account/processor_test.rb b/test/models/binance_account/processor_test.rb index 9a08ab158..0e363508e 100644 --- a/test/models/binance_account/processor_test.rb +++ b/test/models/binance_account/processor_test.rb @@ -86,4 +86,120 @@ class BinanceAccount::ProcessorTest < ActiveSupport::TestCase assert_equal "USD", @account.currency assert_in_delta 1000.0, @account.balance, 0.01 end + + test "processes futures trades correctly" do + @family.update!(currency: "USD") + @ba.update!(raw_payload: { "assets" => [ { "symbol" => "BTC", "total" => "1.0" } ] }) + + provider = mock + @item.stubs(:binance_provider).returns(provider) + @ba.stubs(:binance_item).returns(@item) + provider.stubs(:get_spot_trades).returns([]) + provider.stubs(:get_spot_price).returns("50000.0") + provider.stubs(:get_all_p2p_trades).returns([]) # Skip P2P + + # Mock futures trades + provider.stubs(:get_futures_trades).returns([]) + provider.stubs(:get_futures_trades).with("BTCUSDT", limit: 1000, from_id: nil, startTime: nil).returns([ + { "id" => 1, "time" => 1610000000000, "qty" => "0.1", "price" => "40000.0", "quoteQty" => "4000.0", "commission" => "0.0", "commissionAsset" => "USDT", "buyer" => true } + ]) + + Security.create!(ticker: "CRYPTO:BTC", name: "Bitcoin", price_provider: "binance_public") + + assert_difference "Entry.count", 1 do + BinanceAccount::Processor.new(@ba).process + end + + assert @account.entries.exists?(external_id: "binance_futures_BTCUSDT_1") + end + + test "processes P2P BUY trades with double-entry logic and exact native fiat" do + @family.update!(currency: "USD") + @account.update!(currency: "USD") + + provider = mock + @item.stubs(:binance_provider).returns(provider) + @ba.stubs(:binance_item).returns(@item) + + # Silence other importers + provider.stubs(:get_spot_trades).returns([]) + provider.stubs(:get_futures_trades).returns([]) + + # Mock the exact TZS/USDT payload with actual fiat transfer amounts + provider.stubs(:get_all_p2p_trades).returns([ + { + "orderNumber" => "22883918231657005056", + "createTime" => 1777736533166, + "tradeType" => "BUY", + "asset" => "USDT", + "fiat" => "TZS", + "totalPrice" => "31500.00", + "unitPrice" => "2746.29", + "amount" => "11.47", # Gross crypto + "takerAmount" => "11.41", # Net crypto + "takerCommission" => "0.06" # Crypto fee + } + ]) + + Security.create!(ticker: "CRYPTO:USDT", name: "Tether", price_provider: "binance_public") + + # It MUST create 2 entries: 1 Deposit (Transaction) and 1 Purchase (Trade) + assert_difference "Entry.count", 2 do + BinanceAccount::Processor.new(@ba).process + end + + # Verify the Deposit (Transaction) - Should be native fiat + deposit = @account.entries.find_by(external_id: "binance_p2p_22883918231657005056_funding") + assert_not_nil deposit + assert_equal "Transaction", deposit.entryable_type + assert_equal (-31500.00), deposit.amount.to_f # Negative = Fiat Cash INFLOW + assert_equal "TZS", deposit.currency + + # Verify the Buy (Trade) - Should reflect the fiat cost basis + trade = @account.entries.find_by(external_id: "binance_p2p_22883918231657005056") + assert_not_nil trade + assert_equal "Trade", trade.entryable_type + assert_equal 31500.00, trade.amount.to_f # Positive = Fiat Cash OUTFLOW + assert_equal "TZS", trade.currency + assert_equal "Buy", trade.entryable.investment_activity_label + + # Verify the specific crypto math and fiat fee conversion + assert_equal 11.41, trade.entryable.qty.to_f + + # Fiat Fee = Crypto Fee (0.06) * Unit Price (2746.29) = 164.7774 (rounds to 164.78) + assert_equal 164.78, trade.entryable.fee.to_f + end + + test "skips processing if P2P external_id already exists" do + @family.update!(currency: "USD") + @account.update!(currency: "USD") + + # Pre-create the trade in the database + @account.entries.create!( + date: Date.current, + name: "Existing P2P", + amount: 10, + currency: "USD", + external_id: "binance_p2p_existing_123", + entryable: Transaction.new + ) + + provider = mock + @item.stubs(:binance_provider).returns(provider) + @ba.stubs(:binance_item).returns(@item) + provider.stubs(:get_spot_trades).returns([]) + provider.stubs(:get_futures_trades).returns([]) + + # Mock a payload with the SAME orderNumber + provider.stubs(:get_all_p2p_trades).returns([ + { "orderNumber" => "existing_123", "tradeType" => "BUY", "asset" => "USDT", "amount" => "10.0" } + ]) + + Security.create!(ticker: "CRYPTO:USDT", name: "Tether", price_provider: "binance_public") + + # Assert that NO new entries are created + assert_no_difference "Entry.count" do + BinanceAccount::Processor.new(@ba).process + end + end end diff --git a/test/models/binance_item/futures_importer_test.rb b/test/models/binance_item/futures_importer_test.rb new file mode 100644 index 000000000..6ce9dba1a --- /dev/null +++ b/test/models/binance_item/futures_importer_test.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require "test_helper" + +class BinanceItem::FuturesImporterTest < ActiveSupport::TestCase + setup do + @provider = mock + @family = families(:dylan_family) + @item = BinanceItem.create!(family: @family, name: "B", api_key: "k", api_secret: "s") + end + + test "returns normalized assets from USDⓈ-M futures with source=futures" do + @provider.stubs(:get_futures_account).returns({ + "assets" => [ + { "asset" => "USDT", "walletBalance" => "100.0", "unrealizedProfit" => "5.0", "availableBalance" => "90.0" }, + { "asset" => "BUSD", "walletBalance" => "0.0", "unrealizedProfit" => "0.0", "availableBalance" => "0.0" } + ], + "positions" => [ + { "symbol" => "BTCUSDT", "positionAmt" => "0.5" } + ] + }) + + result = BinanceItem::FuturesImporter.new(@item, provider: @provider).import + + assert_equal "futures", result[:source] + assert_equal 1, result[:assets].size + usdt = result[:assets].first + assert_equal "USDT", usdt[:symbol] + assert_equal "105.0", usdt[:total] # walletBalance + unrealizedProfit + assert_equal "90.0", usdt[:free] + assert_equal "10.0", usdt[:locked] # walletBalance - availableBalance + end + + test "returns empty on API error" do + @provider.stubs(:get_futures_account).raises(Provider::Binance::ApiError, "WAF") + + result = BinanceItem::FuturesImporter.new(@item, provider: @provider).import + + assert_equal "futures", result[:source] + assert_equal [], result[:assets] + end +end diff --git a/test/models/binance_item/importer_test.rb b/test/models/binance_item/importer_test.rb index 0e9506155..a38bf60a5 100644 --- a/test/models/binance_item/importer_test.rb +++ b/test/models/binance_item/importer_test.rb @@ -12,6 +12,7 @@ class BinanceItem::ImporterTest < ActiveSupport::TestCase stub_spot_result([ { symbol: "BTC", free: "1.0", locked: "0.0", total: "1.0" } ]) stub_margin_result([]) stub_earn_result([]) + stub_futures_result([]) end test "creates a binance_account of type combined" do @@ -48,6 +49,7 @@ class BinanceItem::ImporterTest < ActiveSupport::TestCase stub_spot_result([]) stub_margin_result([]) stub_earn_result([]) + stub_futures_result([]) assert_no_difference "@item.binance_accounts.count" do BinanceItem::Importer.new(@item, binance_provider: @provider).import @@ -61,6 +63,7 @@ class BinanceItem::ImporterTest < ActiveSupport::TestCase assert ba.raw_payload.key?("spot") assert ba.raw_payload.key?("margin") assert ba.raw_payload.key?("earn") + assert ba.raw_payload.key?("futures") end private @@ -82,4 +85,10 @@ class BinanceItem::ImporterTest < ActiveSupport::TestCase { assets: assets, raw: {}, source: "earn" } ) end + + def stub_futures_result(assets) + BinanceItem::FuturesImporter.any_instance.stubs(:import).returns( + { assets: assets, raw: {}, source: "futures" } + ) + end end