Files
sure/test/models/kraken_account/ledger_processor_test.rb
ghost d329a4f69d feat(kraken): import deposits, withdrawals, staking & fees via Ledgers API (#2451)
* feat(kraken): fetch and import Ledgers API for deposits, withdrawals, staking, fees

Closes #2450

Kraken TradesHistory only returns spot buy/sell trades. The Ledgers API
(/0/private/Ledgers) covers deposits, withdrawals, staking rewards, Earn
income, and standalone fees — everything that was missing from syncs.

Changes:
- Provider::Kraken#get_ledgers — new method forwarding start/type/offset params
- KrakenItem::Importer#fetch_ledgers — paginated fetch (up to 200 pages) with
  graceful fallback if the API key lacks Query Ledger Entries permission
- Importer#upsert_kraken_account — stores "ledgers" alongside "trades" in
  raw_transactions_payload
- KrakenAccount::LedgerProcessor — new class; maps each supported ledger type
  (deposit, withdrawal, staking, earn, fee) to a Transaction entry with the
  correct investment_activity_label, kind, and sign convention; skips trade/
  transfer/margin types to avoid double-counting with TradesHistory
- KrakenAccount::Processor#process — calls LedgerProcessor after process_trades
- Multi-currency: fiat amounts converted via ExchangeRate (non-USD fiat bridged
  through USD); crypto amounts use the spot price cached in raw_payload["assets"]
  with a price_missing flag when no price is available
- Dedup guard: external_id "kraken_ledger_<id>" + source "kraken" prevents
  re-importing on repeated syncs

* fix(kraken): correct sign convention, fee inclusion, and earn subtype filtering

- Sign convention: deposits/staking/earn → negative (inflow), withdrawals/fees
  → positive (outflow), matching Sure's global convention (inflow is negative)
- Fee inclusion: use (amount - fee).abs as abs_impact so withdrawal fees are
  counted in the total outflow rather than discarded
- Earn subtypes: skip allocation/deallocation ledger entries (internal fund
  movements); only import rewardallocation/bonusallocation as Interest income
- DebugLogEntry: replace Rails.logger.warn/error with DebugLogEntry.capture
  throughout LedgerProcessor and the Ledgers permission fallback in Importer,
  so support-relevant incidents surface in /settings/debug
- Importer test: stub get_ledgers in setup so existing tests do not error
  on the new fetch_ledgers call

* fix(kraken): route duplicate ledger-id warning through DebugLogEntry

* perf(kraken): batch ledger idempotency check; strengthen tests

Address review feedback (jjmata):

- N+1: LedgerProcessor#process_ledger_entry ran `account.entries.exists?(...)`
  per ledger entry (up to ~10k per sync). Load the existing Kraken external IDs
  once into a Set and test membership in memory (newly created IDs are added so
  the same run stays idempotent) — same pattern as #2452.
- Tests: the idempotency test now asserts the first pass actually creates the
  entry (assert_difference) before asserting the second is a no-op; add a guard
  asserting the second (all-skipped) pass issues a single bulk external_id pluck,
  not one query per entry.

No behavior change to imported entries.

* perf(kraken): scope ledger idempotency pluck to kraken_ledger_ prefix

Only load existing ledger external IDs (not trade entries) into the idempotency
Set, matching the reviewed approach. No behavior change.
2026-06-30 07:35:14 +02:00

316 lines
12 KiB
Ruby
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# frozen_string_literal: true
require "test_helper"
class KrakenAccount::LedgerProcessorTest < ActiveSupport::TestCase
setup do
@family = families(:dylan_family)
@account = @family.accounts.create!(
name: "Kraken", balance: 0, currency: "USD",
accountable: Crypto.new
)
@item = KrakenItem.create!(
family: @family, name: "Kraken", api_key: "k", api_secret: "s"
)
@kraken_account = @item.kraken_accounts.create!(
name: "Kraken", account_id: "combined", account_type: "combined", currency: "USD",
current_balance: 0,
raw_payload: {
"asset_metadata" => { "XXBT" => { "altname" => "BTC" }, "ZUSD" => { "altname" => "USD" }, "ZEUR" => { "altname" => "EUR" } },
"assets" => [ { "symbol" => "BTC", "price_usd" => "50000.00" } ]
},
raw_transactions_payload: { "trades" => {}, "ledgers" => {} }
)
@kraken_account.ensure_account_provider!(@account)
end
# ---------------------------------------------------------------------------
# sign convention: Sure uses negative = inflow, positive = outflow
# ---------------------------------------------------------------------------
test "creates a deposit entry with negative amount (inflow)" do
set_ledgers(
"LABC01" => ledger_entry(type: "deposit", asset: "ZUSD", amount: "1000.00", fee: "0.00", time: 1_700_000_000)
)
assert_difference "@account.entries.count", 1 do
process
end
entry = @account.entries.find_by(external_id: "kraken_ledger_LABC01", source: "kraken")
assert entry, "deposit entry must exist"
assert entry.amount.negative?, "deposit is an inflow — must be negative in Sure's convention"
assert_in_delta(-1000.0, entry.amount.to_f, 0.01)
assert_equal "USD", entry.currency
assert_match(/Deposit.*USD/, entry.name)
txn = entry.entryable
assert_equal "funds_movement", txn.kind
assert_equal "Contribution", txn.investment_activity_label
assert_equal "LABC01", txn.extra.dig("kraken", "ledger_id")
assert_equal "deposit", txn.extra.dig("kraken", "type")
end
test "creates a withdrawal entry with positive amount (outflow)" do
set_ledgers(
"LWIT01" => ledger_entry(type: "withdrawal", asset: "ZUSD", amount: "-500.00", fee: "0.00", time: 1_700_000_000)
)
assert_difference "@account.entries.count", 1 do
process
end
entry = @account.entries.find_by(external_id: "kraken_ledger_LWIT01", source: "kraken")
assert entry
assert entry.amount.positive?, "withdrawal is an outflow — must be positive in Sure's convention"
assert_in_delta 500.0, entry.amount.to_f, 0.01
assert_match(/Withdrawal.*USD/, entry.name)
assert_equal "Withdrawal", entry.entryable.investment_activity_label
assert_equal "funds_movement", entry.entryable.kind
end
# ---------------------------------------------------------------------------
# fee inclusion in amount
# ---------------------------------------------------------------------------
test "includes the Kraken fee in the total withdrawal amount" do
# Kraken: balance_change = amount - fee = -500 - 1 = -501 total outflow
set_ledgers(
"LWIT02" => ledger_entry(type: "withdrawal", asset: "ZUSD", amount: "-500.00", fee: "1.00", time: 1_700_000_000)
)
process
entry = @account.entries.find_by(external_id: "kraken_ledger_LWIT02", source: "kraken")
assert entry
assert_in_delta 501.0, entry.amount.to_f, 0.01
end
# ---------------------------------------------------------------------------
# BTC deposit (crypto → family currency conversion)
# ---------------------------------------------------------------------------
test "creates a deposit entry for BTC using stored price" do
set_ledgers(
"LBTC01" => ledger_entry(type: "deposit", asset: "XXBT", amount: "0.10000000", fee: "0.00000000", time: 1_700_000_000)
)
process
entry = @account.entries.find_by(external_id: "kraken_ledger_LBTC01", source: "kraken")
assert entry
assert entry.amount.negative?, "BTC deposit is an inflow — must be negative"
# 0.1 BTC × $50,000/BTC = $5,000 (family currency = USD, no conversion needed)
assert_in_delta(-5000.0, entry.amount.to_f, 1.0)
assert_match(/Deposit.*BTC/, entry.name)
end
# ---------------------------------------------------------------------------
# staking
# ---------------------------------------------------------------------------
test "creates a staking reward entry (negative = inflow)" do
set_ledgers(
"LSTK01" => ledger_entry(type: "staking", asset: "XXBT", amount: "0.00050000", fee: "0.00", time: 1_700_000_000)
)
assert_difference "@account.entries.count", 1 do
process
end
entry = @account.entries.find_by(external_id: "kraken_ledger_LSTK01", source: "kraken")
assert entry
assert entry.amount.negative?, "staking reward is an inflow — must be negative"
assert_match(/Staking reward.*BTC/, entry.name)
assert_equal "Dividend", entry.entryable.investment_activity_label
assert_equal "standard", entry.entryable.kind
end
# ---------------------------------------------------------------------------
# earn
# ---------------------------------------------------------------------------
test "creates an earn reward entry for rewards subtype" do
set_ledgers(
"LERN01" => ledger_entry(type: "earn", subtype: "rewardallocation", asset: "ZUSD", amount: "5.00", fee: "0.00", time: 1_700_000_000)
)
assert_difference "@account.entries.count", 1 do
process
end
entry = @account.entries.find_by(external_id: "kraken_ledger_LERN01", source: "kraken")
assert entry
assert entry.amount.negative?, "earn reward is an inflow — must be negative"
assert_equal "Interest", entry.entryable.investment_activity_label
end
test "skips earn allocation entries (internal fund movement, not income)" do
set_ledgers(
"LALLOC" => ledger_entry(type: "earn", subtype: "allocation", asset: "ZUSD", amount: "500.00", fee: "0.00", time: 1_700_000_000),
"LDEALLOC" => ledger_entry(type: "earn", subtype: "deallocation", asset: "ZUSD", amount: "-500.00", fee: "0.00", time: 1_700_000_000)
)
assert_no_difference "@account.entries.count" do
process
end
end
# ---------------------------------------------------------------------------
# standalone fee
# ---------------------------------------------------------------------------
test "creates a fee entry with positive amount (outflow)" do
set_ledgers(
"LFEE01" => ledger_entry(type: "fee", asset: "ZUSD", amount: "-7.50", fee: "0.00", time: 1_700_000_000)
)
assert_difference "@account.entries.count", 1 do
process
end
entry = @account.entries.find_by(external_id: "kraken_ledger_LFEE01", source: "kraken")
assert entry
assert entry.amount.positive?, "fee is an outflow — must be positive"
assert_in_delta 7.5, entry.amount.to_f, 0.01
assert_equal "Fee", entry.entryable.investment_activity_label
end
# ---------------------------------------------------------------------------
# skipped types
# ---------------------------------------------------------------------------
test "skips trade-type ledger entries (handled by TradesHistory)" do
set_ledgers(
"LTRD01" => ledger_entry(type: "trade", asset: "XXBT", amount: "-0.1", fee: "0.0", time: 1_700_000_000)
)
assert_no_difference "@account.entries.count" do
process
end
end
test "skips transfer-type ledger entries" do
set_ledgers(
"LTRN01" => ledger_entry(type: "transfer", asset: "XXBT", amount: "0.1", fee: "0.0", time: 1_700_000_000)
)
assert_no_difference "@account.entries.count" do
process
end
end
# ---------------------------------------------------------------------------
# idempotency
# ---------------------------------------------------------------------------
test "does not duplicate entries on repeated processing" do
set_ledgers(
"LIDEM01" => ledger_entry(type: "deposit", asset: "ZUSD", amount: "100.00", fee: "0.00", time: 1_700_000_000)
)
# First pass must actually create the entry...
assert_difference "@account.entries.count", 1 do
process
end
# ...and a second pass must be a no-op.
assert_no_difference "@account.entries.count" do
process
end
end
test "idempotency check does not scale entries queries with ledger count" do
set_ledgers(
"LQ1" => ledger_entry(type: "deposit", asset: "ZUSD", amount: "10.00", fee: "0.00", time: 1_700_000_000),
"LQ2" => ledger_entry(type: "deposit", asset: "ZUSD", amount: "20.00", fee: "0.00", time: 1_700_000_100),
"LQ3" => ledger_entry(type: "deposit", asset: "ZUSD", amount: "30.00", fee: "0.00", time: 1_700_000_200)
)
process # first pass creates the 3 entries
assert_equal 3, @account.entries.count
# On a second pass every entry is already present, so all are skipped. The
# existence check must be a single bulk pluck regardless of ledger count —
# the previous per-entry `exists?` would issue one query per entry instead.
queries = capture_sql_queries { process }
entries_selects = queries.count { |q| q.match?(/from "entries"/i) }
assert_equal 1, entries_selects,
"second pass should issue exactly one bulk external_id pluck, not one per entry"
end
# ---------------------------------------------------------------------------
# non-USD family currency
# ---------------------------------------------------------------------------
test "converts USD deposit to non-USD family currency" do
@family.update!(currency: "EUR")
ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date: Date.current, rate: 0.92)
set_ledgers(
"LEUR01" => ledger_entry(type: "deposit", asset: "ZUSD", amount: "1000.00", fee: "0.00", time: Time.current.to_i)
)
process
entry = @account.entries.find_by(external_id: "kraken_ledger_LEUR01", source: "kraken")
assert entry
assert_equal "EUR", entry.currency
assert entry.amount.negative?, "deposit is inflow — negative"
assert_in_delta(-920.0, entry.amount.to_f, 1.0)
end
# ---------------------------------------------------------------------------
# missing crypto price
# ---------------------------------------------------------------------------
test "records zero amount and price_missing flag when no price data available" do
set_raw_payload_assets([])
set_ledgers(
"LNOPRICE" => ledger_entry(type: "deposit", asset: "XXBT", amount: "0.5", fee: "0.00", time: 1_700_000_000)
)
assert_difference "@account.entries.count", 1 do
process
end
entry = @account.entries.find_by(external_id: "kraken_ledger_LNOPRICE", source: "kraken")
assert entry
assert_equal 0, entry.amount.to_f
assert entry.entryable.extra.dig("kraken", "price_missing")
end
private
def process
KrakenAccount::LedgerProcessor.new(@kraken_account).process
end
def set_ledgers(ledgers)
@kraken_account.update!(
raw_transactions_payload: @kraken_account.raw_transactions_payload.merge("ledgers" => ledgers)
)
end
def set_raw_payload_assets(assets)
@kraken_account.update!(
raw_payload: @kraken_account.raw_payload.merge("assets" => assets)
)
end
def ledger_entry(type:, asset:, amount:, fee:, time:, subtype: "")
{
"refid" => "S#{SecureRandom.hex(4).upcase}",
"time" => time,
"type" => type,
"subtype" => subtype,
"aclass" => "currency",
"asset" => asset,
"amount" => amount,
"fee" => fee,
"balance" => "1.00000000"
}
end
end