diff --git a/app/controllers/mcp_controller.rb b/app/controllers/mcp_controller.rb index cf4906738..159fdcc75 100644 --- a/app/controllers/mcp_controller.rb +++ b/app/controllers/mcp_controller.rb @@ -70,7 +70,7 @@ class McpController < ApplicationController end def handle_tools_list - tools = Assistant.function_classes.map do |fn_class| + tools = Assistant.function_classes(mcp_user).map do |fn_class| fn_instance = fn_class.new(mcp_user) { name: fn_instance.name, @@ -86,7 +86,10 @@ class McpController < ApplicationController name = params&.dig("name") arguments = params&.dig("arguments") || {} - fn_class = Assistant.function_classes.find { |fc| fc.name == name } + # Scoped to the authenticated user so a tool hidden from tools/list is not + # callable by guessing its name — a non-preview caller gets the same + # "Unknown tool" response as for a name that does not exist. + fn_class = Assistant.function_classes(mcp_user).find { |fc| fc.name == name } unless fn_class render_jsonrpc_error(request_id, -32602, "Unknown tool: #{name}") diff --git a/app/models/assistant.rb b/app/models/assistant.rb index 708c0aabf..af5e31672 100644 --- a/app/models/assistant.rb +++ b/app/models/assistant.rb @@ -6,6 +6,17 @@ module Assistant "external" => Assistant::External }.freeze + # Statement Vault + provenance tools, for users who opted into preview features + # in Settings -> Preferences. They back the patrimonial agent-harness workflow + # documented in docs/llm-guides/patrimonial-agent-harness.md. + PREVIEW_FUNCTION_CLASSES = [ + Function::UploadAccountStatement, + Function::ListAccountStatements, + Function::GetAccountStatement, + Function::GetStatementCoverage, + Function::RecordValuation + ].freeze + class << self def for_chat(chat) implementation_for(chat).for_chat(chat) @@ -20,8 +31,12 @@ module Assistant REGISTRY.keys end - def function_classes - [ + # The single registry behind both the builtin chat and the /mcp endpoint's + # tools/list — a function class added here is immediately callable by an + # external agent, so pass the user to keep preview tools out of the default + # surface. + def function_classes(user = nil) + classes = [ Function::GetTransactions, Function::GetAccounts, Function::GetHoldings, @@ -38,6 +53,9 @@ module Assistant Function::CreateCategory, Function::UpdateCategory ] + + classes += PREVIEW_FUNCTION_CLASSES if user&.preview_features_enabled? + classes end private diff --git a/app/models/assistant/configurable.rb b/app/models/assistant/configurable.rb index 8c68ffb4f..d50bbd318 100644 --- a/app/models/assistant/configurable.rb +++ b/app/models/assistant/configurable.rb @@ -14,7 +14,7 @@ module Assistant::Configurable else { instructions: default_instructions(preferred_currency, preferred_date_format), - functions: default_functions + functions: default_functions(chat.user) } end end @@ -51,8 +51,8 @@ module Assistant::Configurable PROMPT end - def default_functions - Assistant.function_classes + def default_functions(user = nil) + Assistant.function_classes(user) end def default_instructions(preferred_currency, preferred_date_format) diff --git a/app/models/assistant/function/get_account_statement.rb b/app/models/assistant/function/get_account_statement.rb new file mode 100644 index 000000000..a6a7f3846 --- /dev/null +++ b/app/models/assistant/function/get_account_statement.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +class Assistant::Function::GetAccountStatement < Assistant::Function + include Assistant::Function::StatementVaultSupport + + DOWNLOAD_URL_TTL = 15.minutes + + class << self + def name + "get_account_statement" + end + + def description + <<~INSTRUCTIONS + Fetch one Statement Vault document by ID: its identity (SHA-256, filename, + period), the balances read off the document, and the reconciliation checks + comparing those balances against the ledger. + + The reconciliation checks are the trustworthy part. Each compares a figure + printed on the statement against the account's recorded balance: + + - `opening_balance` / `closing_balance` — the statement's figure vs the ledger's + - `period_movement` — the change across the period, both sides + + Each check reports `matched` or `mismatched` (tolerance: 0.01). A mismatch + means the ledger and the document disagree — report it, and do not paper + over it by adjusting the number to fit. + + Also returns a short-lived download URL (valid #{DOWNLOAD_URL_TTL.inspect}) + for the original file when one is attached. + + Example: + + ``` + get_account_statement({ statement_id: "abc123-def456" }) + ``` + INSTRUCTIONS + end + end + + def strict_mode? + false + end + + def params_schema + build_schema( + required: [ "statement_id" ], + properties: { + statement_id: { + type: "string", + description: "UUID of the statement, as returned by list_account_statements or upload_account_statement." + } + } + ) + end + + def call(params = {}) + return not_a_statement_manager unless statement_manager? + + statement = find_statement(params["statement_id"]) + + unless statement&.viewable_by?(user) + return error("not_found", "No statement found with that ID that this user can view.") + end + + { + success: true, + statement: statement_payload(statement).merge( + reconciliation_status: statement.reconciliation_status, + reconciliation_checks: reconciliation_payload(statement), + download_url: download_url(statement) + ).compact + } + end + + private + def reconciliation_payload(statement) + statement.reconciliation_checks.map do |check| + { + check: check[:key], + statement_amount: check[:statement_amount].to_s, + ledger_amount: check[:ledger_amount].to_s, + difference: check[:difference].to_s, + status: check[:status] + } + end + end + + # Chat and MCP clients render outside the request that produced the record, so + # the URL has to be absolute. Falls back to nil when no host is configured + # (e.g. a self-hosted worker with no APP_DOMAIN) rather than handing back a + # relative path an external agent cannot resolve. + def download_url(statement) + return nil unless statement.original_file.attached? + + host_opts = Rails.application.config.action_mailer.default_url_options || {} + return nil if host_opts[:host].blank? + + Rails.application.routes.url_helpers.rails_blob_url( + statement.original_file, + host_opts.merge(disposition: "attachment", expires_in: DOWNLOAD_URL_TTL) + ) + end +end diff --git a/app/models/assistant/function/get_statement_coverage.rb b/app/models/assistant/function/get_statement_coverage.rb new file mode 100644 index 000000000..832c56381 --- /dev/null +++ b/app/models/assistant/function/get_statement_coverage.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +class Assistant::Function::GetStatementCoverage < Assistant::Function + include Assistant::Function::StatementVaultSupport + + class << self + def name + "get_statement_coverage" + end + + def description + <<~INSTRUCTIONS + Report, month by month, which statements the family actually holds for an + account in a given year — the document-coverage map behind the numbers. + + Each month comes back with one status: + + - `covered` — a linked statement covers the month and reconciles + - `mismatched` — a statement covers it, but its balances disagree with the ledger + - `missing` — no statement on record; the month's figures have no document behind them + - `ambiguous` — a statement was suggested for this account but nobody has confirmed the link + - `duplicate` — two or more linked statements overlap the same month + - `not_expected` — outside the account's expected statement range + + Use it before asserting anything about a period: "no statement on record" + is a legitimate and necessary answer, and is very different from "the + balance was zero". Use it to tell the user exactly which documents to go + find. + + Example: + + ``` + get_statement_coverage({ account_id: "abc123-def456", year: 2026 }) + ``` + INSTRUCTIONS + end + end + + def strict_mode? + false + end + + def params_schema + build_schema( + required: [ "account_id" ], + properties: { + account_id: { + type: "string", + description: "UUID of the account to report coverage for." + }, + year: { + type: "integer", + description: "Calendar year. Defaults to the most recent year with expected statements." + } + } + ) + end + + def call(params = {}) + return not_a_statement_manager unless statement_manager? + + account_id = params["account_id"].to_s + return error("invalid_account_id", "account_id must be a UUID.") unless valid_uuid?(account_id) + + account = family.accounts.accessible_by(user).find_by(id: account_id) + return error("account_not_found", "No accessible account found with that ID.") unless account + + coverage = AccountStatement::Coverage.for_year(account, params["year"]) + + { + success: true, + account: account_ref(account), + year: coverage.selected_year, + available_years: coverage.available_years, + summary: coverage.summary_counts, + months: coverage.months.map { |month| month_payload(month) } + } + end + + private + def month_payload(month) + { + month: month.date.strftime("%Y-%m"), + status: month.status, + statement_ids: month.statements.map(&:id), + unconfirmed_statement_ids: month.ambiguous_statements.map(&:id) + }.compact_blank + end +end diff --git a/app/models/assistant/function/list_account_statements.rb b/app/models/assistant/function/list_account_statements.rb new file mode 100644 index 000000000..125eef6b8 --- /dev/null +++ b/app/models/assistant/function/list_account_statements.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +class Assistant::Function::ListAccountStatements < Assistant::Function + include Assistant::Function::StatementVaultSupport + + DEFAULT_LIMIT = 25 + MAX_LIMIT = 100 + + class << self + def name + "list_account_statements" + end + + def description + <<~INSTRUCTIONS + List documents in the family's Statement Vault with their identity and + provenance: SHA-256, filename, statement period, linked account, and + review status. + + Use this to answer "which statements do we hold?", to find the document + backing a figure, to check whether a file is already archived (filter by + `content_sha256`), or to work the review queue (filter by + `review_status: "unmatched"` for documents awaiting a human's account + decision). + + This returns document identity, not document contents. To search inside + uploaded documents use `search_family_files`; to fetch one statement's + reconciliation figures and a download link use `get_account_statement`. + + Example: + + ``` + list_account_statements({ + review_status: "unmatched", + period_start_on_or_after: "2026-01-01" + }) + ``` + INSTRUCTIONS + end + end + + def strict_mode? + false + end + + def params_schema + build_schema( + properties: { + account_id: { + type: "string", + description: "Only statements linked to this account UUID." + }, + review_status: { + type: "string", + enum: AccountStatement.review_statuses.keys, + description: "unmatched = awaiting a human account decision, linked = attached to an account, rejected = the suggested match was declined." + }, + content_sha256: { + type: "string", + description: "Look up a specific document by the SHA-256 of its contents. Use this to check whether a file is already archived." + }, + period_start_on_or_after: { + type: "string", + description: "ISO 8601 date (YYYY-MM-DD). Only statements whose period ends on or after this date." + }, + period_end_on_or_before: { + type: "string", + description: "ISO 8601 date (YYYY-MM-DD). Only statements whose period starts on or before this date." + }, + limit: { + type: "integer", + description: "Maximum statements to return (default #{DEFAULT_LIMIT}, max #{MAX_LIMIT})." + } + } + ) + end + + def call(params = {}) + return not_a_statement_manager unless statement_manager? + + scope = family.account_statements.includes(:account, :suggested_account).ordered + + if params["account_id"].present? + return error("invalid_account_id", "account_id must be a UUID.") unless valid_uuid?(params["account_id"]) + + scope = scope.where(account_id: params["account_id"]) + end + + if params["review_status"].present? + status = params["review_status"].to_s + unless AccountStatement.review_statuses.key?(status) + return error("invalid_review_status", "review_status must be one of: #{AccountStatement.review_statuses.keys.join(", ")}.") + end + + scope = scope.where(review_status: status) + end + + scope = scope.where(content_sha256: params["content_sha256"].to_s.strip) if params["content_sha256"].present? + + if params["period_start_on_or_after"].present? + date = parse_date(params["period_start_on_or_after"]) + return error("invalid_date", "period_start_on_or_after must be an ISO 8601 date (YYYY-MM-DD).") unless date + + scope = scope.where("period_end_on >= ?", date) + end + + if params["period_end_on_or_before"].present? + date = parse_date(params["period_end_on_or_before"]) + return error("invalid_date", "period_end_on_or_before must be an ISO 8601 date (YYYY-MM-DD).") unless date + + scope = scope.where("period_start_on <= ?", date) + end + + limit = (params["limit"] || DEFAULT_LIMIT).to_i.clamp(1, MAX_LIMIT) + # A statement with no account is visible to any statement manager; a linked + # one follows the account's sharing rules, so filter after the query. Counting + # before that filter would report statements this user may not know exist, so + # the page is over-fetched by one and reported as has_more instead. + rows = scope.limit(limit + 1).to_a + statements = rows.first(limit).select { |statement| statement.viewable_by?(user) } + + { + success: true, + returned: statements.size, + has_more: rows.size > limit, + statements: statements.map { |statement| statement_payload(statement) } + } + end +end diff --git a/app/models/assistant/function/record_valuation.rb b/app/models/assistant/function/record_valuation.rb new file mode 100644 index 000000000..02e4b5f08 --- /dev/null +++ b/app/models/assistant/function/record_valuation.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +class Assistant::Function::RecordValuation < Assistant::Function + class << self + def name + "record_valuation" + end + + def description + <<~INSTRUCTIONS + Record the value of an account on a given date, with a citation for where + the number came from. + + Use this for holdings the app cannot value on its own — property, + off-bank positions, private company stakes, collectibles — and to correct + a manual account's balance from a primary-source document. + + The `source` citation is required and is checked against this grammar: + + ["estimated: "] citation [" (grade: A|B|C)"] + + - `estimated: ` prefix — the value was interpolated or proxied, not read + off a document. Estimates must carry a grade. + - grade `A` — an official document for that exact date (statement, filed + return, capital account); `B` — derived with a document; `C` — a proxy + or assumption, i.e. a standing TODO to re-derive from the real source. + + Examples of valid citations: + + "Private bank statement 2026-03-31, securities subtotal (grade: A)" + "estimated: linear interpolation over 2024-08 / 2024-12 anchors (grade: C)" + + If you do not have a document to cite, do not invent one and do not call + this function — ask the user for the source. A value with no provenance + is worse than a missing value, because it looks authoritative. + + Recording a valuation on a date that already has one replaces it. + + Example: + + ``` + record_valuation({ + account_id: "abc123-def456", + date: "2026-03-31", + amount: 412500.00, + source: "Appraisal report 2026-03-12 by {firm} (grade: A)" + }) + ``` + INSTRUCTIONS + end + end + + def strict_mode? + false + end + + def params_schema + build_schema( + required: %w[account_id date amount source], + properties: { + account_id: { + type: "string", + description: "UUID of the account to value. The user must have write access to it." + }, + date: { + type: "string", + description: "ISO 8601 date (YYYY-MM-DD) the value applies to. For a period-end value, use the last day of the period." + }, + amount: { + type: "number", + description: "The account's value on that date, in the account's own currency." + }, + source: { + type: "string", + description: "Required citation naming the document this value came from, in the grammar: [\"estimated: \"] citation [\" (grade: A|B|C)\"]." + } + } + ) + end + + def call(params = {}) + citation = Provenance::Citation.parse!(params["source"]) + + date = parse_date(params["date"]) + return error("invalid_date", "date must be an ISO 8601 date (YYYY-MM-DD).") unless date + + amount = parse_decimal(params["amount"]) + return error("invalid_amount", "amount must be a number.") if amount.nil? + + account_id = params["account_id"].to_s + return error("invalid_account_id", "account_id must be a UUID.") unless valid_uuid?(account_id) + + account = family.accounts.writable_by(user).find_by(id: account_id) + return error("account_not_found", "No account found with that ID that this user can write to.") unless account + + entry = nil + replaced_existing = false + failure_message = nil + + ActiveRecord::Base.transaction do + account.lock! + replaced_existing = account.entries.valuations.exists?(date: date) + + result = account.create_reconciliation(balance: amount, date: date) + + unless result.success? + failure_message = result.error_message + raise ActiveRecord::Rollback + end + + entry = account.entries.valuations.find_by!(date: date) + entry.update!(notes: citation.to_s) + end + + unless entry + return error("valuation_failed", failure_message.presence || "The valuation could not be recorded.") + end + + { + success: true, + entry_id: entry.id, + account: { id: account.id, name: account.name, currency: account.currency }, + date: date.iso8601, + amount: entry.amount.to_s, + amount_formatted: entry.amount_money.format, + replaced_existing: replaced_existing, + provenance: citation.to_h, + message: "Recorded #{entry.amount_money.format} for #{account.name} on #{date.iso8601}, cited as: #{citation}." + } + rescue Provenance::Citation::InvalidError => e + error( + "invalid_source_citation", + "#{e.message}. Every recorded value must cite its source, in the grammar: #{Provenance::Citation.grammar}." + ) + rescue ActiveRecord::RecordInvalid => e + error("validation_failed", e.record.errors.full_messages.join("; ")) + end + + private + def parse_date(value) + return nil if value.blank? + + Date.iso8601(value.to_s) + rescue Date::Error + nil + end + + def parse_decimal(value) + return nil if value.nil? || value.to_s.strip.empty? + + BigDecimal(value.to_s) + rescue ArgumentError, TypeError + nil + end + + def error(key, message, extras = {}) + { success: false, error: key, message: message }.merge(extras) + end +end diff --git a/app/models/assistant/function/statement_vault_support.rb b/app/models/assistant/function/statement_vault_support.rb new file mode 100644 index 000000000..a24bfac1f --- /dev/null +++ b/app/models/assistant/function/statement_vault_support.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +# Shared plumbing for the Statement Vault tools. The vault is reachable from the +# web UI through AccountStatementsController, which enforces the manager role and +# per-account permissions with before_actions. MCP tool calls never pass through +# a controller, so every tool re-checks the same rules here rather than trusting +# that the caller came from a gated surface. +module Assistant::Function::StatementVaultSupport + private + def statement_manager? + AccountStatement.statement_manager?(user) + end + + def not_a_statement_manager + error( + "forbidden", + "This user is not allowed to manage the Statement Vault. Statement access requires an admin or member role." + ) + end + + def find_statement(id) + return nil unless id.present? && valid_uuid?(id) + + family.account_statements.find_by(id: id) + end + + # Identity and provenance first: the fields an agent needs to cite a document + # and decide whether to fetch it. Excerpts are the vector store's job. + def statement_payload(statement) + { + id: statement.id, + filename: statement.filename, + content_sha256: statement.content_sha256, + content_type: statement.content_type, + byte_size: statement.byte_size, + period_start_on: statement.period_start_on&.iso8601, + period_end_on: statement.period_end_on&.iso8601, + currency: statement.statement_currency, + opening_balance: statement.opening_balance&.to_s, + closing_balance: statement.closing_balance&.to_s, + review_status: statement.review_status, + account: account_ref(statement.account), + suggested_account: account_ref(statement.suggested_account), + match_confidence: statement.match_confidence&.to_f, + institution_name_hint: statement.institution_name_hint, + account_last4_hint: statement.account_last4_hint, + uploaded_at: statement.created_at.iso8601 + }.compact + end + + def account_ref(account) + return nil unless account + + { id: account.id, name: account.name, currency: account.currency } + end + + def error(key, message, extras = {}) + { success: false, error: key, message: message }.merge(extras) + end + + def parse_date(value) + return nil if value.blank? + + Date.iso8601(value.to_s) + rescue Date::Error + nil + end +end diff --git a/app/models/assistant/function/upload_account_statement.rb b/app/models/assistant/function/upload_account_statement.rb new file mode 100644 index 000000000..dd7b6b422 --- /dev/null +++ b/app/models/assistant/function/upload_account_statement.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true + +class Assistant::Function::UploadAccountStatement < Assistant::Function + include Assistant::Function::StatementVaultSupport + + class << self + def name + "upload_account_statement" + end + + def description + <<~INSTRUCTIONS + Store a statement document (PDF, CSV or XLSX) in the family's Statement Vault, + the canonical archive of primary-source financial documents. + + The vault keeps the original bytes and indexes them by SHA-256, so uploading + the same file twice is safe: the existing statement is returned with + `duplicate: true` and nothing new is created. + + On upload the vault reads the document's period, institution and account + hints, and proposes a matching account with a confidence score. It does NOT + link the statement to that account — linking is a human decision made in + Settings -> Statement Vault. Report the suggestion; don't claim the link. + + Provide the file as base64 in `content_base64`. Maximum size is 25 MB. + + Example: + + ``` + upload_account_statement({ + filename: "private-bank_2026-03-31_monthly-statement.pdf", + content_base64: "JVBERi0xLjcK..." + }) + ``` + INSTRUCTIONS + end + end + + def strict_mode? + false + end + + def params_schema + build_schema( + required: %w[filename content_base64], + properties: { + filename: { + type: "string", + description: "Filename including extension. Must be one of: #{AccountStatement::ACCEPTED_FILE_EXTENSIONS.join(", ")}." + }, + content_base64: { + type: "string", + description: "The document's raw bytes, base64-encoded." + }, + account_id: { + type: "string", + description: "Optional account UUID to link the statement to on upload. Only pass this when the user has told you which account the document belongs to — otherwise omit it and let the vault suggest a match for the user to confirm." + } + } + ) + end + + def call(params = {}) + return not_a_statement_manager unless statement_manager? + + filename = params["filename"].to_s.strip + return error("filename_required", "Please provide a filename with an extension.") if filename.blank? + + unless AccountStatement::ACCEPTED_FILE_EXTENSIONS.include?(File.extname(filename).downcase) + return error( + "unsupported_file_type", + "The Statement Vault accepts #{AccountStatement::ACCEPTED_FILE_EXTENSIONS.join(", ")} files only.", + accepted_extensions: AccountStatement::ACCEPTED_FILE_EXTENSIONS + ) + end + + content = decode_content(params["content_base64"]) + return error("invalid_content", "content_base64 could not be decoded as base64.") if content.nil? + return error("empty_file", "The decoded file is empty.") if content.empty? + + if content.bytesize > AccountStatement::MAX_FILE_SIZE + return error( + "file_too_large", + "The file is #{content.bytesize} bytes; the maximum is #{AccountStatement::MAX_FILE_SIZE} bytes." + ) + end + + account = nil + if params["account_id"].present? + account = family.accounts.writable_by(user).find_by(id: params["account_id"]) if valid_uuid?(params["account_id"]) + + unless account + return error( + "account_not_found", + "No writable account matched that account_id. Omit account_id to upload the statement unlinked and let the vault suggest a match." + ) + end + end + + prepared = AccountStatement.prepare_upload!(upload_for(content, filename)) + statement = AccountStatement.create_from_prepared_upload!(family: family, account: account, prepared_upload: prepared) + + { + success: true, + duplicate: false, + statement: statement_payload(statement), + message: "Stored #{statement.filename} in the Statement Vault." + } + rescue AccountStatement::DuplicateUploadError => e + duplicate_response(e.statement) + rescue AccountStatement::InvalidUploadError + error( + "invalid_file", + "The file failed validation: its contents don't match its extension, or it isn't a readable #{AccountStatement::ACCEPTED_FILE_EXTENSIONS.join("/")} document." + ) + rescue ActiveRecord::RecordInvalid => e + error("validation_failed", e.record.errors.full_messages.join("; ")) + end + + private + # The existing copy may be filed against an account this user cannot see, so + # the dedup result is reported without the details that would disclose it. + def duplicate_response(statement) + if statement.viewable_by?(user) + { + success: true, + duplicate: true, + statement: statement_payload(statement), + message: "This document is already in the vault (same SHA-256). Returning the existing statement; nothing was created." + } + else + { + success: true, + duplicate: true, + statement: { content_sha256: statement.content_sha256 }, + message: "This document is already in the vault, filed against an account this user cannot see. Nothing was created." + } + end + end + + # Whitespace is stripped first because agents routinely wrap long base64 + # across lines, but decoding stays strict after that: Base64.decode64 quietly + # discards characters it doesn't understand, which would archive corrupted + # bytes under a hash that looks perfectly legitimate. + def decode_content(value) + return nil if value.blank? + + Base64.strict_decode64(value.to_s.gsub(/\s+/, "")) + rescue ArgumentError + nil + end + + # AccountStatement.prepare_upload! expects an uploaded-file-like object so it + # can stream, size-check and sniff the content type. Reusing it (rather than + # building a PreparedUpload by hand) keeps the MCP path under exactly the same + # validations as the web upload form. Content type is left nil on purpose so + # the vault sniffs it from the bytes rather than trusting the caller. + def upload_for(content, filename) + DecodedUpload.new(StringIO.new(content), filename) + end + + class DecodedUpload + attr_reader :original_filename, :content_type + + def initialize(io, filename, content_type = nil) + @io = io + @original_filename = filename + @content_type = content_type + end + + def read(*args) + @io.read(*args) + end + + def rewind + @io.rewind + end + + def size + @io.size + end + end +end diff --git a/app/models/provenance/citation.rb b/app/models/provenance/citation.rb new file mode 100644 index 000000000..ad35563a8 --- /dev/null +++ b/app/models/provenance/citation.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +# Value object for the citation grammar that agent-recorded values must follow: +# +# source := ["estimated: "] citation [" (grade: " ("A"|"B"|"C") ")"] +# +# e.g. "Private bank integrated statement 2026-03-31, category subtotals (grade: A)" +# "estimated: linear interpolation over 2024-08 / 2024-12 anchors (grade: C)" +# +# A citation is the only thing separating a sourced number from an invented one, +# so the grammar is parsed rather than trusted. A free-styled string is rejected +# at the write boundary instead of landing in the ledger looking authoritative, +# and an estimate can never lose its marker on the way in. +module Provenance + class Citation + ESTIMATED_PREFIX = "estimated: " + GRADES = %w[A B C].freeze + MIN_TEXT_LENGTH = 3 + MAX_LENGTH = 500 + + FORMAT = /\A(?estimated:\s)?(?.+?)(?:\s\(grade:\s(?[ABC])\))?\z/ + # Matched separately so "(grade: D)" fails loudly instead of folding into the + # citation text and passing as an ungraded — but plausible-looking — source. + GRADE_SUFFIX = /\(grade:\s*(?[^)]*)\)\s*\z/ + ESTIMATED_MARKER = /\Aestimated\s*:/i + + class InvalidError < StandardError; end + + attr_reader :raw, :text, :grade + + class << self + # Returns a Citation, or raises InvalidError with a message written for the + # caller that has to fix it (an agent, usually). + def parse!(raw) + value = raw.to_s.strip + + raise InvalidError, "source citation is required" if value.blank? + raise InvalidError, "source citation must be #{MAX_LENGTH} characters or fewer" if value.length > MAX_LENGTH + + if value.match?(ESTIMATED_MARKER) && !value.start_with?(ESTIMATED_PREFIX) + raise InvalidError, "estimated values must start with the exact prefix #{ESTIMATED_PREFIX.inspect}" + end + + if (suffix = value.match(GRADE_SUFFIX)) && !GRADES.include?(suffix[:grade]) + raise InvalidError, "reliability grade must be one of #{GRADES.join(", ")} (got #{suffix[:grade].inspect})" + end + + match = value.match(FORMAT) + raise InvalidError, "source citation does not match #{grammar}" unless match + + text = match[:text].to_s.strip + if text.length < MIN_TEXT_LENGTH + raise InvalidError, "source citation must name the document it came from" + end + + estimated = match[:estimated].present? + grade = match[:grade] + + if estimated && grade.blank? + raise InvalidError, "estimated values must carry a reliability grade, e.g. \"#{ESTIMATED_PREFIX}... (grade: C)\"" + end + + new(raw: value, text: text, estimated: estimated, grade: grade) + end + + def valid?(raw) + parse!(raw) + true + rescue InvalidError + false + end + + def grammar + %(["#{ESTIMATED_PREFIX}"] citation [" (grade: #{GRADES.join("|")})"]) + end + end + + def initialize(raw:, text:, estimated:, grade:) + @raw = raw + @text = text + @estimated = estimated + @grade = grade + end + + def estimated? + @estimated + end + + # Reliability C is a standing TODO: a proxy or assumption that should be + # re-derived from a primary source rather than left to age in place. + def proxy? + grade == "C" + end + + def to_s + raw + end + + def to_h + { source: raw, citation: text, estimated: estimated?, grade: grade } + end + end +end diff --git a/docs/hosting/ai.md b/docs/hosting/ai.md index 32912fc1d..98d0d523a 100644 --- a/docs/hosting/ai.md +++ b/docs/hosting/ai.md @@ -450,6 +450,21 @@ EXTERNAL_ASSISTANT_AGENT_ID=your-agent-name EXTERNAL_ASSISTANT_URL=http://my-agent.my-namespace.svc.cluster.local:18789/v1/chat/completions ``` +#### Giving the agent a long-term memory of its own + +An external agent can do more than answer questions about the current balance: +with its own repository behind it, it can maintain a document-backed history of +a family's wealth, where every figure traces back to the statement it came from. + +That is a two-install shape — Sure as the system of record, the agent harness as +the model and the compiler — rather than a feature inside Sure. Sure exposes a +set of preview MCP tools for it (the Statement Vault, coverage gaps, and +valuations that require a source citation). + +See [Patrimonial history with an external agent harness](../llm-guides/patrimonial-agent-harness.md) +for which side owns what, and [the blueprint](../llm-guides/patrimonial-blueprint.md) +it implements. + ### Security with Pipelock When [Pipelock](https://github.com/luckyPipewrench/pipelock) is enabled (`pipelock.enabled=true` in Helm, or the `pipelock` service in Docker Compose), all traffic between Sure and the external agent is scanned: diff --git a/docs/hosting/mcp.md b/docs/hosting/mcp.md index 39b60bd8e..7acc55964 100644 --- a/docs/hosting/mcp.md +++ b/docs/hosting/mcp.md @@ -114,6 +114,26 @@ The MCP endpoint exposes these financial tools: These are the same tools used by Sure's builtin AI assistant. +### Preview Tools + +These additional tools appear only when the MCP user has opted into preview +features (Settings → Preferences). Until then they are absent from `tools/list`, +and calling one by name returns an "Unknown tool" error. The Statement Vault +tools additionally require the user to be an admin or member, matching the +permissions enforced in the web UI. + +| Tool | Description | +|------|-------------| +| `upload_account_statement` | Store a statement document (PDF/CSV/XLSX) in the Statement Vault; deduplicates by SHA-256 | +| `list_account_statements` | List vault documents with their SHA-256, period, linked account and review status | +| `get_account_statement` | One statement's details, its reconciliation checks against the ledger, and a short-lived download URL | +| `get_statement_coverage` | Month-by-month statement coverage for an account: covered, missing, mismatched, ambiguous | +| `record_valuation` | Record an account's value on a date, with a required source citation | + +They exist for agents that maintain a document-backed record of a family's +wealth over time. See +[Patrimonial history with an external agent harness](../llm-guides/patrimonial-agent-harness.md). + ## Example Requests ### Initialize diff --git a/docs/llm-guides/patrimonial-agent-harness.md b/docs/llm-guides/patrimonial-agent-harness.md new file mode 100644 index 000000000..0b75c9075 --- /dev/null +++ b/docs/llm-guides/patrimonial-agent-harness.md @@ -0,0 +1,210 @@ +# Patrimonial history with an external agent harness + +How to run a provenance-first patrimonial model — one where every number walks +back to the document it came from — on top of Sure, without putting that model +inside Sure. + +The model itself is specified by +[the patrimonial blueprint](patrimonial-blueprint.md). This guide is the seam: +which half owns what, which MCP tool serves which layer of the blueprint, and +which invariants Sure cannot enforce for you. + +## The two-install shape + +**Sure** is the system of record. Accounts, values and statement documents live +here, and the agent's job against Sure is to keep them clean, complete and +cited. + +**The harness** is your own repository, driven by an agent (OpenClaw, Claude +Code, anything that speaks MCP). It holds the patrimonial memory: the position +catalog, the fiscal criteria, the numbered-delta compiler, the golden tests and +the compiled workbook. It reads and writes Sure over `/mcp`. + +```text + your repo (the harness) Sure + ─────────────────────── ──── + catalogs/ positions, criteria, FX ──MCP──► accounts, entries, valuations + fiscal_data/*.jsonl Statement Vault (documents + sha256) + data/YYYY-MM/*.csv (snapshot) ◄─MCP─── coverage + reconciliation checks + src/NNN_*.py → workbook.xlsx + tests/ goldens +``` + +Sure is deliberately not the compiler. The blueprint's §2 principle 1 — the +workbook is a build output regenerated from source — only works if the source is +versioned, diffable and immutable once closed. That is what your git repo is +for. + +## Who owns which layer + +| Blueprint layer | Owner | In Sure | +|---|---|---| +| L0/L1 entities, holders | Sure, partially | `Family`, `User`, `accounts.owner_id`. Holders that are not Sure users — a holding company, a trust — have no home here; keep them in the harness's entity registry. | +| L1 account registry | **Sure** | `Account` + its `accountable`. Off-bank positions model well as `OtherAsset` or `Property` accounts. | +| L1 positions, parameters, FX table, fiscal criteria | **Harness** | — | +| L2 patrimonial series | **Sure** is authoritative | `Entry` / `Valuation`, `Holding`. The harness snapshots it into `data/YYYY-MM/*.csv` so `git diff` and the goldens have something to bite on. | +| L2 fiscal series — criterion, `applicable`, `declared`, `legal_max` | **Harness only** | Sure has one value per account per date. "One value, one criterion" (§2 principle 6) has no representation here and should not be forced into one. | +| L3 vault — account statements | **Sure** | The **Statement Vault**: original bytes in storage, SHA-256 dedup, period detection, account matching, review queue. | +| L3 vault — tax returns, annual accounts, capital accounts, contracts, minutes, email | **Harness** | The Statement Vault is statement-shaped and accepts PDF/CSV/XLSX only. Keep other primary sources in the harness's own git-ignored vault. | +| `_control.csv` / reconcile-or-abort | **Sure** | `get_account_statement` returns reconciliation checks against the ledger. | +| Gap map / `PENDING` policy | **Sure** | `get_statement_coverage` reports covered / missing / mismatched / ambiguous per month. | +| Numbered deltas, goldens, workbook, divergences report | **Harness only** | — | + +### Two invariants Sure cannot give you + +- **Closed periods are immutable** (§2 principle 7). A Postgres row is mutable + and keeps no history you can diff. Immutability lives in the harness snapshot + and its commits. +- **Golden tests** (§14). Same reason: pin row counts and snapshot totals in the + harness, against the snapshot, not against a live query. + +Treat Sure as a source you re-derive from, not as the archive of what you +already derived. + +## The tools + +These are preview features. Enable them per user in **Settings → Preferences**; +until then they do not appear in `tools/list` and calling one by name returns +"Unknown tool". The MCP user must also be an admin or member — the Statement +Vault is closed to guests, over MCP exactly as in the UI. + +| Tool | Use it for | +|---|---| +| `upload_account_statement` | Ingest a statement (PDF/CSV/XLSX, ≤25 MB, base64). Returns the SHA-256. Re-uploading identical bytes returns the existing record with `duplicate: true` — dedup is free and idempotent, so a re-run is safe. | +| `list_account_statements` | The vault index and the review queue. Filter by account, period, `review_status`, or `content_sha256` to check whether a document is already archived. | +| `get_account_statement` | One document: identity, the balances read off it, the reconciliation checks against the ledger, and a 15-minute download URL. | +| `get_statement_coverage` | The month-by-month gap map for an account: `covered`, `missing`, `mismatched`, `ambiguous`, `duplicate`, `not_expected`. | +| `record_valuation` | Write a value for a date, with a mandatory citation. | +| `search_family_files` | Semantic search inside uploaded documents (needs a vector store configured). Complements the vault: identity from `list_account_statements`, contents from here. | +| `get_accounts`, `get_holdings`, `get_balance_sheet`, `get_transactions` | Pulling L1/L2 into the harness snapshot. | + +### The citation grammar + +`record_valuation` requires a `source` and parses it. This is the one place Sure +can enforce §2 principle 2 — never invent a datum — so it fails loud rather than +storing an uncited number: + +```text +source := ["estimated: "] citation [" (grade: A|B|C)"] +``` + +- `A` — an official document for that exact date. +- `B` — derived with a document. +- `C` — a proxy or assumption: a standing TODO to re-derive from the real source. +- `estimated: ` — interpolated or proxied rather than read off a document. + Estimates must carry a grade. + +Valid: + +```text +Private bank statement 2026-03-31, securities subtotal (grade: A) +estimated: linear interpolation over 2024-08 / 2024-12 anchors (grade: C) +``` + +Rejected: a blank citation, `(grade: D)`, `Estimated:` with a capital E, and an +`estimated:` value with no grade. The citation is stored on the entry's notes, +so it travels with the value and shows in the UI. + +### What the agent does not get to do + +`link` and `reject` are **not** exposed. Attaching a statement to an account is +the human sign-off step of §8.3, and §17 keeps external actions with the owner. +Sure already proposes a match — `suggested_account` with a confidence score, +computed at upload — and the user confirms it in Settings → Statement Vault. + +Report the suggestion. Do not describe a suggested match as a link. + +## Vocabulary map + +Reading the blueprint against this codebase: + +| Blueprint | Sure | +|---|---| +| `{inbox}` / staging inbox | statements with `review_status: "unmatched"` | +| triage (mechanical-first) | `AccountStatement::MetadataDetector` — period, institution, last-4 from filename and contents | +| triage proposal | `AccountStatement::AccountMatcher` → `suggested_account` + `match_confidence` | +| dry-run manifest + human OK (§8.3) | the link/reject step in Settings → Statement Vault | +| sha256 index (§8.2) | `account_statements.content_sha256`, unique per family | +| `TOKEN_YYYY-MM-DD_Title.ext` (§8.1) | not enforced; the SHA-256 is the stable identifier, and filenames are free text | +| `_control.csv` official total | `opening_balance` / `closing_balance` read off the statement | +| reconcile-or-abort (§7 pass 3) | `reconciliation_checks`, tolerance 0.01, reported as `matched` / `mismatched` | +| gap policy `PENDING` (§13) | coverage status `missing` | +| reliability grade (§13.4) | the `(grade: A\|B\|C)` suffix on `record_valuation`'s `source` | + +## The monthly runbook, against Sure + +Blueprint §15.2, rewritten: + +1. **Ingest.** `upload_account_statement` for each new document. A `duplicate: + true` response means it was already archived — that is a normal outcome, not + an error. +2. **Hand off the match.** Report each unmatched statement and its suggested + account to the user; they confirm in Settings → Statement Vault. Do not + proceed as if the link exists. +3. **Reconcile.** `get_account_statement` on each new statement. A `mismatched` + check means the ledger and the document disagree — stop and report it. Never + adjust the figure to make it agree. +4. **Close the gaps.** `get_statement_coverage` per account for the year. Every + `missing` month is either a document to go find or an explicit `PENDING` in + the harness — never a silently interpolated number. +5. **Value the off-bank positions.** `record_valuation` with a citation, one per + position that moved. +6. **Snapshot.** Pull L1/L2 into `data/YYYY-MM/*.csv` with the byte-stable + writer. A no-op re-run must produce an empty `git diff`. +7. **Goldens.** Run them. When a count moves, update the constant *with its + one-line justification*. +8. **Build.** Regenerate the workbook, recalc, confirm zero formula errors. +9. **Commit** the snapshot, the golden update, and the working-memory sync in one + commit naming the period and the reconciliation result. + +## Building the harness: the phases, remapped + +Blueprint §20 assumes you build everything. Against Sure, several phases are +already done: + +- **Phase 0 (map the sources)** — unchanged, and still the phase people skip. + `list_account_statements` and `get_statement_coverage` give you the inventory + for anything already in Sure. +- **Phase 1 (catalogs, schemas, skeleton)** — build the harness's schemas and + the snapshot writer. The account registry comes from `get_accounts` instead of + being hand-curated; keep the positions catalog local. +- **Phases 2–4 (extractors)** — mostly replaced. Sure's provider syncs and + statement parsing already produce L2. The harness's "extractor" is a + `pull_from_sure` step writing the snapshot. Write extractors only for source + families Sure does not handle. +- **Phase 5 (views)** — unchanged, harness-side. +- **Phase 6 (the vault)** — **already built** for account statements. Do not + rebuild it. Build only the vault for non-statement sources. +- **Phase 7 (fiscal layer)** — entirely harness-side. Sure has no criterion + dimension and should not grow one. +- **Phase 8 (forensics, intel, agent memory)** — entirely harness-side. Email + sweeps never touch Sure; facts they establish enter as an + `upload_account_statement` (if a document backs them) plus a + `record_valuation` citing it. + +## Known gaps + +Worth knowing before you promise the user something: + +- **Non-user holders.** Sure's holder concept is a `User` in a `Family`. A + holding company as a first-class holder needs the harness's own registry, with + its Sure accounts mapped to it. +- **Non-statement documents.** PDF/CSV/XLSX statements only. Tax returns and + capital accounts can be uploaded if they fit those formats, but the vault will + treat them as statements — period detection and account matching will produce + noise. Prefer the harness's own vault for those. +- **One value per account per date.** Storing two criteria for the same position + and date, exactly one applicable, is the harness's job. +- **Statement periods are detected, not guaranteed.** `MetadataDetector` reads + them from the filename and contents; check `period_start_on` / `period_end_on` + on the upload response before relying on coverage. + +## See also + +- [The patrimonial blueprint](patrimonial-blueprint.md) — the full spec. +- [MCP Server for External AI Assistants](../hosting/mcp.md) — endpoint, + authentication, tool list. +- [External AI Assistant configuration](../hosting/ai.md#openclaw-gateway-example) + — pointing OpenClaw at Sure. +- [Gating a preview feature](gating-a-preview-feature.md) — the toggle these + tools sit behind. diff --git a/docs/llm-guides/patrimonial-blueprint.md b/docs/llm-guides/patrimonial-blueprint.md new file mode 100644 index 000000000..777131947 --- /dev/null +++ b/docs/llm-guides/patrimonial-blueprint.md @@ -0,0 +1,1419 @@ +> [!NOTE] +> **This document specifies a system that lives outside Sure.** Sure does not +> implement the pipeline, the numbered deltas, the fiscal layer or the Excel +> generators described here, and there is no plan to. It is kept in this +> repository because it is the spec an external agent harness implements when it +> uses Sure as its system of record — having it in-repo means an agent with a +> Sure checkout can read the design it is working to. +> +> For how the two halves fit together — which layer Sure owns, which MCP tools +> serve which section, and what the harness must own itself — read +> [the patrimonial agent harness guide](patrimonial-agent-harness.md) first. +> +> The document is reproduced verbatim as supplied. It contains no real names, +> institutions, positions, amounts or identifiers. + +--- + +# Blueprint: a provenance-first patrimonial + fiscal modelling system + +> **What this is.** A reusable design document — a "meta-prompt" — for an autonomous agent (or a +> developer) who wants to replicate a system we built: an auditable model of a family's wealth and +> tax position, compiled *entirely* from primary-source documents (bank statements, tax returns, +> company balance sheets, capital accounts, emails) into two deliverables — a **patrimonial Excel** +> and a **fiscal audit Excel + divergences report** — where *every single number is traceable back +> to the document it came from*. +> +> **What this is NOT.** It contains no real names, banks, positions, amounts, account numbers, tax +> IDs, ISINs or file paths. Everything identifying is a `{PLACEHOLDER}`. This is the *how* and the +> *why*, stripped of the *what*. See §21 for the placeholder glossary. +> +> **How to use it.** Read it top to bottom once. Then treat §20 (the replication plan) as your +> execution plan: each phase has entry criteria, deliverables, and an acceptance test. Everything +> before §20 is the specification those phases implement. When in doubt, §2 (principles) wins over +> any other section. + +--- + +## Contents + +- [0. The one-paragraph domain (anonymized)](#0-the-one-paragraph-domain-anonymized) +- [1. Tech stack & system sizing (calibration)](#1-tech-stack--system-sizing-calibration) +- [2. Non-negotiable principles (the spine)](#2-non-negotiable-principles-the-spine) +- [3. Repository layout](#3-repository-layout) +- [4. The layered architecture](#4-the-layered-architecture) +- [5. The data layer in detail](#5-the-data-layer-in-detail) +- [6. Data dictionary (full column specs)](#6-data-dictionary-full-column-specs) +- [7. Schemas & the three validation passes](#7-schemas--the-three-validation-passes) +- [8. The document vault (Layer 3)](#8-the-document-vault-layer-3) +- [9. The extractor pattern (parsers)](#9-the-extractor-pattern-parsers) +- [10. The numbered-delta compiler](#10-the-numbered-delta-compiler) +- [11. The Excel generators](#11-the-excel-generators) +- [12. The fiscal layer](#12-the-fiscal-layer) +- [13. Estimation & gap policy](#13-estimation--gap-policy) +- [14. Testing & validation](#14-testing--validation) +- [15. The recurring build & the monthly runbook](#15-the-recurring-build--the-monthly-runbook) +- [16. A worked month, end to end](#16-a-worked-month-end-to-end) +- [17. Agent operating protocol (how the builder works)](#17-agent-operating-protocol-how-the-builder-works) +- [18. Lessons learned, generalized](#18-lessons-learned-generalized) +- [19. Anti-patterns (what NOT to do)](#19-anti-patterns-what-not-to-do) +- [20. Replication plan (phased, with acceptance criteria)](#20-replication-plan-phased-with-acceptance-criteria) +- [21. Glossary of placeholders and terms](#21-glossary-of-placeholders-and-terms) + +--- + +## 0. The one-paragraph domain (anonymized) + +We model the net worth of **three first-class holders** — two individuals (`{OWNER_A}`, +`{OWNER_B}`) and one holding company (`{ENTITY_C}`). Their wealth sits in: managed bank/broker +portfolios (`{BROKER}`, `{PRIVATE_BANK}`, a `{ROBO_ADVISOR}` with a separate custodian), a Lombard +credit line collateralized by those portfolios, real estate, and a long tail of **off-bank direct +holdings** (venture funds, startup equity, operating companies) valued at cost / fiscal value +rather than market. On top of the patrimonial model sits a **fiscal layer**: the value of each +direct investment at 31 December of each year, under *fiscal* valuation criteria, used to +reconstruct and cross-check the annual wealth-tax return (`{WEALTH_TAX_FORM}`). None of that domain +detail matters to replicate the *system* — swap it for any portfolio of heterogeneous, +document-backed assets (an art collection, a real-estate book, a corporate treasury). + +--- + +## 1. Tech stack & system sizing (calibration) + +The entire system is deliberately low-tech. Replicate the *shape*, not the brands: + +| Component | Choice | Why | +|---|---|---| +| Language | Python 3, in a mandatory `venv`, deps pinned in `requirements.txt` | ubiquitous; agents write it well | +| Workbook generation | `openpyxl` | writes named tables + formulas without Excel installed | +| Formula verification | headless LibreOffice recalc script | proves **zero formula errors** before shipping | +| Schema validation | `jsonschema` (draft-07) | one contract per table, enforced pre-compile | +| PDF text extraction | `pdftotext` (poppler) + a custom glyph decoder for pathological PDFs (§9.2) | covers ~all statements | +| Email forensics | a read-only IMAP CLI | reconstructs facts that exist only in correspondence (§9.3) | +| Storage | CSV + JSON/JSONL, plain text, in git | `git diff` *is* the changelog (§5.3) | +| Orchestration | two plain Python scripts (`build_all`, `fiscal_runner`) | no framework; order is explicit | + +Orders of magnitude that this design comfortably handles (so you can tell whether you are in the +same regime — if you are 100× bigger, revisit §5.3): + +- ~40 accounts in the registry, 3 holders, ~30 off-bank positions. +- ~40 monthly periods, ~400 value rows, ~750 workbook formulas. +- ~1,200 documents in the vault, indexed by sha256. +- ~130 fiscal valuation rows across 7 tax years. +- Full rebuild from sources: seconds to low minutes. Test suite: ~100 tests, under a minute. + +CLI surface of the finished system (the *entire* operational interface): + +``` +python src/tools/build_all.py # regenerate EVERYTHING from sources → patrimonial .xlsx +python src/runner.py --check # fast gate: validate catalogs + data, write nothing +python src/runner.py # validate, then compile the workbook +python src/tools/extract_{source}.py # one extractor: diagnostic (parse+check, no write) +python src/tools/extract_{source}.py --write # …and persist to the data layer +python src/tools/fiscal_runner.py # fiscal pipeline → audit .xlsx + divergences.md +python src/tools/vault_ops_cli.py plan … # vault mutation → dry-run CSV manifest +python src/tools/vault_ops_cli.py --apply … # execute a human-approved manifest +python -m unittest discover -s tests # golden + schema + (where sources exist) re-parse tests +``` + +--- + +## 2. Non-negotiable principles (the spine) + +These are the invariants. Everything else is implementation. If a replica keeps only these, it is +already 80% of the value. + +1. **Source vs compiled artifact.** The Excel is a *build output*, like a binary. It is never + edited by hand and never the source of truth. The source is: **immutable numbered deltas** + (`NNN_*.py`) + a **data layer** (catalogs in JSON, time series in CSV/JSONL). One command + regenerates the whole workbook from source. + +2. **Never invent a datum.** Every value carries `source: {document}, {date}, {reference}`. If + there is no source, there is no row — the gap is recorded *explicitly* as `PENDING`, never + silently filled. "Not on record" is a valid, necessary answer. + +3. **Reconcile-or-abort.** Wherever an official total exists (a tax-form summary box, a + balance-sheet subtotal, a report's grand total), the parser cross-checks the sum of its parsed + parts against it — to the cent, within a tiny tolerance — or **aborts with context** (which + file, which field). No malformed period ever passes silently. + +4. **One grain, chosen to survive future questions.** The atomic row of the values table is + `account × month × asset-class`. Fine enough that any scenario or aggregation is a `GROUP BY`, + never a re-parse. Pick the grain that blinds you to no future question you can foresee. + +5. **Tidy data; totals only in views.** Data sheets are long-format: no merged cells, no totals, + no colors, pure numbers, enum-constrained columns, named tables, zero formula errors. Totals + and cross-tabs live only in dedicated View sheets, computed by formula. + +6. **One value, one criterion.** (Fiscal layer.) The same position at the same date may + legitimately have *several* values under different valuation criteria (theoretical book value + vs earnings capitalization; nominal vs NAV; with/without equity kickers). Store them as + **separate rows**, exactly one flagged `applicable=true`, the rest as documented alternatives. + This captures judgement calls natively instead of burying them in code. + +7. **Closed periods are immutable.** Once a month/year is reconciled and committed, its files + never change. New data goes in new period directories; a correction to a closed period is a + new commit with an explained golden-test update, never a silent edit. + +8. **Provenance beats convenience — re-derive from the original, never freeze an estimate.** When + a value was once entered by hand or proxied from a weaker source, the fix is to *re-derive it + from the primary source*, not to prettify the hand-entered number. A staging inbox is only ever + an inbox; the recurring pipeline reads exclusively from the canonical store. + +9. **"The document exists" ≠ "the fact happened."** Always distinguish the two, especially when + reconstructing history from correspondence. A statement mentioning a sale is not the sale. + +10. **Fail loud, in context.** Parsers abort with `{file} + {field}` on any layout change, never + an anonymous stack trace. A silent wrong number is the only unacceptable outcome. + +--- + +## 3. Repository layout + +All names below are English; use them verbatim in a replica (the original system used +Spanish-language names — a translation table is in §21 in case you ever read its docs). + +``` +{repo}/ +├── CLAUDE.md / AGENTS.md # working memory of the agent building the model (see §17) +├── requirements.txt # pinned deps; a venv is mandatory +├── catalogs/ # LAYER 0-1: static catalogs (JSON), hand-curated +│ ├── accounts.json # account registry (the master data every series row points at) +│ ├── entities.json # entity/counterparty registry (holders, issuers, aliases) +│ ├── parameters.json # constants: FX table, credit params, migration dates, off-bank costs +│ ├── positions.json # fiscal: static identity of each direct holding +│ ├── positions_intel.json # agent-facing dossiers — working state, not a true catalog (§17) +│ └── sources.json # provenance registry: one entry per source document +├── data/ # LAYER 2: patrimonial time series, one directory per period +│ └── YYYY-MM/ +│ ├── values.csv # account × date × asset-class → value +│ ├── flows.csv # dated flows (internal transfers vs external in/out) +│ ├── costs.csv # fees/costs (explicit vs estimated) +│ ├── debt.csv # debt (Lombard drawn, limit, interest, collateral) +│ └── _control.csv # per-account official report total → reconcile-or-abort input +├── fiscal_data/ # LAYER 2 (fiscal): JSONL series +│ ├── valuations.jsonl # position × date × criterion → value + full provenance +│ ├── events.jsonl # fiscally-relevant events (sales, calls, filings…) +│ └── documents.jsonl # LAYER 3 index: every vault document by sha256 +├── schemas/ # JSON Schemas — the contract for every catalog & CSV/JSONL +│ ├── *.schema.json +│ └── fiscal/*.schema.json +├── src/ +│ ├── NNN_*.py # numbered build deltas, immutable once consolidated (§10) +│ ├── runner.py # validates everything, then compiles the workbook +│ ├── lib/ # shared library: schema_cols, csv_out, naming, vault_ops, errors… +│ └── tools/ # extractors (source → data layer) + orchestrators + fiscal tools +├── tests/ # golden tests + schema validation + extractor re-parse tests +├── docs/ # design docs, source map, runbooks, decision log (this file) +└── {vault}/ # the document vault — SOURCES, git-ignored (§8) + └── {inbox}/ # staging inbox for un-triaged documents, git-ignored +``` + +**Golden rule of the layout:** `src/`, `catalogs/`, `schemas/`, `data/`, `fiscal_data/`, `tests/`, +`docs/` are versioned and carry *no secrets* (they carry structure, surrogate IDs and derived +numbers — acceptable for a private repo; redact IDs if the repo is ever shared). The raw source +documents (`{vault}/`, `{inbox}/`) are git-ignored and never committed. + +> ⚠️ **`.gitignore` gotcha we learned the hard way:** ignore data/source directories **without a +> trailing slash** (`inbox/*` plus a tracked `!inbox/.gitkeep`), and **never `git add -A` when a +> worktree contains symlinks into those dirs** — a trailing-slash ignore does not match a symlink, +> and a later merge can treat the ignored real directory as disposable and delete it. If you have +> no git remote, keep a `git bundle` backup outside the working tree and refresh it at every batch +> close. + +--- + +## 4. The layered architecture + +Think of it as a compiler with several front-ends (parsers) and two back-ends (Excel generators). + +```mermaid +flowchart TD + subgraph SRC["Layer 3 — primary sources: immutable, git-ignored"] + direction LR + S["bank statements · tax returns · balance sheets · capital accounts · emails · dashboards"] + end + SRC -->|"extractors (front-ends): parse → reconcile-or-abort → emit tidy rows"| DL + + subgraph DL["Layers 0-2 — data layer: versioned, plain text"] + direction LR + C["catalogs/*.json
static master data"] + V["data/YYYY-MM/*.csv
patrimonial series"] + F["fiscal_data/*.jsonl
fiscal series"] + end + DL --> RUN{"runner: schema +
referential + reconciliation
all valid?"} + RUN -->|no| ABORT[["Abort with context
(file + field)"]] + RUN -->|yes| DELTAS["NNN_*.py deltas
applied in order to one openpyxl Workbook"] + DELTAS --> ART + + subgraph ART["Compiled artifacts (back-ends)"] + direction LR + A1["{Patrimonial}.xlsx
data sheets + View sheets"] + A2["{Fiscal audit}.xlsx
+ divergences.md"] + end + ART -->|"headless recalc → 0 formula errors"| DONE([Deliverables]) +``` + +The layer numbering used throughout this document: + +- **Layer 0** — registries of *who exists*: entities, aliases, canonical tokens. +- **Layer 1** — registries of *what exists*: accounts, positions, parameters, source documents. +- **Layer 2** — *time series*: patrimonial CSVs per month, fiscal JSONL per year-end. +- **Layer 3** — the *documents themselves*: the vault plus its sha256 index. + +Two orchestrators drive it: + +- `src/tools/build_all.py` — the recurring patrimonial pipeline: runs the extractors in dependency + order (§15), then the runner, then a best-effort headless recalc. One command regenerates + *everything*. +- `src/tools/fiscal_runner.py` — the fiscal pipeline: parses tax returns + balance sheets, builds + the fiscal data layer, cross-checks against the official tax-form summary box, compiles the + audit Excel and the divergences report. + +--- + +## 5. The data layer in detail + +### 5.1 Catalogs (JSON) — hand-curated master data + +Catalogs are the *only* place a human curates. They change rarely, are small, and are +schema-checked. Design choices worth copying: + +- **Account registry** (`accounts.json`), one object per account. The canonical column order for + the corresponding workbook sheet lives in **one** module (`lib/schema_cols.py`) so the + header-writer (delta `001`) and the data-loader (delta `002`) can never drift. + +- **Stable surrogate IDs.** `{BANK}-{HOLDER}` for bank accounts (the holder token makes them + unique); `DIRECT-{HOLDER}-{POSITION}` for off-bank holdings — one account per position. IDs + never change once issued; renames happen in display fields, not keys. + +- **`tracked` boolean.** Spending/destination accounts exist in the registry as flow targets but + are *not* part of net worth. The runner rejects any value row on a non-tracked account. Don't + let plumbing accounts inflate the total. + +- **Natural key for off-bank = `(position_id, holder)`.** The *position* (the underlying asset) is + stored separately from the *holder*, so the same asset held by two holders shares one + `position_id` and is aggregable as family exposure without an ID collision. + +- **Account lifecycle is modelled, not overwritten.** When a portfolio migrates between banks, + do **not** reuse the account: close `{OLD_BANK}-{HOLDER}` on the migration date, open + `{NEW_BANK}-{HOLDER}`, and record an **internal `transfer` flow** between them dated on the real + migration date. Debt collateral references switch banks *by epoch* — the old bank's accounts are + collateral up to the migration month, the new bank's from it onward. + +- **Parameters** (`parameters.json`). Constants that would otherwise be magic numbers in code: + the FX table (rate + its own source per date), credit-line balances/limits, the migration date, + off-bank position costs. Extracting these means code carries *logic*, data carries *values* — + the same separation as everywhere else. + +- **Entity registry** (`entities.json`): every counterparty with its canonical `TOKEN` (the same + token the vault naming grammar uses, §8.1) plus known aliases. **Fiscal position registry** + (`positions.json`): see §12. + +### 5.2 Time series (CSV) — machine-written, per period + +- One directory per period (`data/YYYY-MM/`). A closed month is immutable (principle #7). +- Values are **end-of-month**; flows carry the **real operation date** (which must fall inside the + directory's month) — you need real dates to reconcile against statements. +- **`_control.csv`** per period holds the per-account grand total *as printed on the source + report*. The runner cross-checks `sum(asset-class values) == control total` per account, or + aborts. Crucially, `_control.csv` is **written by the extractor**, never by hand — which also + enforces the pipeline's build order (a downstream step that needs the control file cannot run + before the extractor that writes it). +- Modelling conventions that matter: + - **Native currency + a `currency` column.** FX conversion to the base currency happens only in + View sheets, from the FX table in `parameters.json`. The rate date is **always the row's + `date`** (the reporting month-end), never `last_valuation_date` — every row in a period must + convert at the same date or the period's totals are incomparable. Never store a converted + number as if it were native (we had USD holdings recorded as base currency — a real bug, + found late). + - **Internal flows ≠ external flows.** Contributions/redemptions/transfers move value *inside* + the perimeter; spending/income/taxes cross the *boundary*. Both are enum-constrained. An + internal transfer nets to zero — model it, but never let it move the net. + - **Estimated costs are never a cash outflow.** A fund's expense ratio (TER) is already embedded + in its NAV, so it lives in `costs.csv` with `nature=estimated`, strictly separated from + explicit, invoiced fees. Summing the two as if both were cash double-counts. + - **Beware sheet overlap.** Some outflows in `flows.csv` *are* the interest charges in + `debt.csv` (the bank sweeps interest from the account). Never sum across sheets without + de-duplicating; document any known overlap next to the data. + +### 5.3 Why CSV + JSON rather than a database + +Deliberate. Plain text = `git diff` is the changelog, every value is greppable, no migration +ceremony, and the whole model is reviewable in a PR. A database buys query power the grain already +gives you for free via `GROUP BY` in the View sheets. Choose text unless you have millions of rows. + +Two supporting library rules make plain text safe: + +- **One centralized CSV writer** (`lib/csv_out.py`): RFC-4180 quoting, floats fixed to 2 decimals, + stable row ordering — so output is **byte-stable** and a no-op re-run yields an empty `git diff`. +- **One centralized schema-column module** (`lib/schema_cols.py`): the single source of truth for + column order, imported by the CSV writer, the workbook deltas, and the schemas' test. + +--- + +## 6. Data dictionary (full column specs) + +This is the complete contract of the patrimonial series. Types are the post-parse types; every +file also validates against its JSON Schema (§7). + +### 6.1 `values.csv` — the heart of the patrimonial layer + +| Column | Type | Semantics | +|---|---|---| +| `account_id` | string, FK → accounts.json | which account this slice belongs to; must be `tracked` | +| `date` | ISO date | end-of-month of the directory's period | +| `asset_class` | enum | e.g. `fixed_income`, `equity`, `cash`, `venture_funds`, `startups`, `companies`, `real_estate` | +| `currency` | enum (`EUR`,`USD`,…) | native currency of the value | +| `value` | number | market value (banked) or cost/fiscal value (off-bank), in native currency | +| `last_valuation_date` | ISO date or null | for off-bank rows: when the underlying was last actually valued | +| `source` | string, non-empty | citation in the fixed grammar of §13.4: optional `estimated: ` prefix + citation + optional `(grade: A\|B\|C)` suffix | + +Grain: **one row per `account_id × date × asset_class`** — the runner rejects duplicates. + +The asset-class enum splits into two halves with **different valuation semantics** — keep them +distinguishable forever: *banked* classes (`fixed_income`, `equity`, `cash`) are market-valued +monthly; *off-bank* classes (`venture_funds`, `startups`, `companies`, `real_estate`) are at +cost/fiscal value with heterogeneous `last_valuation_date`s. The Consolidated View reports the two +gross subtotals separately (§11) precisely because averaging them would be a category error. + +### 6.2 `flows.csv` + +| Column | Type | Semantics | +|---|---|---| +| `date` | ISO date | **real operation date**, inside the directory's month | +| `from_account` | string or empty | source account (required for internal categories) | +| `to_account` | string or empty | destination account (required for internal categories) | +| `category` | enum | internal: `contribution`, `redemption`, `transfer` · external: `living_expenses`, `family_expenses`, `taxes`, `pension`, `special_outflow`, `external_income` | +| `amount` | number | positive; direction is given by from/to | +| `currency` | enum | | +| `description` | string | free text | +| `source` | string | citation | + +### 6.3 `costs.csv` + +| Column | Type | Semantics | +|---|---|---| +| `date` | ISO date | | +| `holder` | string | which first-class holder bears the cost | +| `institution` | string | who charges it | +| `account_id` | string, FK | | +| `cost_type` | enum | `advisory`, `management`, `custody`, `credit_interest`, `VAT`, `estimated_TER`, `tax`, `other` | +| `nature` | enum | `explicit` (invoiced, cash) vs `estimated` (embedded, e.g. TER) — never mix in sums | +| `amount` | number | | +| `currency` | enum | | +| `source` | string | citation | + +### 6.4 `debt.csv` + +| Column | Type | Semantics | +|---|---|---| +| `date` | ISO date | end-of-month | +| `account_id` | string, FK | the borrowing account | +| `drawn_balance` | number | Lombard balance drawn at month-end | +| `limit` | number or empty | credit limit if known | +| `type` | string | e.g. `lombard` | +| `spread_pct` | number or empty | pricing over the reference rate, if known | +| `interest_accrued` | number or empty | interest charged in the month | +| `interest_capitalized` | number or empty | | +| `amortization` | number or empty | | +| `collateral_accounts` | string | semicolon-list of pledged accounts **valid for that month's epoch** | +| `currency` | enum | | +| `source` | string | citation; reliability grade may ride here (§13) | + +### 6.5 `_control.csv` + +| Column | Type | Semantics | +|---|---|---| +| `account_id` | string | | +| `official_total` | number | the grand total **printed on the source report** for this account & month | +| `source` | string | the report it came from | + +### 6.6 Fiscal tables (JSONL — one JSON object per line) + +**`valuations.jsonl`** — `position × date × criterion → value + provenance`: + +| Field | Type | Semantics | +|---|---|---| +| `position_id` | string, FK → positions.json | | +| `holder` | string | | +| `ref_date` | string `^\d{4}-12-31$` | fiscal reference date: 31 December | +| `criterion` | enum | `declared`, `theoretical_book`, `earnings_capitalization`, `nav`, `cost`, `liquidation_value`, `listed_average`, `cadastral`, `nominal`, `market` | +| `native_value`, `currency` | number, enum | value in native currency | +| `fx`, `fx_source_id` | number, FK | rate used and its own provenance (e.g. central-bank fixing) | +| `base_value` | number | converted value in the base currency | +| `source_id` | string, FK → sources.json | the backing document | +| `method` | string | how the value was derived (e.g. "units × NAV from capital account Q4") | +| `reliability` | enum `A`/`B`/`C` | §13 | +| `applicable` | boolean | **exactly one `true` per (position, holder, ref_date)** — the criterion actually used | +| `declared` | boolean | whether this value appeared on a filed tax return | +| `legal_max` | boolean | whether a legal "greater-of" rule selects this row | + +**`events.jsonl`** — fiscally-relevant events that *explain deltas* between two year-end +valuations: `position_id`, `date`, `event_type` (enum: `subscription`, `capital_call`, `sale`, +`redemption`, `conversion`, `write_off`, `insolvency`, `dissolution`, `tax_filing`, …), `amount`, +`description`, `source_id`. The fiscal runner can warn when a valuation jump has no event +justifying it. + +**`documents.jsonl`** — the vault index (§8.2): `sha256`, `path` (canonical filename), `token`, +`doc_date`, `title`, `ext`, `size`, `indexed_at`. + +--- + +## 7. Schemas & the three validation passes + +Every catalog and every CSV/JSONL row validates against a **JSON Schema (draft-07)** before +anything is compiled. Example (the values row) — note the enum guards and the ISO-date pattern: + +```json +{ + "title": "values.csv row", + "type": "object", + "additionalProperties": false, + "required": ["account_id", "date", "asset_class", "currency", "value", "source"], + "properties": { + "account_id": {"type": "string", "minLength": 1}, + "date": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$"}, + "asset_class": {"enum": ["fixed_income", "equity", "cash", + "venture_funds", "startups", "companies", "real_estate"]}, + "currency": {"enum": ["EUR", "USD"]}, + "value": {"type": "number"}, + "last_valuation_date": {"type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$"}, + "source": {"type": "string", "minLength": 1} + } +} +``` + +The runner performs three passes and refuses to compile on any failure: + +1. **Schema pass** — every catalog + every data row against its schema. +2. **Referential pass** — + - every `account_id` in any series exists in the account registry; + - value rows appear only on `tracked` accounts; + - every off-bank account's `position_id` exists in `positions.json`, and every non-terminal + `positions.json` entry (lifecycle not `sold`/`redeemed`/`struck_off`) has a matching off-bank + account — the two catalogs that share the key are cross-validated, never merely co-edited; + - every enum value used is also present in the workbook's dimension sheet (so a dropdown/enum + drift between code and workbook is impossible); + - one row per `(account_id, date, asset_class)` in values; internal flow categories have both + accounts; period-directory dates match row dates. +3. **Reconciliation pass** — per account and period: `|sum(asset-class values) − official_total| ≤ + tolerance`. The default tolerance is **1.00 unit of base currency per account-period** (the + sources themselves round line by line, so exact-to-the-cent equality is not always attainable + from printed figures); a source family known to round more coarsely may override it via a + `tolerances` block in `parameters.json`. Pin the number — a tolerance that gates a hard abort + cannot be "tiny". Abort with the account, period, both numbers, and the delta. + +``` +python src/runner.py --check # the three passes, write nothing — run constantly and in CI +python src/runner.py # the three passes, then compile the workbook +``` + +`--check` is the fast gate. The full run differs only by writing the `.xlsx`. + +--- + +## 8. The document vault (Layer 3) + +The sources are hundreds to thousands of heterogeneous PDFs/spreadsheets/emails. Left as a pile +they are unusable. The vault turns the pile into an addressable, deduplicated, single-writer store. + +### 8.1 Naming grammar + +Every file in the vault obeys one grammar: `TOKEN_YYYY-MM-DD_Title.ext`, enforced by a single +module: + +```python +TOKEN_RE = re.compile(r"^[A-Z0-9][A-Z0-9-]*$") +FILE_RE = re.compile( + r"^(?P[A-Z0-9][A-Z0-9-]*)_(?P\d{4}-\d{2}-\d{2})_(?P.+)\.(?P<ext>[A-Za-z0-9]{1,6})$") +``` + +`TOKEN` is the entity/position/account this document belongs to (an account ID or a position ID, +resolved to its canonical form through the entity registry — aliases map to one token). The date +is the **document's own date**, not the ingestion date. The title is human-readable. A +`parse_filename()` / `build_filename()` pair is the *only* way names are made or read — plus a +"flex" variant that tolerates legacy names when indexing pre-existing files. + +Directory shape inside the vault is *by function/owner*, not by source or arrival order: + +``` +{vault}/ +├── Accounts/{ACCOUNT_ID}/{year}/… # statements, credit settlements, portfolio reports +├── Tax/{year}/… # filed returns, receipts, correspondence with the accountant +├── Investments/{POSITION}/… # subscription packs, capital accounts, annual accounts, decks +└── Documents/… # everything else worth keeping (contracts, simulations) +``` + +The recurring extractors read from these canonical locations via `rglob` patterns, **never from +the staging inbox**. + +### 8.2 Hash index + +`documents.jsonl` indexes every vault document **by sha256**. This gives you: free dedup (same +content = same hash regardless of name), tamper detection (a source that changes content is a +visible re-index event), and stable `source_id → file` resolution that survives renames. The index +is versioned; the documents are not. + +The vault root is resolved at runtime via a three-step fallback — environment variable → +`parameters.json` entry → conventional sibling directory — so the same code runs on any machine +without edits. + +### 8.3 Single-writer motor + dry-run manifest + +Exactly one module (`lib/vault_ops.py`) is allowed to *write* into the vault. Every mutation +(move / rename / ingest / dedupe / delete) is first emitted as a **CSV manifest in dry-run**; a +human reviews it; only then does `--apply` execute it and update the sha256 index. This is how you +do a 1,000-file reorganization without fear: you read the plan before it runs, and the plan is a +diff. + +```mermaid +sequenceDiagram + participant Agent + participant Motor as vault_ops (sole writer) + participant Human + participant Vault + Agent->>Motor: plan mutation (move / rename / ingest / dedupe) + Motor-->>Agent: CSV manifest — dry-run, nothing touched + Agent->>Human: review manifest (it's a diff) + Human-->>Agent: OK + Agent->>Motor: --apply + Motor->>Vault: execute + update sha256 index + Note over Motor,Vault: no other module may write here +``` + +Manifest columns: `operation` (`copy_to_vault`/`move`/`rename`/`delete`), `src_path`, +`dst_path`, `sha256`, `reason`. Deletions always log a reason; a deletion of something you did not +ingest yourself requires explicit human sign-off. + +### 8.4 Staging inbox + +`{inbox}/` is an inbox, nothing more. New documents land there; a triage step classifies them; the +motor ingests the keepers into the vault (updating the hash index); the rest are discarded with a +logged reason. Documents the triage cannot classify go to a review queue for a human decision — +they do not linger unclassified in the inbox. The recurring pipeline is **forbidden** from reading +the inbox — principle #8. (One deliberate exception: the fiscal *consolidator* reads the inbox, +because ingesting is its job.) + +**How triage classifies** (at ~1,000+ documents this cannot be pure manual judgement): +mechanical-first, human-confirmed. Try, in order: parse the filename with the flex grammar +(§8.1); sniff the content for an issuer name and a document date; where both fail, an LLM pass +proposes `token` + `date` + `destination` from the document's text. Every proposal becomes a +manifest row, and the human OK of §8.3 is the confirmation step — triage never writes directly. + +Both state files are append-only JSONL with minimal, auditable shapes (mirror the manifest's +spirit — enough fields to reconstruct every decision): + +- `state/triage_log.jsonl`: `sha256`, `src_path`, `decision` (`ingested`/`discarded`/`queued`), + `dst_path` (when ingested), `reason`, `decided_by` (`rule`/`llm`/`human`), `date`. +- `state/review_queue.jsonl`: `sha256`, `src_path`, `proposed_token`, `proposed_date`, + `proposed_dst`, `question` (what the human must decide), `status` (`open`/`resolved`), + `resolution`, `date`. + +Purging the inbox is **sha-safe by construction**: a staged file may be deleted only if its sha256 +already exists in the vault index, or its discard reason is logged. Never bulk-delete an inbox on +faith. + +--- + +## 9. The extractor pattern (parsers) + +One extractor per source family. Every extractor follows the same skeleton — this is the single +most copied pattern in the codebase: + +```python +# src/tools/extract_{source}.py +"""Parse {SOURCE} → rows for {sheet}. Reconcile-or-abort against {official total}. +Idempotent; runs in the recurring pipeline. Reads from the VAULT, never from staging.""" + +def parse_period(doc_path): + text = read(doc_path) # pdftotext / openpyxl / glyph-decoded stream + parts = extract_subtotals(text) # the labelled line items we care about + total_parsed = sum(parts.values()) + total_official = read_official_total(text) # the report's own grand total / summary box + if abs(total_parsed - total_official) > TOL: + abort(f"{doc_path}: reconciliation {total_parsed:.2f} != {total_official:.2f}") + return rows(parts), total_official # tidy rows + the control total + +OWNED_ACCOUNTS = {...} # the accounts THIS extractor is authoritative for — nothing else + +def merge_write(path, new_rows): + """Merge, don't overwrite: keep other extractors' rows, replace only our own.""" + kept = [r for r in csv_out.read(path) if r["account_id"] not in OWNED_ACCOUNTS] + csv_out.write(path, kept + new_rows) # centralized, byte-stable, stable row order + +def main(write): # --write persists; default = diagnostic (parse + check, write nothing) + for doc in vault.rglob(PATTERN): + rows, ctrl = parse_period(doc) + if write: + merge_write(period_dir(doc) / "values.csv", rows) + merge_write(period_dir(doc) / "_control.csv", [ctrl]) + print(f"{doc.name}: OK") # diagnostic line the golden tests grep +``` + +The control flow, drawn out — note the two exits (loud abort vs byte-stable write) and the +diagnostic-by-default fork: + +```mermaid +flowchart TD + A["for each source doc in vault (rglob)"] --> B["read + extract labelled subtotals"] + B --> C["total_parsed = sum(parts)"] + C --> D["total_official = report's own total / section subtotal"] + D --> E{"abs(parsed − official) ≤ tolerance?"} + E -->|no| F[["ABORT — loud, with file + field"]] + E -->|yes| G{"--write?"} + G -->|"no (default)"| H["print diagnostic line<br/>(golden tests grep this)"] + G -->|yes| I["merge-write, own accounts only:<br/>byte-stable rows + _control.csv"] +``` + +Properties every extractor must have: + +- **Idempotent.** Re-running produces byte-identical output. Ownership of a row (which extractor + may rewrite it) is decided by membership in the extractor's declared account mapping — never by + a fragile line-prefix heuristic on the CSV. +- **Merge, don't overwrite.** When two extractors contribute to the same period file (e.g. a + robo-advisor extractor and a private-bank extractor both writing `values.csv` for the same + month), the later one **merges its own accounts' rows into the existing file** instead of + rewriting it. We once overwrote and silently dropped another extractor's rows, orphaning their + cost entries — a golden test caught it. The skeleton's `merge_write`, keyed on the extractor's + declared `OWNED_ACCOUNTS`, *is* the enforcement — copy it literally, don't re-derive it. +- **Diagnostic mode by default.** No `--write` = parse + check + print, touch nothing. This is + what the golden re-parse tests exercise, and what you run to inspect reconciliations without + mutating the data layer. +- **Reconcile-or-abort against the source's own total.** If the source prints a grand total, use + it. If it prints only a section subtotal (e.g. a balance-sheet section), check against *that*. + Some sources allow a **double reconciliation** (e.g. `cash + securities = savings` and + `savings − credit = integrated position`) — use both; each equation is a free tripwire. +- **Centralized CSV writing** (`lib/csv_out.py`) for byte-stable output (§5.3). +- **Reads from the vault**, resolving the vault root via env var → parameter file → sibling dir. +- **Fails loud with context** on any layout change: a `safe_parse(field_name, file)` helper wraps + every fragile read so the abort message names the file and the field, not a line number in a + stack trace. + +### 9.1 The gotcha catalog (why parsers abort loudly) + +These are real failure modes, generalized. Every one produced a *plausible wrong number* before a +reconciliation or a golden test caught it. Keep this list; it is the accumulated scar tissue. + +| Gotcha | What happens | Defence | +|---|---|---| +| **Column drift** | A report silently adds a "Cost" column, so the *valuation* becomes the penultimate number on the line, not the first/last. | Never index a fixed column; locate by header, take the value *relative to* an anchor, and reconcile. | +| **Newest-first columns** | A balance sheet lists years `2024│2023│2022│2021` left-to-right; `nums[-1]` grabs the *oldest*. We shipped three-year-old values for months before this was caught. | Locate the target column by its **year header**, then verify against the section subtotal, or abort. | +| **Leap-year month-end** | Naive end-of-month arithmetic breaks in February of a leap year. | Use a calendar function for month-end, always. | +| **Duplicated lines** | Some reports print the cash line twice; re-summing leaf lines double-counts. | Prefer the labelled subtotal over re-summing leaves. | +| **Unmapped-glyph PDFs** | PDFs with subset Identity-H CID fonts and **no ToUnicode table** extract as mojibake. | A dedicated glyph decoder maps CIDs → Unicode (§9.2). | +| **Currency masquerade** | A source system with a single currency field stores USD holdings; summed as base currency they are simply wrong. | Store native + `currency`; convert only in views; never sum a mixed column blind. | +| **"Shares" vs "called capital"** | "75,000" turned out to be 75,000 *currency units of called capital*, not 75,000 shares. | Read the unit, not just the number. | +| **Proxy staleness** | A value proxied from a weaker source silently ages into a stale prior-year figure. | Re-derive from the primary source; grade reliability; never freeze the proxy (principle #8). | +| **Mis-dated transcriptions** | A hand-kept spreadsheet booked two months' interest under the wrong months (total right, distribution wrong). | Re-derive per-month figures from the statement's own settlement lines, then diff against the transcription. | +| **Nominal ≠ NAV** | A fund position declared at "number of units × 1.00" (nominal) when the capital account showed a NAV well above 1. | For fund positions, always look for the capital account; store both criteria as rows (§12). | + +### 9.2 Decoding "unreadable" PDFs + +Some statements are PDFs whose fonts are subset CID fonts (Identity-H) with **no ToUnicode CMap**, +so standard text extraction yields garbage. The fix is a small, maintained **glyph table** mapping +the font's CIDs to Unicode, applied to the decoded content stream. Build the table once by +eyeballing a known page against its rendered image; then it works for every document from that +issuer (they subset the same font). Publish the decoder as its own tool with its own tests — it is +reusable across issuers using the same font pipeline. Keep decoded text files only as *reference* +artifacts; the extractor must decode from the PDF directly at build time so no stale intermediate +can drift. + +### 9.3 Reconstructing facts from correspondence (email forensics) + +Some facts (a redemption, a conversion, a year-end value never formally certified) exist only in +email. A read-only IMAP CLI sweep reconstructs them. Hard-won rules: + +- **Read-only, always.** Preview mode only (never set the "seen" flag); never delete/move/flag/ + send; the only permitted write is downloading an attachment. +- **Never scope to a thematic folder.** Users barely file mail; folder filters produce *false + negatives*. Sweep the catch-all ("All Mail" / the general boxes), then filter locally. +- **Non-ASCII characters break IMAP SEARCH** on many servers. Search with ASCII word *roots* only + — substring matching catches the accented/inflected full word. +- **CLI flag order matters:** options *before* the positional query string, or the positional + swallows them and the command misparses silently. +- **Retry transient auth failures.** OAuth-backed IMAP may fail on the first call and succeed + after the token refresh — do not record a "no results" verdict off a first-attempt failure. +- **Dedup by Message-ID.** The same mail appears in several folders; that does not mean the fact + happened twice. +- **Many key documents are links, not attachments** (e-signature, file-transfer, data-room links) + — they expire; record the link and its state, don't assume you can re-fetch it later. +- **Distinguish "document exists" from "fact happened"** (principle #9). And don't trust + third-party company-data websites — they gave us a false insolvency flag once; use the official + registry, and only when a real email/document corroborates it. +- **Fan-out pattern:** verify the toolchain once, then run **one subagent per position in + parallel**, each with a strict output contract — the full playbook, including the prompt + template, is §9.4. +- **Keep a per-position intel dossier** (§17): before sweeping, read it (validated search terms, + known gaps, last-sweep cursor); after sweeping, update it. This turns each sweep into + compounding intel instead of repeated rediscovery. + +### 9.4 The sweep playbook: prompts, output contracts, consolidation + +This is the operational core of email forensics — the part that cannot be improvised, because +every clause below encodes a failure we actually hit. + +**Step 0 — verify the toolchain once per session, before any fan-out.** A checklist, not a vibe: + +1. List accounts; list folders per account (folder names differ per provider — never assume). +2. Run a **canary search**: a query you *know* is non-empty (e.g. the name of a position you have + already seen mail about). An empty canary means the toolchain is broken — wrong flag order, + auth failure, encoding issue — **not** that the mailbox is empty. Without a canary, a broken + toolchain and an empty mailbox are indistinguishable, and you will record false "not on + record" verdicts with fiscal consequences. +3. Confirm the CLI's flag-before-query ordering and that output parses as expected. + +Only after all three pass do subagents launch. Subagents receive the verified environment as +fact and are forbidden from re-deriving it (wasted tokens, divergent setups). + +**Step 1 — one subagent per position, with this prompt template** (anonymized; brace fields come +from the position's dossier and the catalogs): + +``` +You are sweeping email for facts about {POSITION} ({legal name}, tax ID {TAX_ID}). +Fiscal context: we need its value at 31 December of {YEARS}, with a backing document. +"Not on record" is a valid and necessary answer — never fill a gap with a guess. + +ENVIRONMENT (already verified — do not re-verify, do not deviate): +- accounts: {ACCOUNT_1} ({provider}), {ACCOUNT_2} ({provider}) +- search ONLY the catch-all folders: {folders}; never thematic subfolders +- read in preview mode only; never mark/move/flag/delete/send; + the only write allowed is downloading attachments to {download_dir} + +SCOPE (from the dossier at {dossier_path} — read it first): +- validated search terms (ASCII roots): {search_terms} +- issuer/advisor domains: {domains}; known contacts: {contacts} +- only mail after {last_sweep_cursor} unless a gap explicitly predates it +- open gaps you are trying to close: {gaps} + +METHOD: +- combine term × date-window queries; on transient auth failure, retry at least twice +- dedup by Message-ID; the same mail in two folders is ONE mail, ONE fact +- distinguish "the document exists" from "the fact happened" — a mail *mentioning* a + sale is not the sale; find the executed document or say so +- record links (e-signature / file-transfer / data-room) with their state; assume they + expire — download what you can NOW, note what you cannot + +OUTPUT CONTRACT — return exactly this structure: +1. VERDICT — one line. +2. VALUES — per year-end: value + backing document, or "not on record". +3. EVIDENCE — per fact: account, folder, date, sender, subject, Message-ID, + and a verbatim quote of the load-bearing sentence. +4. NEGATIVE SPACE — every query you ran (verbatim) that returned nothing relevant, + and where you ran it. Absence claims are only as good as this section. +5. DOSSIER DELTAS — terms that worked / failed, new domains or contacts, gotchas + ("the mail titled Q1 is actually the Q2 report"), proposed new cursor date. +6. CONFIDENCE — high / medium / low, with the reason. +``` + +Sections 4 and 5 are the ones agents skip if you let them — and they are the whole point. +Negative space is what makes a "not on record" verdict *citable* later ("we searched X, Y, Z on +{date}, nothing"); dossier deltas are what make the next sweep cheaper than this one. + +**Step 2 — consolidation (the parent agent, never the subagents):** + +1. Merge the per-position reports into one dated consolidation document at + `state/sweeps/SWEEP_{YYYY-MM-DD}_{scope}.md`: a header (date, accounts swept, toolchain + verification result, positions covered) followed by each subagent's six-part output verbatim. + This file is the sweep's audit trail — what a later session cites when it asks "did we already + look for this?". +2. Every **accepted fact** flows into the data layer — cite the strongest artifact: + - fact borne by the **email body** → a `sources.json` entry with + `location: email:{message-id}`; + - fact borne by an **attachment** → the downloaded file goes to `{inbox}/` and through the + standard §8.3 triage → manifest → human OK → `--apply` loop like any other document (the + sweep **never** writes to the vault directly; the single-writer rule has no exceptions). + Once ingested it gets a `vault:` source entry, the valuation cites *that*, and the carrying + email's Message-ID goes in the entry's `note` ("arrived via email {message-id}"). One + document = one source entry; the vault entry supersedes any provisional email entry. + Then the valuation/event row cites the `source_id`. An email-backed fact that never becomes a + cited row has not been captured — it has been read. +3. Every **dossier delta** is applied to `positions_intel.json`. +4. The cursor (`last_sweep`) advances **only after** the facts and attachments are archived — a + cursor advanced on a sweep whose output was lost silently hides that mail from every future + sweep. +5. Conflicts between subagent reports (two positions citing the same mail differently) are + resolved by re-reading the mail, not by preferring either report. + +--- + +## 10. The numbered-delta compiler + +The workbook is built by small, ordered Python "deltas": `001_…`, `002_…`, `003_…` The runner +discovers `NNN_*.py`, validates the data layer (§7), then executes each delta's `run(ctx)` in +order against one shared `openpyxl` Workbook: + +```python +class Ctx: # shared across deltas + wb, catalog, params, data, root, xb # xb = workbook-building helpers + +def discover_deltas(src): # sorted NNN_*.py + return sorted(p for p in src.iterdir() if DELTA_RE.match(p.name)) +``` + +- `001` — build the sheet skeleton, named tables, and the account-registry sheet from the catalog. + No data. +- `002` — load `data/YYYY-MM/*.csv` into the data sheets in canonical column order + (from `lib/schema_cols.py`), resize the named tables. +- `003` — the **Consolidated View** (§11). +- `004` — the **Debt/Risk View** (§11). + +Each delta starts with a docstring: purpose, date, what it touches, why. The numbered sequence +*is* the changelog of the model's construction. Once a delta is consolidated it is immutable in +spirit; with git as the authoritative changelog, later view-deltas may be edited in place (the +commit is the changelog entry) — but the numbering keeps the *build order* explicit. The runner +aborts on any delta lacking a `run(ctx)` or raising a validation error. + +--- + +## 11. The Excel generators + +Two design rules make the Excel trustworthy and diff-stable: + +1. **Sheets by function, not by entity/bank.** Entity, bank, account, class are *columns*, not + tabs. You never have a "Bank A" sheet and a "Bank B" sheet; you have one `Values` sheet with a + `bank` column. This keeps the model tidy and queryable. +2. **Totals only in Views.** Data sheets are inert tidy tables. Aggregation lives in **View + sheets** built from `SUMIFS` over the named data tables, with formula cells colored so a reader + can tell "live link" from "typed number" at a glance. + +The two views: + +- **Consolidated View:** a month × asset-class matrix with — *gross banked* (market-valued liquid + classes), *gross off-bank* (cost/fiscal-valued illiquid classes, mixed valuation dates), gross + total, the Lombard debt (`SUMIFS` over the debt sheet), and `Net = Gross − Debt`, plus a line + chart. The banked/off-bank split matters because the two halves have different valuation + semantics and you must never blur them (§6.1). +- **Debt/Risk View:** pledged collateral (`SUMIFS` over the collateral accounts — written so that + accounts which don't yet exist in a period simply sum to zero, letting the same formula survive + a bank migration with no code change), drawn Lombard, `LTV = Debt / Collateral`, monthly + interest, and an LTV chart. + + The collateral formula, worked out (month in `$A2`): enumerate the union of **every account + ever pledged**, one static `SUMIFS` term per account — + + ``` + = SUMIFS(tbl_Values[value], tbl_Values[account_id], "{OLD_BANK}-{OWNER_A}", tbl_Values[date], $A2) + + SUMIFS(tbl_Values[value], tbl_Values[account_id], "{OLD_BANK}-{OWNER_B}", tbl_Values[date], $A2) + + SUMIFS(tbl_Values[value], tbl_Values[account_id], "{NEW_BANK}-{OWNER_A}", tbl_Values[date], $A2) + + SUMIFS(tbl_Values[value], tbl_Values[account_id], "{NEW_BANK}-{OWNER_B}", tbl_Values[date], $A2) + ``` + + Terms for accounts with no rows in a month contribute zero — that zero-sum property is exactly + what makes the formula epoch-proof. The view does **not** parse the semicolon list in + `debt.csv`'s `collateral_accounts`; that column documents which epoch's accounts are actually + pledged (for the reader and the golden collateral-by-epoch test), while the view relies on the + static union of terms giving the same result with no string parsing in a spreadsheet formula. + The term list itself is **generated, not hand-typed**: at compile time the view delta iterates + the union of every account appearing in any `debt.csv` `collateral_accounts` value and emits + one `SUMIFS` term per account — a future migration adds terms by adding data, never by editing + code (per §19's ban on hardcoding derivable values; the string parsing happens in Python at + build time, where it belongs, not in the spreadsheet). + +After compiling, **recalc headlessly** (spreadsheet apps evaluate formulas on open; a headless +recalc proves **zero formula errors** before you ship). The orchestrator locates the recalc script +best-effort and runs it; a lock-file check warns if the workbook is currently open in an editor. + +> A consciously *rejected* refactor, preserved as an example of writing down roads not taken: +> converting the text dates in the data sheets to real spreadsheet dates. It would touch every +> `SUMIFS` (text↔date comparison semantics) for a purely cosmetic gain (a nicer chart axis). +> Documented as "don't do this unless there's another reason to rewrite the views." **Write down +> what you decided not to do, and why** — it saves the next agent from re-litigating it. + +--- + +## 12. The fiscal layer + +A second data layer, at year-end (31 December) and *fiscal* valuation criteria, keyed by the same +`position_id` as the off-bank holdings. It does not replace the monthly banked values — it is an +orthogonal view of the same world. Stored as JSONL because the rows are wider and more +heterogeneous than the patrimonial CSVs. + +Three catalogs/tables (full field specs in §6.6): + +- **`positions.json`** — static identity per holding: legal name, tax IDs, instrument type (which + drives which section of the tax form it belongs in), holder, ownership %, country, native + currency, flags (`listed`, `foreign_reporting_obligation`, `audited`), the tax-form section, the + accounting sub-account (for the holding company's investees), the provider account/user, and + lifecycle state (`alive` / `insolvency` / `liquidation` / `struck_off` / `sold` / `redeemed`) + with a date. Lifecycle matters fiscally: an insolvent-but-not-liquidated company may still have + to be declared at its last value. + +- **`sources.json`** — the **provenance backbone**. One entry per source document. It is a JSON + **object keyed by `source_id`** — a short, stable, human-readable slug assigned when the entry + is created, convention `{TOKEN}-{DOCTYPE}-{PERIOD}` (e.g. `{POSITION}-CAPACC-2025Q4` for that + position's Q4-2025 capital account). Every valuation's and event's `source_id` is that key; + a slug never changes once anything cites it: + + ```json + { + "required": ["type", "location"], + "properties": { + "type": {"enum": ["tax_form", "balance_sheet", "annual_accounts", "certificate", + "capital_account", "statement", "contract", "minutes", "dashboard", + "trade_confirmation", "web", "crm_export", "email", "report"]}, + "location": {"type": "string"}, + "issuer": {"type": ["string", "null"]}, + "document_date": {"type": ["string", "null"]}, + "filing_receipt_code": {"type": ["string", "null"]}, + "sha256": {"type": ["string", "null"]}, + "note": {"type": "string"} + } + } + ``` + + `location` uses a URI-ish convention: `vault:{canonical filename}` · `web:{url}` · + `email:{message-id}`. Vault citations use the **canonical filename only** (the sha index + resolves it), so citations survive directory reshuffles. + +- **`valuations.jsonl`** — the heart: `position × date × criterion → value + full provenance` + (§6.6). This is where **"one value, one criterion"** (principle #6) lives. Two worked examples, + anonymized: + + - A holding whose tax rule forces "the **greater of** theoretical book value vs earnings + capitalization" gets *two* rows for the same date; the greater one carries + `applicable=true, legal_max=true`. The comparison is explicit and auditable, not a hidden + `max()` in code. + - A fund declared at nominal (units × 1.00) whose capital account shows a NAV of 1.18 gets both + rows — `nominal` with `declared=true, applicable=true` (what was filed) and `nav` with + `applicable=false` (what it was worth). The JOIN of *declared* vs *worth* **surfaces + under-declarations automatically**; each becomes a numbered item in the divergences report. + +The **fiscal runner** (`fiscal_runner.py`): + +1. Parses each filed tax return from the vault, classifying line items into form sections by + *shape* (a line with tax-ID/ISIN + ownership % is an "identified securities" item; a bare + description + value is a residual "other assets" item) — because PDF extraction scrambles + section headers, and one filing may even be in a different co-official language. +2. **Reconciles the residual section against the form's own summary box**, and applies a **hard + floor on parsed item count** (if a return yields fewer than N items, the parse is presumed + broken and the run aborts — this guards against a silently-empty parse passing as "nothing + declared"). Derive `N` from the data, not from taste: the minimum item count across all + known-good filed years **minus one item**; recalibrate it whenever a new year is filed. +3. Reconstructs the missing year between two declared anchor years with a **fixed decision + procedure**, applied per `(position, holder)` — so two runs (or two replicators) produce the + same audit from the same filings: + 1. **Terminated?** If `events.jsonl` shows a terminating event (sale, redemption, dissolution) + before the missing ref-date → no valuation row; the event, with its own source, is the + evidence of absence. + 2. **Primary document for the missing date?** If one exists (capital account, annual accounts, + dated certificate) → recompute under the position's applicable criterion from that document + (reliability A/B; `method` records the computation). + 3. **Otherwise carry forward** the last applicable criterion and value (reliability C; + `method = "carry_forward from {year}"`), leaving the row as a standing TODO per §13. + 4. **Greater-of rules are re-evaluated** whenever step 2 supplies new inputs; both candidate + rows are always written, exactly one `applicable=true` (principle #6). + 5. **Every carry-forward vs recompute choice is logged** as a numbered item in the divergences + report — reconstruction choices are judgement calls the accountant must see. +4. Compiles the **audit Excel** (one sheet per year, value + citation side by side, `PENDING` + where nothing is on record) and the **Markdown divergences report**: every judgement call, + numbered, phrased as a question the accountant can answer (declare/not, criterion A/B, + amend/not). + +The audit artifacts are *inventories with traceability*, explicitly **not** tax filings. Keeping +that framing honest is what lets you show them to a professional advisor as input rather than as a +claimed conclusion. + +--- + +## 13. Estimation & gap policy + +Principle #2 forbids inventing data; real life still has gaps. The policy that reconciles the two: + +1. **First, exhaust recovery.** Before estimating anything, prove the primary source is + unrecoverable (portal closed, issuer never emailed it, account cancelled). Document the failed + recovery attempts next to the estimate. +2. **Then interpolate only between hard anchors, and mark it.** A gap bounded by two (better: + three) reconciled anchor dates may be filled by linear interpolation *per asset class*. Every + interpolated row carries an `estimated:` prefix in its `source` column, and a note of the + anchors used. Estimation without anchors is refused — that gap stays `PENDING`. +3. **Never let a partial month masquerade as a total.** If a period directory contains only some + accounts, the consolidated series will show a fake collapse. Either complete the month (with + marked estimates if justified) or exclude it from the view — never ship the artefact with a + silent partial. +4. **Grade reliability on every derived figure:** + - **A** — official document for that exact date (statement, filed return, capital account). + - **B** — derived with a document (e.g. a value computed from a filed return's cadastral + figure, or extrapolated one month from a dated statement). + - **C** — proxy or assumption (a hand-kept spreadsheet, a placeholder awaiting appraisal). + Record the grade in the row — fiscal layer: the `reliability` field; patrimonial layer: inside + the `source` string, which follows **one fixed grammar** so the tests can parse it: + + ``` + source := ["estimated: "] citation [" (grade: " ("A"|"B"|"C") ")"] + ``` + + e.g. `estimated: linear interpolation over 2024-08 / 2024-12 / 2025-11 anchors (grade: C)`, or + `{PRIVATE_BANK} integrated statement 2026-03-31, category subtotals (grade: A)`. The golden + marker test regex-parses exactly this grammar — free-styling the field breaks the build, by + design. + **Every C is a standing TODO** to be upgraded by re-deriving from a primary source — and when + the primary source arrives, it *replaces* the proxy (principle #8). +5. **Let the tests police the marking.** A golden invariant asserts that every + interpolated/estimated row is marked and that **no real row carries the marker** — so an + estimate can never masquerade as a hard datum, and a hard datum is never diluted into an + estimate (§14). + +--- + +## 14. Testing & validation + +Three complementary layers, all runnable with `python -m unittest discover -s tests`: + +1. **Schema + referential + reconciliation** — the runner's three passes (§7), run on every build + and in `--check`. This is validation, not testing, but it is the base of the pyramid. + +2. **Golden tests on the versioned data layer** (`tests/test_golden.py::TestDataLayer`). These + need no sources and no PDF tooling — they assert invariants on the committed `data/`: + - exact **row counts** per sheet and the **month count** — one number that changes only with a + documented reason; + - the **gross total of a snapshot month**, to the cent; + - the **per-account control totals** of that snapshot; + - **debt balances** per account (flat where proxied, real per-month where sourced); + - **collateral by epoch** (which accounts are pledged before/after the migration date); + - **marker invariants** — every estimated/interpolated row is marked `estimated` and no real + row is (§13.5); + - `runner.py --check` exits 0. + + The golden constants carry an inline comment explaining *why* each number is what it is — which + period added which rows, which bug removed which. When a change moves a golden, either it is a + regression, or you update the golden **together with its justification**. The comment block is + a mini-changelog: + + ```python + GOLDEN = { + # +{period}: +N rows ({reason}). −M rows ({bug fixed}). {value} corrected {old}→{new}. + "rows": {"values": ..., "flows": ..., "costs": ..., "debt": ...}, + "months": ..., + "gross_{snapshot}": ..., # gross total of the anchor snapshot, to the cent + "control_{snapshot}": {...}, # per-account official totals + "debt_balances": {...}, # flat where proxied + "debt_{era}": {...}, # real per-month where sourced + } + ``` + +3. **Extractor re-parse tests** (`TestExtractors`, skipped automatically unless the sources and + PDF tooling are present on the machine). These re-run each extractor in diagnostic mode and + assert its reconciliation output lines (`"OK: N"`, `"failed reconciliations: 0"`, + `"subtotal X == Y"`). They protect the *parsers* against silent regressions when a refactor + changes extraction logic. + +Why the split works: the DataLayer tests run anywhere (CI, a fresh clone, no secrets) and pin the +*output*; the Extractor tests run where the sources live and pin the *process*. An agent +replicating this should treat **"golden green" as the definition of done** for any data-layer +change — and should never change a golden constant without writing the one-line justification next +to it. + +--- + +## 15. The recurring build & the monthly runbook + +### 15.1 Build order (load-bearing) + +`python src/tools/build_all.py` runs the steps in an order where **the arrows are dependencies, +not just sequence**: + +```mermaid +flowchart LR + EV["extract_values<br/>banked portfolio values<br/>(writes values.csv + _control.csv)"] + --> EF["extract_flows<br/>flows + costs + debt<br/>(reads _control for the migration transfer)"] + EF --> EO["extract_offbank<br/>off-bank values<br/>(MERGES into shared values.csv)"] + EO --> EP["extract_{private_bank}<br/>post-migration values + real Lombard<br/>(MERGE, NOT overwrite)"] + EP --> RUN["runner<br/>3 validation passes + compile<br/>→ {Patrimonial}.xlsx"] + RUN --> RC(["headless recalc (best-effort)<br/>verify 0 formula errors"]) +``` + +Why each edge exists — encode this reasoning as comments in the orchestrator: + +- `extract_flows` after `extract_values`: the migration `transfer` amount is **read from the + snapshot month's `_control.csv`**, which `extract_values` writes. Deriving it (rather than + hardcoding it) both removes a magic number and physically enforces the order. +- `extract_offbank` after both: it merges off-bank rows into period `values.csv` files that + already exist. +- `extract_{private_bank}` last among extractors, and it **merges**: overwriting once silently + dropped the robo-advisor's rows for shared months, orphaning their cost entries — caught by a + golden row count. + +### 15.2 The monthly runbook (day-2 operations) + +When a new month's statements arrive: + +1. Drop the documents into `{inbox}/`. +2. Triage → `vault_ops` dry-run manifest → human OK → `--apply` (ingest into + `Accounts/{ACCOUNT_ID}/{year}/` with canonical names; index updates). +3. Run the relevant extractor in **diagnostic mode** (no `--write`): read the reconciliation + lines. Any abort → fix the parser or flag the source anomaly; never patch the output. +4. Re-run with `--write`, then `runner.py --check`. +5. Run the test suite. The golden row/month counts *will* move — update them **with a one-line + justification** in the golden comment block. +6. `build_all.py` → recalc → confirm zero formula errors. +7. Commit: data layer + golden update + (if the parser changed) the parser, in one commit whose + message states the period and the reconciliation result. +8. Update the working-memory doc (§17): new counts, anything learned, anything now `PENDING`. + +--- + +## 16. A worked month, end to end + +A concrete trace of the whole machine on one new statement (names are placeholders): + +1. **Arrival.** `statement_march.pdf` (from `{PRIVATE_BANK}`, for `{OWNER_A}`) lands in + `{inbox}/`. +2. **Ingest.** Triage classifies it → `vault_ops` plan emits one manifest row: + `copy_to_vault, {inbox}/statement_march.pdf, Accounts/{PRIVATE_BANK}-{OWNER_A}/2026/ + {PRIVATE_BANK}-{OWNER_A}_2026-03-31_Monthly integrated statement.pdf, {sha256}, monthly ingest`. + Human OKs; `--apply` copies it and appends to `documents.jsonl`. +3. **Parse (diagnostic).** `extract_{private_bank}.py` finds the new file via `rglob`, decodes it + (glyph decoder, §9.2), pulls the labelled subtotals: securities, cash, credit drawn. It checks + the double reconciliation — `cash + securities = savings total` and `savings − credit = + integrated position` — to the cent, and prints `2026-03 {PRIVATE_BANK}-{OWNER_A}: OK`. +4. **Write.** With `--write`, it **merges** into `data/2026-03/`: its rows in `values.csv` + (asset-class split per the report's category subtotals), its `debt.csv` row (drawn balance, + interest from the credit settlement line, collateral = this epoch's pledged accounts), and its + `_control.csv` line (the report's own printed total). +5. **Validate.** `runner.py --check`: schema pass, referential pass (account exists, is tracked, + classes in enum, no duplicate grain), reconciliation pass against `_control.csv`. Exit 0. +6. **Test.** Golden row count moves +3 → update `GOLDEN` with + `# +2026-03 {PRIVATE_BANK}-{OWNER_A}: +2 value rows, +1 debt row (monthly statement)`. +7. **Build.** `build_all.py` regenerates everything; delta `002` reloads the CSVs; the views pick + up the month via their `SUMIFS`; headless recalc reports 0 formula errors. +8. **Close.** One commit; working-memory doc updated with the new counts. The `.xlsx` ships. Every + number in it can be walked back: cell → `SUMIFS` → data sheet row → `source` column → canonical + filename → sha256 → the PDF. + +That final walk-back chain is the entire point of the system. + +--- + +## 17. Agent operating protocol (how the builder works) + +The system is built and maintained *by an agent across many sessions*. These rules are as much a +part of the design as the schemas: + +- **Working memory file** (`CLAUDE.md` / `AGENTS.md` in the repo root): current state (row counts, + reconciliation status, what is green), conventions, dated decisions, and open questions — the + first thing read in a new session. **Close every work batch by syncing it**: verify the summary + figures *against the files*, not from conversational memory; memory drifts, the data does not. +- **Decision log with dates.** Every judgement call gets a dated entry: what was decided, by whom + (the human owner decides valuation criteria and scope; the agent proposes), and why. Include + **rejected options** (§11's rejected refactor) — a documented road-not-taken prevents + re-litigation. +- **Ask, don't invent.** When a source is ambiguous (is "75,000" shares or currency?), the agent + asks the owner and records the answer as a dated decision. An invented assumption in this domain + is a future wrong tax filing. +- **Per-position intel dossiers** (`positions_intel.json`) — agent-facing, not consumed by code. + It lives in `catalogs/` for discoverability, but it is the one file there that is **not a real + catalog**: it changes with every sweep, is append-only in spirit, and is deliberately **not + schema-enforced** — operationally it belongs with `state/`, not with `accounts.json`, and the + runner's schema pass skips it. Per position: `domains` (issuer/advisor email domains), `contacts`, `search_terms` (**validated + ASCII roots** — §9.3), `gaps`, `expected_documents`, `findings[]` (dated free-text learnings), + `last_doc_date`, `last_sweep` (the cursor), `priority` (`normal` / `closed` = don't sweep). + Discipline: read the dossier *before* sweeping; *after* sweeping, + leave it better than found — append any finding whose answer to "would knowing this save time + next session?" is yes (e.g. "the email titled Q1 is actually the Q2 report", "that share link + expires", "search by tax ID, not name"). Advance the cursor **only after successfully + archiving** what was found. + + **Bootstrapping the intel layer** (how the dossiers come to exist at all): + 1. **Structure first.** Create *skeleton* dossiers for **every** position in one pass — all + keys present, values empty. Discoverability beats completeness: an agent cannot update a + dossier it doesn't know should exist, and an all-positions index makes "which positions have + no intel yet" a trivial query instead of an unknown unknown. + 2. **Populate opportunistically.** Every sweep, every parsed document, every conversation with + the owner leaves its residue in the dossier as a dated, append-only `findings[]` entry — + filtered by the single test above (would this save time next session?), anchored to a + Message-ID or `source_id` whenever possible. Never rewrite old findings; supersede them. + 3. **Harvest deliberately when it pays.** When a position accumulates open gaps, run a + dedicated harvest pass (a full §9.4 sweep scoped to that position, no cursor limit) rather + than letting five future sessions each rediscover a slice. One planned harvest is cheaper + than N interrupted rediscoveries. + 4. **Mark closure.** When a position's lifecycle ends and its fiscal history is complete, set + `priority: closed` — an explicit "do not sweep" is intel too; it prevents every future + session from re-checking a settled question. +- **A source map** (`docs/source_map.md`): which document family backs which datum, what each + family's reconciliation total is, and where the known gaps are. Written in Phase 0, kept + current. +- **Subagent fan-out with output contracts.** For per-position forensics (§9.3), one subagent per + position, in parallel, each returning: verdict, values with citations or "not on record", + verbatim quotes, negative-space report (what was searched and not found), confidence. +- **External actions are the human's.** The agent never sends email, never signs, never files + anything with an authority, and mutates the vault only through the manifest+OK loop (§8.3). +- **Definition of done, always the same:** goldens green · `--check` exits 0 · recalc shows zero + formula errors · byte-stable re-run (empty diff) · working-memory doc synced. + +--- + +## 18. Lessons learned, generalized + +Every one of these came from a real mistake. Stated as portable rules: + +1. **Fail loud beats fail silent.** In a correctness-critical domain (tax, accounting, medical), a + *silent* wrong number is the dangerous failure mode — it gets used. A controlled + abort-with-context is not a crash; it is a designed stop that costs you a build, not a wrong + filing. This trade-off *inverts* where availability outranks precision (dashboards, real-time + systems) — scope it to your domain. Note the middle path the system itself uses: an approximate + value **marked `estimated`**, rather than an abort, when a documented gap must be filled (§13). +2. **Locate by meaning, never by position.** Header names and labelled subtotals survive layout + changes; `nums[-1]` and fixed column indices do not. +3. **Provenance is a column, not a comment.** `source` on every row; a source registry with + sha256. When you cannot cite it, the row does not exist yet. +4. **Re-derive from the original; never freeze an estimate.** A hand-entered or proxied value is a + TODO to re-derive from source, not a number to preserve. +5. **Separate logic from values.** Magic numbers go to the parameters catalog. Code carries *how*; + data carries *what*. +6. **One writer for anything dangerous.** A single module owns all vault mutations, and it plans + in dry-run before it applies. Same idea, smaller scale: one centralized byte-stable CSV writer. +7. **Idempotent + byte-stable ⇒ a no-op re-run is an empty diff.** This is what makes "regenerate + everything from source" safe to run constantly — drift becomes visible as a non-empty diff. +8. **Distinguish existence from occurrence** when mining unstructured sources (principle #9). +9. **Write down the roads not taken.** A rejected refactor documented with its reasoning is as + valuable as a decision made. +10. **`.gitignore` + symlinks + `git add -A` is a footgun.** Ignore data dirs without a trailing + slash; keep a bundle backup; never blanket-add a worktree containing symlinks into ignored + source dirs. +11. **Reconcile the closing summary against the files, not your memory.** Working memory drifts; + the data does not. +12. **Merge, don't overwrite, on shared files.** When two producers write one period file, the + second must merge — and a golden count must watch the seam. +13. **Give every gap the source it deserves.** Recover the real document first; interpolate + between anchors only when recovery is proven impossible; and mark everything estimated. + +--- + +## 19. Anti-patterns (what NOT to do) + +- ❌ Editing the `.xlsx` by hand. It is a build output; the edit dies on the next build. +- ❌ A tab per bank/entity. Use columns; one sheet per *function*. +- ❌ Totals or merged cells in data sheets. Totals live in Views. +- ❌ Reading the staging inbox from the recurring pipeline. +- ❌ Fixed column indices in a parser; `nums[-1]` on a multi-year table. +- ❌ Summing a mixed-currency column without per-date FX that has its own source. +- ❌ Filling a data gap silently. Mark it `estimated`/`PENDING` and let a golden test police the + marking. +- ❌ Overwriting a shared period file when a second extractor contributes to it. +- ❌ Hardcoding a derivable number (a transfer amount, an FX rate, a credit balance) in code. +- ❌ Committing source PDFs or any real identifier into the versioned tree. +- ❌ Trusting a third-party data aggregator over the official registry. +- ❌ Changing a golden constant without a written justification. + +--- + +## 20. Replication plan (phased, with acceptance criteria) + +Build in this order. **Do not start a phase before the previous one's acceptance test passes** — +the phases are rungs, and each one's outputs are the next one's inputs. + +**Phase 0 — Map the sources.** +*Do:* inventory every source family (statements, tax returns, balance sheets, capital accounts, +emails, dashboards). For each: which official total can I reconcile against? What cadence? Which +period range? Where does it physically live? Write `docs/source_map.md`. Get the owner's decisions +on: holders, perimeter (which accounts are wealth vs plumbing), grain, base currency, series start. +*Accept when:* the source map names a reconciliation total for every family, and the open +questions are written down as questions rather than assumptions. + +**Phase 1 — Catalogs + schemas + skeleton + vault stub.** +*Do:* define the grain; write `accounts.json`, `entities.json`, `parameters.json`; write all JSON +Schemas; write `lib/schema_cols.py` and `lib/csv_out.py`; write deltas `001`/`002` and the runner +with its three passes. Also write the **vault stub**: the naming-grammar module (§8.1) plus a +flat, git-ignored canonical source directory that Phase 2-5 extractors read from — no motor, +manifest or hash index yet; Phase 6 upgrades this stub in place. (Without the stub, Phase 2's +"reads from the vault" contract has nothing to read from.) +*Accept when:* `runner.py --check` passes on an empty data layer, and the compiled workbook has +all sheets, named tables and headers, with zero data. + +**Phase 2 — First extractor (the pattern-setter).** +*Do:* pick the richest source family; implement `extract_values` per §9 (diagnostic default, +reconcile-or-abort, byte-stable `--write`, `rglob` over the Phase-1 vault stub); emit +`values.csv` + `_control.csv` for every available month; write the first golden tests (row +counts, snapshot totals, control totals) and the first re-parse test. +*Accept when:* every parsed month reconciles to the cent; goldens green; a second `--write` run +produces an empty `git diff`. + +**Phase 3 — Flows, costs, debt.** +*Do:* the flow/cost/debt extractor: internal vs external categories; explicit vs estimated costs; +debt with collateral-by-epoch; any derivable amount read from the data layer, not hardcoded. +*Accept when:* runner passes all three validation passes on the enlarged layer; goldens updated +with justifications; known sheet overlaps (interest in both flows and debt) documented. + +**Phase 4 — Off-bank / illiquid holdings.** +*Do:* parse from tax return / balance sheet / capital account; natural key +`(position_id, holder)`; cost/fiscal valuation with `last_valuation_date`; **merge** into shared +period files. +*Accept when:* the anchor snapshot's gross total matches the golden to the cent, and the newest- +first-columns defence (locate by year header + verify section subtotal) is tested. + +**Phase 5 — View sheets.** +*Do:* Consolidated (banked/off-bank split + debt + net + chart) and Debt/Risk (collateral, LTV, +interest + chart); formulas survive accounts that don't exist yet in a period; headless recalc in +the orchestrator. +*Accept when:* recalc reports zero formula errors and spot-checked view cells equal hand-computed +sums from the CSVs. + +**Phase 6 — The vault (upgrade the stub in place).** +*Do:* upgrade the Phase-1 stub into the full vault: directory shape by function/owner; sha256 +index; single-writer motor with dry-run manifest; staging inbox with triage mechanism, triage log +and review queue (§8.4); re-point every extractor at the canonical layout (env → parameter → +sibling resolution); sha-safe inbox purge rule. +*Accept when:* all extractors read only from the vault; the index covers every document; a full +rebuild from the vault is byte-identical to the pre-vault build. + +**Phase 7 — The fiscal layer.** +*Do:* `positions.json`, `sources.json`, `valuations.jsonl`, `events.jsonl` + their schemas; the +tax-return parser (classification by shape, summary-box reconciliation, hard floor on item count); +the missing-year reconstruction; the audit Excel + divergences report. +*Accept when:* every filed return reconciles against its own summary box; exactly one +`applicable=true` per (position, holder, year); every valuation resolves to a source **or is an +explicit `PENDING`** — facts that only correspondence can back may legitimately stay `PENDING` +until Phase 8's email forensics closes them (do not block Phase 7 on evidence Phase 8 collects); +the divergences report lists every judgement call as an answerable question. + +**Phase 8 — Forensics + intel + agent memory.** +*Do:* read-only email sweeps per §9.3 with subagent fan-out and output contracts; populate the +intel dossiers; establish the working-memory doc, decision log, and batch-close ritual (§17). +*Accept when:* each swept position has a dossier with validated search terms and a cursor, and +every reconstructed fact carries a citation or an explicit "not on record". + +**Continuous (every phase):** goldens green · byte-stable re-runs · one command rebuilds +everything · closed periods untouched · working memory synced against the files. + +--- + +## 21. Glossary of placeholders and terms + +| Placeholder / term | Stands for | +|---|---| +| `{OWNER_A}`, `{OWNER_B}` | the two individual holders | +| `{ENTITY_C}` | the holding company (a first-class holder with its own analysis) | +| `{BROKER}`, `{PRIVATE_BANK}`, `{ROBO_ADVISOR}` | the managed-portfolio providers | +| `{POSITION}` | a direct/off-bank holding (fund, startup, operating company) | +| `{WEALTH_TAX_FORM}` | the annual wealth-tax return being reconstructed | +| `{Patrimonial}.xlsx`, `{Fiscal audit}.xlsx` | the two compiled deliverables | +| `{vault}`, `{inbox}` | the canonical document store and its staging inbox | +| **reconcile-or-abort** | the balance check of parsed parts against an official total, aborting on mismatch | +| **golden test** | a pinned constant (row count, snapshot total) that may only change with a written justification | +| **reliability A/B/C** | official document / derived-with-document / proxy-assumption (§13) | +| **epoch** | a date range in which a structural fact holds (e.g. which accounts are pledged as collateral) | + +Because the original system was built in Spanish, its own docs and file names use Spanish terms. +If you ever read them, this maps the vocabulary (a replica should just use the English names): + +| Original (Spanish) | This document (English) | +|---|---| +| `catalogos/`, `datos/`, `datos_fiscal/` | `catalogs/`, `data/`, `fiscal_data/` | +| `valores`, `flujos`, `costes`, `deuda` | `values`, `flows`, `costs`, `debt` | +| `dimensiones` (registro de cuentas) | `accounts` (account registry) | +| `entidades`, `parametros`, `posiciones`, `fuentes` | `entities`, `parameters`, `positions`, `sources` | +| `valoraciones`, `eventos`, `documentos` | `valuations`, `events`, `documents` | +| `cuenta_id`, `titular`, `clase_activo`, `divisa`, `valor`, `fuente` | `account_id`, `holder`, `asset_class`, `currency`, `value`, `source` | +| `cuadre` | reconciliation (reconcile-or-abort) | +| `estimado` / `explicito` | `estimated` / `explicit` | +| `fiabilidad`, `aplicable` | `reliability`, `applicable` | +| `para_ordenar` | the staging inbox | +| `RF` / `RV` / `liquidez` | `fixed_income` / `equity` / `cash` | + +--- + +*This document describes the architecture, conventions, testing strategy and hard-won lessons of +the system. It intentionally omits all real names, institutions, positions, amounts and +identifiers — those live only in git-ignored source documents, never in the versioned tree. An +agent following §20 with the principles of §2 should be able to reproduce a system of the same +shape for any document-backed portfolio.* diff --git a/test/controllers/mcp_controller_test.rb b/test/controllers/mcp_controller_test.rb index b614407bc..0f2713b74 100644 --- a/test/controllers/mcp_controller_test.rb +++ b/test/controllers/mcp_controller_test.rb @@ -230,7 +230,7 @@ class McpControllerTest < ActionDispatch::IntegrationTest tools = body["result"]["tools"] assert_kind_of Array, tools - assert_equal Assistant.function_classes.size, tools.size + assert_equal Assistant.function_classes(@user).size, tools.size tool_names = tools.map { |t| t["name"] } assert_includes tool_names, "get_transactions" @@ -249,8 +249,55 @@ class McpControllerTest < ActionDispatch::IntegrationTest end end + test "tools/list omits preview tools for a user without preview features" do + @user.update!(preferences: (@user.preferences || {}).merge("preview_features_enabled" => false)) + + with_mcp_env do + post "/mcp", params: jsonrpc_request("tools/list").to_json, + headers: mcp_headers(@token) + + assert_response :ok + tool_names = JSON.parse(response.body)["result"]["tools"].map { |t| t["name"] } + + assert_includes tool_names, "get_transactions" + Assistant::PREVIEW_FUNCTION_CLASSES.each do |fn_class| + assert_not_includes tool_names, fn_class.name + end + end + end + + test "tools/list includes preview tools for an opted-in user" do + @user.update!(preferences: (@user.preferences || {}).merge("preview_features_enabled" => true)) + + with_mcp_env do + post "/mcp", params: jsonrpc_request("tools/list").to_json, + headers: mcp_headers(@token) + + assert_response :ok + tool_names = JSON.parse(response.body)["result"]["tools"].map { |t| t["name"] } + + Assistant::PREVIEW_FUNCTION_CLASSES.each do |fn_class| + assert_includes tool_names, fn_class.name + end + end + end + # -- tools/call -- + test "tools/call rejects a preview tool for a user without preview features" do + @user.update!(preferences: (@user.preferences || {}).merge("preview_features_enabled" => false)) + + with_mcp_env do + post "/mcp", params: jsonrpc_request("tools/call", { name: "list_account_statements", arguments: {} }, id: 42).to_json, + headers: mcp_headers(@token) + + assert_response :ok + body = JSON.parse(response.body) + assert_equal(-32602, body["error"]["code"]) + assert_includes body["error"]["message"], "list_account_statements" + end + end + test "tools/call returns error for unknown tool with request id preserved" do with_mcp_env do post "/mcp", params: jsonrpc_request("tools/call", { name: "nonexistent_tool", arguments: {} }, id: 99).to_json, diff --git a/test/models/assistant/function/get_account_statement_test.rb b/test/models/assistant/function/get_account_statement_test.rb new file mode 100644 index 000000000..c4e338ebb --- /dev/null +++ b/test/models/assistant/function/get_account_statement_test.rb @@ -0,0 +1,73 @@ +require "test_helper" + +class Assistant::Function::GetAccountStatementTest < ActiveSupport::TestCase + setup do + @user = users(:family_admin) + @account = accounts(:depository) + @function = Assistant::Function::GetAccountStatement.new(@user) + end + + test "returns statement identity and reconciliation checks" do + statement = create_statement(account: @account) + statement.update!( + period_start_on: Date.new(2024, 1, 1), + period_end_on: Date.new(2024, 1, 31), + opening_balance: 100, + closing_balance: 200, + currency: @account.currency + ) + + result = @function.call("statement_id" => statement.id) + + assert result[:success] + assert_equal statement.id, result[:statement][:id] + assert_equal statement.content_sha256, result[:statement][:content_sha256] + assert_equal "2024-01-31", result[:statement][:period_end_on] + assert result[:statement].key?(:reconciliation_checks) + end + + test "returns not_found for an unknown id" do + result = @function.call("statement_id" => SecureRandom.uuid) + + assert_not result[:success] + assert_equal "not_found", result[:error] + end + + test "returns not_found for a non-uuid id" do + result = @function.call("statement_id" => "nope") + + assert_not result[:success] + assert_equal "not_found", result[:error] + end + + test "returns not_found for a statement the user cannot view" do + statement = create_statement(account: accounts(:other_asset)) + + result = Assistant::Function::GetAccountStatement.new(users(:family_member)).call("statement_id" => statement.id) + + assert_not result[:success] + assert_equal "not_found", result[:error] + end + + test "refuses a user who cannot manage the vault" do + statement = create_statement(account: @account) + + result = Assistant::Function::GetAccountStatement.new(family_guest).call("statement_id" => statement.id) + + assert_not result[:success] + assert_equal "forbidden", result[:error] + end + + private + def create_statement(account:) + AccountStatement.create_from_upload!( + family: @user.family, + account: account, + file: uploaded_file( + filename: "statement-#{SecureRandom.hex(4)}.csv", + content_type: "text/csv", + content: "date,amount\n2024-01-01,#{SecureRandom.random_number(1000)}\n" + ) + ) + end +end diff --git a/test/models/assistant/function/get_statement_coverage_test.rb b/test/models/assistant/function/get_statement_coverage_test.rb new file mode 100644 index 000000000..b9a0308ff --- /dev/null +++ b/test/models/assistant/function/get_statement_coverage_test.rb @@ -0,0 +1,56 @@ +require "test_helper" + +class Assistant::Function::GetStatementCoverageTest < ActiveSupport::TestCase + setup do + @user = users(:family_admin) + @account = accounts(:depository) + @function = Assistant::Function::GetStatementCoverage.new(@user) + end + + test "reports a covered month and the months with no statement on record" do + # A full prior year: every month is inside the expected window, so the one + # month with a statement is covered and the rest report as missing. + january = Date.current.prev_year.beginning_of_year + statement = AccountStatement.create_from_upload!( + family: @user.family, + account: @account, + file: uploaded_file(filename: "statement.csv", content_type: "text/csv", content: "date,amount\n2024-01-01,1\n") + ) + statement.update!(period_start_on: january, period_end_on: january.end_of_month) + + result = @function.call("account_id" => @account.id, "year" => january.year) + + assert result[:success] + assert_equal @account.id, result[:account][:id] + assert_equal january.year, result[:year] + + covered = result[:months].find { |m| m[:month] == january.strftime("%Y-%m") } + assert_equal "covered", covered[:status] + assert_includes covered[:statement_ids], statement.id + + february = result[:months].find { |m| m[:month] == january.next_month.strftime("%Y-%m") } + assert_equal "missing", february[:status] + end + + test "rejects a non-uuid account id" do + result = @function.call("account_id" => "nope") + + assert_not result[:success] + assert_equal "invalid_account_id", result[:error] + end + + test "rejects an account the user cannot access" do + result = Assistant::Function::GetStatementCoverage.new(users(:family_member)) + .call("account_id" => accounts(:other_asset).id) + + assert_not result[:success] + assert_equal "account_not_found", result[:error] + end + + test "refuses a user who cannot manage the vault" do + result = Assistant::Function::GetStatementCoverage.new(family_guest).call("account_id" => @account.id) + + assert_not result[:success] + assert_equal "forbidden", result[:error] + end +end diff --git a/test/models/assistant/function/list_account_statements_test.rb b/test/models/assistant/function/list_account_statements_test.rb new file mode 100644 index 000000000..20f264f11 --- /dev/null +++ b/test/models/assistant/function/list_account_statements_test.rb @@ -0,0 +1,84 @@ +require "test_helper" + +class Assistant::Function::ListAccountStatementsTest < ActiveSupport::TestCase + setup do + @user = users(:family_admin) + @account = accounts(:depository) + @function = Assistant::Function::ListAccountStatements.new(@user) + end + + test "lists statements with identity and provenance" do + statement = create_statement(account: @account, content: "date,amount\n2024-01-01,1\n") + + result = @function.call + + assert result[:success] + assert_equal 1, result[:returned] + assert_not result[:has_more] + payload = result[:statements].first + assert_equal statement.id, payload[:id] + assert_equal statement.content_sha256, payload[:content_sha256] + assert_equal @account.id, payload[:account][:id] + end + + test "filters by review status" do + create_statement(account: @account, content: "date,amount\n2024-01-01,1\n") + create_statement(account: nil, content: "date,amount\n2024-02-01,2\n") + + result = @function.call("review_status" => "unmatched") + + assert result[:success] + assert_equal 1, result[:returned] + assert_equal "unmatched", result[:statements].first[:review_status] + end + + test "filters by content sha256" do + statement = create_statement(account: @account, content: "date,amount\n2024-01-01,1\n") + create_statement(account: @account, content: "date,amount\n2024-02-01,2\n") + + result = @function.call("content_sha256" => statement.content_sha256) + + assert_equal 1, result[:returned] + assert_equal statement.id, result[:statements].first[:id] + end + + test "rejects an invalid review status" do + result = @function.call("review_status" => "whatever") + + assert_not result[:success] + assert_equal "invalid_review_status", result[:error] + end + + test "rejects an invalid date filter" do + result = @function.call("period_start_on_or_after" => "last tuesday") + + assert_not result[:success] + assert_equal "invalid_date", result[:error] + end + + test "hides statements linked to accounts the user cannot see" do + private_account = accounts(:other_asset) + create_statement(account: private_account, content: "date,amount\n2024-03-01,3\n") + + result = Assistant::Function::ListAccountStatements.new(users(:family_member)).call + + assert result[:success] + assert_empty result[:statements] + end + + test "refuses a user who cannot manage the vault" do + result = Assistant::Function::ListAccountStatements.new(family_guest).call + + assert_not result[:success] + assert_equal "forbidden", result[:error] + end + + private + def create_statement(account:, content:) + AccountStatement.create_from_upload!( + family: @user.family, + account: account, + file: uploaded_file(filename: "statement-#{Digest::MD5.hexdigest(content)}.csv", content_type: "text/csv", content: content) + ) + end +end diff --git a/test/models/assistant/function/record_valuation_test.rb b/test/models/assistant/function/record_valuation_test.rb new file mode 100644 index 000000000..3ff3d8334 --- /dev/null +++ b/test/models/assistant/function/record_valuation_test.rb @@ -0,0 +1,89 @@ +require "test_helper" + +class Assistant::Function::RecordValuationTest < ActiveSupport::TestCase + setup do + @user = users(:family_admin) + @account = accounts(:other_asset) + @function = Assistant::Function::RecordValuation.new(@user) + @source = "Appraisal report 2024-06-30 (grade: A)" + end + + test "records a valuation and stores the citation on the entry" do + result = nil + + assert_difference "@account.entries.valuations.count", 1 do + result = @function.call(params) + end + + assert result[:success] + assert_not result[:replaced_existing] + assert_equal "A", result[:provenance][:grade] + + entry = Entry.find(result[:entry_id]) + assert_equal @source, entry.notes + assert_equal 1234.56.to_d, entry.amount + end + + test "flags when an existing valuation on the same date is replaced" do + @function.call(params) + + result = @function.call(params(amount: 2000)) + + assert result[:success] + assert result[:replaced_existing] + end + + test "rejects a citation that does not follow the grammar" do + result = @function.call(params(source: "estimated: pulled from a spreadsheet")) + + assert_not result[:success] + assert_equal "invalid_source_citation", result[:error] + assert_match(/reliability grade/, result[:message]) + end + + test "rejects a missing citation" do + result = @function.call(params(source: "")) + + assert_not result[:success] + assert_equal "invalid_source_citation", result[:error] + end + + test "rejects an unparseable date" do + result = @function.call(params(date: "June 30th")) + + assert_not result[:success] + assert_equal "invalid_date", result[:error] + end + + test "rejects a non-numeric amount" do + result = @function.call(params(amount: "a lot")) + + assert_not result[:success] + assert_equal "invalid_amount", result[:error] + end + + test "rejects an account the user cannot write to" do + result = Assistant::Function::RecordValuation.new(users(:family_member)).call(params) + + assert_not result[:success] + assert_equal "account_not_found", result[:error] + end + + test "accepts an estimated citation carrying a grade" do + result = @function.call(params(source: "estimated: linear interpolation over 2024-01 / 2024-12 anchors (grade: C)")) + + assert result[:success] + assert result[:provenance][:estimated] + assert_equal "C", result[:provenance][:grade] + end + + private + def params(overrides = {}) + { + "account_id" => @account.id, + "date" => "2024-06-30", + "amount" => 1234.56, + "source" => @source + }.merge(overrides.transform_keys(&:to_s)) + end +end diff --git a/test/models/assistant/function/upload_account_statement_test.rb b/test/models/assistant/function/upload_account_statement_test.rb new file mode 100644 index 000000000..53b7a0202 --- /dev/null +++ b/test/models/assistant/function/upload_account_statement_test.rb @@ -0,0 +1,114 @@ +require "test_helper" + +class Assistant::Function::UploadAccountStatementTest < ActiveSupport::TestCase + setup do + @user = users(:family_admin) + @account = accounts(:depository) + @function = Assistant::Function::UploadAccountStatement.new(@user) + @content = "date,amount\n2024-01-01,1\n" + end + + test "has correct name and is not strict" do + assert_equal "upload_account_statement", @function.name + assert_not @function.strict_mode? + assert_includes @function.params_schema[:required], "content_base64" + end + + test "stores a statement in the vault" do + result = nil + + assert_difference "AccountStatement.count", 1 do + result = @function.call(params(filename: "statement.csv")) + end + + assert result[:success] + assert_not result[:duplicate] + assert_equal Digest::SHA256.hexdigest(@content), result[:statement][:content_sha256] + assert_equal "statement.csv", result[:statement][:filename] + end + + test "re-uploading identical bytes returns the existing statement without creating a row" do + first = @function.call(params(filename: "statement.csv")) + + assert_no_difference "AccountStatement.count" do + second = @function.call(params(filename: "different-name.csv")) + + assert second[:success] + assert second[:duplicate] + assert_equal first[:statement][:id], second[:statement][:id] + end + end + + test "links to an account when one is given" do + result = @function.call(params(filename: "statement.csv", account_id: @account.id)) + + assert result[:success] + assert_equal @account.id, result[:statement][:account][:id] + assert_equal "linked", result[:statement][:review_status] + end + + test "leaves the statement unmatched when no account is given" do + result = @function.call(params(filename: "statement.csv")) + + assert_equal "unmatched", result[:statement][:review_status] + assert_nil result[:statement][:account] + end + + test "reports a duplicate without disclosing a statement filed against a hidden account" do + @function.call(params(filename: "statement.csv", account_id: accounts(:other_asset).id)) + + result = Assistant::Function::UploadAccountStatement.new(users(:family_member)) + .call(params(filename: "statement.csv")) + + assert result[:success] + assert result[:duplicate] + assert_equal Digest::SHA256.hexdigest(@content), result[:statement][:content_sha256] + assert_nil result[:statement][:account] + assert_nil result[:statement][:filename] + end + + test "refuses a user who cannot manage the vault" do + result = Assistant::Function::UploadAccountStatement.new(family_guest).call(params(filename: "statement.csv")) + + assert_not result[:success] + assert_equal "forbidden", result[:error] + end + + test "rejects an unsupported file type" do + result = @function.call(params(filename: "notes.txt")) + + assert_not result[:success] + assert_equal "unsupported_file_type", result[:error] + end + + test "rejects content that is not base64" do + result = @function.call("filename" => "statement.csv", "content_base64" => "not base64 @@@") + + assert_not result[:success] + assert_equal "invalid_content", result[:error] + end + + test "rejects an unknown account_id rather than silently uploading unlinked" do + result = @function.call(params(filename: "statement.csv", account_id: SecureRandom.uuid)) + + assert_not result[:success] + assert_equal "account_not_found", result[:error] + end + + test "rejects a file whose contents do not match its extension" do + result = @function.call( + "filename" => "statement.pdf", + "content_base64" => Base64.strict_encode64("this is not a pdf") + ) + + assert_not result[:success] + assert_equal "invalid_file", result[:error] + end + + private + def params(filename:, account_id: nil, content: @content) + { "filename" => filename, "content_base64" => Base64.strict_encode64(content) }.tap do |p| + p["account_id"] = account_id if account_id + end + end +end diff --git a/test/models/provenance/citation_test.rb b/test/models/provenance/citation_test.rb new file mode 100644 index 000000000..b97f9e646 --- /dev/null +++ b/test/models/provenance/citation_test.rb @@ -0,0 +1,72 @@ +require "test_helper" + +class Provenance::CitationTest < ActiveSupport::TestCase + test "parses a plain citation with a reliability grade" do + citation = Provenance::Citation.parse!("Private bank statement 2026-03-31, category subtotals (grade: A)") + + assert_equal "Private bank statement 2026-03-31, category subtotals", citation.text + assert_equal "A", citation.grade + assert_not citation.estimated? + assert_not citation.proxy? + end + + test "parses an estimated citation" do + citation = Provenance::Citation.parse!("estimated: linear interpolation over 2024-08 / 2024-12 anchors (grade: C)") + + assert citation.estimated? + assert citation.proxy? + assert_equal "C", citation.grade + assert_equal "linear interpolation over 2024-08 / 2024-12 anchors", citation.text + end + + test "grade is optional on a non-estimated citation" do + citation = Provenance::Citation.parse!("Appraisal report 2026-03-12") + + assert_nil citation.grade + assert_not citation.estimated? + end + + test "rejects a blank citation" do + assert_raises(Provenance::Citation::InvalidError) { Provenance::Citation.parse!(" ") } + assert_raises(Provenance::Citation::InvalidError) { Provenance::Citation.parse!(nil) } + end + + test "rejects an unknown reliability grade" do + error = assert_raises(Provenance::Citation::InvalidError) do + Provenance::Citation.parse!("Some document (grade: D)") + end + + assert_match(/grade must be one of/, error.message) + end + + test "rejects an estimate that carries no grade" do + error = assert_raises(Provenance::Citation::InvalidError) do + Provenance::Citation.parse!("estimated: interpolated from neighbouring months") + end + + assert_match(/reliability grade/, error.message) + end + + test "rejects an estimated marker that does not use the exact prefix" do + error = assert_raises(Provenance::Citation::InvalidError) do + Provenance::Citation.parse!("Estimated: interpolated (grade: C)") + end + + assert_match(/exact prefix/, error.message) + end + + test "rejects a citation with no document named" do + assert_raises(Provenance::Citation::InvalidError) { Provenance::Citation.parse!("ab") } + end + + test "rejects an over-long citation" do + assert_raises(Provenance::Citation::InvalidError) do + Provenance::Citation.parse!("x" * (Provenance::Citation::MAX_LENGTH + 1)) + end + end + + test "valid? does not raise" do + assert Provenance::Citation.valid?("Statement 2026-01-31 (grade: A)") + assert_not Provenance::Citation.valid?("") + end +end