sure/app/models/account_statement.rb

# frozen_string_literal: true

require "digest/md5"
require "digest/sha2"
require "stringio"

class AccountStatement < ApplicationRecord
  include Monetizable

  DuplicateUploadError = Class.new(StandardError) do
    attr_reader :statement

    def initialize(statement)
      @statement = statement
      super("Statement file has already been uploaded")
    end
  end
  InvalidUploadError = Class.new(StandardError)

  PreparedUpload = Data.define(:content, :filename, :content_type, :byte_size, :checksum, :content_sha256)

  MAX_FILE_SIZE = 25.megabytes
  READ_CHUNK_SIZE = 1.megabyte
  ALLOWED_EXTENSION_CONTENT_TYPES = {
    ".pdf" => %w[application/pdf],
    ".csv" => %w[text/csv text/plain application/csv application/vnd.ms-excel],
    ".xlsx" => %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet]
  }.freeze
  ALLOWED_CONTENT_TYPES = ALLOWED_EXTENSION_CONTENT_TYPES.values.flatten.uniq.freeze
  ACCEPTED_FILE_EXTENSIONS = ALLOWED_EXTENSION_CONTENT_TYPES.keys.freeze

  belongs_to :family
  belongs_to :account, optional: true
  belongs_to :suggested_account, class_name: "Account", optional: true

  has_one_attached :original_file, dependent: :purge_later

  enum :source, { manual_upload: "manual_upload" }, validate: true, default: "manual_upload"
  enum :upload_status, { stored: "stored", failed: "failed" }, validate: true, default: "stored"
  enum :review_status, { unmatched: "unmatched", linked: "linked", rejected: "rejected" }, validate: true, default: "unmatched", scopes: false

  monetize :opening_balance, :closing_balance

  before_validation :sync_file_metadata, if: -> { original_file.attached? }
  before_validation :normalize_currency
  before_validation :sync_review_status

  validates :filename, :content_type, :checksum, presence: true
  validates :byte_size, presence: true, numericality: { greater_than: 0, less_than_or_equal_to: MAX_FILE_SIZE }
  validates :content_type, inclusion: { in: ALLOWED_CONTENT_TYPES }
  validates :content_sha256,
            format: { with: /\A[0-9a-f]{64}\z/ },
            uniqueness: { scope: :family_id, allow_nil: true, message: :duplicate_statement_file },
            allow_nil: true
  validates :parser_confidence, :match_confidence, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
  validate :account_belongs_to_family
  validate :suggested_account_belongs_to_family
  validate :period_order
  validate :currency_is_valid
  validate :filename_extension_matches_content_type
  validate :original_file_attached
  validate :original_file_constraints, if: -> { original_file.attached? }

  scope :ordered, -> { order(created_at: :desc) }
  scope :with_account, -> { where.not(account_id: nil) }
  scope :unmatched, -> { where(account_id: nil).where(review_status: "unmatched") }
  scope :for_month, ->(month) {
    month_start = month.to_date.beginning_of_month
    month_end = month_start.end_of_month
    where("period_start_on <= ? AND period_end_on >= ?", month_end, month_start)
  }

  class << self
    def statement_manager?(user)
      user&.admin? || user&.member?
    end

    def create_from_upload!(family:, account:, file:)
      prepared_upload = prepare_upload!(file)
      create_from_prepared_upload!(family: family, account: account, prepared_upload: prepared_upload)
    end

    def create_from_prepared_upload!(family:, account:, prepared_upload:)
      statement = nil
      duplicate = duplicate_for(family, prepared_upload)
      raise DuplicateUploadError, duplicate if duplicate

      statement = family.account_statements.build(
        account: account,
        filename: prepared_upload.filename,
        content_type: prepared_upload.content_type,
        byte_size: prepared_upload.byte_size,
        checksum: prepared_upload.checksum,
        content_sha256: prepared_upload.content_sha256,
        source: :manual_upload,
        upload_status: :stored,
        review_status: account.present? ? :linked : :unmatched,
        currency: account&.currency || family.currency
      )

      statement.original_file.attach(
        io: StringIO.new(prepared_upload.content),
        filename: prepared_upload.filename,
        content_type: prepared_upload.content_type
      )

      MetadataDetector.new(statement, content: prepared_upload.content).apply
      statement.assign_account_match unless account.present?
      statement.save!
      statement
    rescue ActiveRecord::RecordNotUnique
      duplicate = duplicate_for(family, prepared_upload)
      purge_original_file(statement)

      if duplicate
        raise DuplicateUploadError, duplicate
      end

      raise
    rescue StandardError
      purge_original_file(statement)
      raise
    end

    def reconciliation_statuses_for(statements, account:)
      statement_list = statements.to_a
      balance_lookup = balance_lookup_for(account, statement_list)

      statement_list.to_h do |statement|
        [ statement.id, statement.reconciliation_status(balance_lookup: balance_lookup) ]
      end
    end

    def prepare_upload!(file)
      filename = file.original_filename.to_s
      content = read_upload_content!(file)
      byte_size = content.bytesize
      raise InvalidUploadError if byte_size.zero?

      content_type = detected_content_type(content:, filename:, declared_content_type: file.content_type)
      raise InvalidUploadError unless allowed_upload?(filename:, content_type:)
      raise InvalidUploadError if content_type == "application/pdf" && !valid_pdf_content?(content)

      PreparedUpload.new(
        content: content,
        filename: filename,
        content_type: content_type,
        byte_size: byte_size,
        checksum: Digest::MD5.base64digest(content),
        content_sha256: Digest::SHA256.hexdigest(content)
      )
    end

    def detected_content_type(content:, filename:, declared_content_type:)
      Marcel::MimeType.for(
        StringIO.new(content),
        name: filename,
        declared_type: declared_content_type.presence
      )
    end

    def allowed_upload?(filename:, content_type:)
      allowed_content_types_for_filename(filename).include?(content_type)
    end

    def allowed_content_types_for_filename(filename)
      ALLOWED_EXTENSION_CONTENT_TYPES.fetch(File.extname(filename.to_s).downcase, [])
    end

    def valid_pdf_content?(content)
      content.start_with?("%PDF-")
    end

    def purge_original_file(statement)
      return unless statement&.original_file&.attached?

      statement.original_file.purge
    rescue StandardError => e
      Rails.logger.warn("AccountStatement staged blob cleanup failed: #{e.class}: #{e.message}")
    end

    def balance_lookup_for(account, statements)
      currencies = statements.map(&:statement_currency).compact.uniq
      dates = statements.flat_map { |statement| [ statement.period_start_on, statement.period_end_on ] }.compact.uniq
      balances = if currencies.any? && dates.any?
        account.balances.where(currency: currencies, date: dates).to_a
      else
        []
      end
      balances_by_key = balances.index_by { |balance| [ balance.date, balance.currency ] }

      ->(date, currency) { balances_by_key[[ date, currency ]] }
    end

    def read_upload_content!(file)
      declared_size = declared_upload_size(file)
      raise InvalidUploadError if declared_size.present? && declared_size > MAX_FILE_SIZE

      content = +"".b
      loop do
        chunk = file.read(READ_CHUNK_SIZE)
        break if chunk.nil? || chunk.empty?

        content << chunk
        raise InvalidUploadError if content.bytesize > MAX_FILE_SIZE
      end

      file.rewind if file.respond_to?(:rewind)
      content
    end

    def declared_upload_size(file)
      if file.respond_to?(:size)
        file.size
      elsif file.respond_to?(:length)
        file.length
      end
    end

    def duplicate_for(family, prepared_upload)
      scope = family.account_statements
      sha_duplicate = scope.find_by(content_sha256: prepared_upload.content_sha256) if prepared_upload.content_sha256.present?
      return sha_duplicate if sha_duplicate

      # Active Storage's MD5 checksum is retained only to catch legacy rows that predate content_sha256.
      legacy_scope = prepared_upload.content_sha256.present? ? scope.where(content_sha256: nil) : scope
      legacy_scope.find_by(checksum: prepared_upload.checksum)
    end
  end

  def viewable_by?(user)
    return false unless user&.family_id == family_id

    account.present? ? account.shared_with?(user) : self.class.statement_manager?(user)
  end

  def manageable_by?(user)
    return false unless user&.family_id == family_id

    return self.class.statement_manager?(user) if account.blank?

    account.permission_for(user).in?([ :owner, :full_control ]) && self.class.statement_manager?(user)
  end

  def link_to_account!(target_account, confidence: 1.0)
    update!(
      account: target_account,
      suggested_account: nil,
      match_confidence: confidence,
      review_status: :linked,
      currency: currency.presence || target_account.currency
    )
  end

  def unlink!
    transaction do
      update!(
        account: nil,
        review_status: :unmatched,
        match_confidence: nil
      )
      assign_account_match
      save!
    end
  end

  def reject_match!
    update!(
      suggested_account: nil,
      match_confidence: nil,
      review_status: :rejected
    )
  end

  def assign_account_match
    match = AccountMatcher.new(self).best_match

    self.suggested_account = match&.account
    self.match_confidence = match&.confidence
    clear_invalid_suggested_account
  end

  def covered_months
    return [] unless period_start_on.present? && period_end_on.present?

    current = period_start_on.beginning_of_month
    last = period_end_on.beginning_of_month
    months = []

    while current <= last
      months << current
      current = current.next_month
    end

    months
  end

  def covers_month?(month)
    covered_months.include?(month.to_date.beginning_of_month)
  end

  def reconciliation_status(balance_lookup: nil)
    checks = reconciliation_checks(balance_lookup: balance_lookup)
    return "unavailable" if checks.empty?

    checks.any? { |check| check[:status] == "mismatched" } ? "mismatched" : "matched"
  end

  def reconciliation_mismatched?(balance_lookup: nil)
    reconciliation_status(balance_lookup: balance_lookup) == "mismatched"
  end

  def reconciliation_checks(balance_lookup: nil)
    return [] unless account.present? && period_start_on.present? && period_end_on.present?

    checks = []
    opening_balance_record = balance_record_for(period_start_on, statement_currency, balance_lookup)
    closing_balance_record = balance_record_for(period_end_on, statement_currency, balance_lookup)

    if opening_balance.present? && opening_balance_record.present?
      checks << reconciliation_check(
        key: "opening_balance",
        statement_amount: opening_balance,
        ledger_amount: opening_balance_record.start_balance
      )
    end

    if closing_balance.present? && closing_balance_record.present?
      checks << reconciliation_check(
        key: "closing_balance",
        statement_amount: closing_balance,
        ledger_amount: closing_balance_record.end_balance
      )
    end

    if opening_balance.present? && closing_balance.present? && opening_balance_record.present? && closing_balance_record.present?
      checks << reconciliation_check(
        key: "period_movement",
        statement_amount: closing_balance - opening_balance,
        ledger_amount: closing_balance_record.end_balance - opening_balance_record.start_balance
      )
    end

    checks
  end

  def statement_currency
    currency.presence || account&.currency || family.currency
  end

  def pdf?
    content_type.in?(ALLOWED_EXTENSION_CONTENT_TYPES[".pdf"])
  end

  def csv?
    content_type.in?(ALLOWED_EXTENSION_CONTENT_TYPES[".csv"])
  end

  def xlsx?
    content_type.in?(ALLOWED_EXTENSION_CONTENT_TYPES[".xlsx"])
  end

  private

    def reconciliation_check(key:, statement_amount:, ledger_amount:)
      difference = statement_amount.to_d - ledger_amount.to_d
      {
        key: key,
        statement_amount: statement_amount.to_d,
        ledger_amount: ledger_amount.to_d,
        difference: difference,
        status: difference.abs <= 0.01.to_d ? "matched" : "mismatched"
      }
    end

    def balance_record_for(date, currency, balance_lookup)
      return balance_lookup.call(date, currency) if balance_lookup

      account.balances.find_by(date: date, currency: currency)
    end

    def sync_file_metadata
      blob = original_file.blob
      self.filename ||= blob.filename.to_s
      self.content_type ||= blob.content_type
      self.byte_size ||= blob.byte_size
      self.checksum ||= blob.checksum
    end

    def normalize_currency
      self.currency = currency.to_s.upcase.presence if currency.present?
    end

    def sync_review_status
      return if rejected?

      self.review_status = "linked" if account.present? && !linked?
      self.review_status = "unmatched" if account.blank? && linked?
    end

    def account_belongs_to_family
      return if account.nil?
      return if account.family_id == family_id

      errors.add(:account, :invalid)
    end

    def suggested_account_belongs_to_family
      return if suggested_account_valid_for_family?

      errors.add(:suggested_account, :invalid)
    end

    def clear_invalid_suggested_account
      return if suggested_account_valid_for_family?

      self.suggested_account = nil
      self.match_confidence = nil
    end

    def suggested_account_valid_for_family?
      suggested_account.nil? || suggested_account.family_id == family_id
    end

    def period_order
      return if period_start_on.blank? || period_end_on.blank?
      return if period_start_on <= period_end_on

      errors.add(:period_end_on, :on_or_after_start)
    end

    def currency_is_valid
      return if currency.blank?

      Money::Currency.new(currency)
    rescue Money::Currency::UnknownCurrencyError, ArgumentError
      errors.add(:currency, :invalid)
    end

    def filename_extension_matches_content_type
      return if filename.blank? || content_type.blank?
      return if self.class.allowed_upload?(filename: filename, content_type: content_type)

      errors.add(:content_type, :invalid)
    end

    def original_file_constraints
      if original_file.byte_size.zero?
        errors.add(:original_file, :blank)
      elsif original_file.byte_size > MAX_FILE_SIZE
        errors.add(:original_file, :too_large, max_mb: MAX_FILE_SIZE / 1.megabyte)
      end

      unless self.class.allowed_upload?(filename: original_file.filename.to_s, content_type: original_file.content_type)
        errors.add(:original_file, :invalid_format, file_format: original_file.content_type)
      end
    end

    def original_file_attached
      errors.add(:original_file, :blank) unless original_file.attached?
    end
end