fix(imports): normalize CSV upload encoding when validations are skipped (#2299)

Import::UploadsController#update persists CSV uploads with
save!(validate: false), which skips the before_validation
:ensure_utf8_encoding callback. Non-UTF-8 files (e.g. ISO-8859-1 /
Windows-1252 exports from Brazilian banks) were written to the text
column as-is and rejected by Postgres with PG::CharacterNotInRepertoire,
surfacing as a 500 during upload.

Also register ensure_utf8_encoding on before_save so the existing
normalization (rchardet detection + Latin-1/Windows fallback) still runs
when validations are skipped. The callback is idempotent and no-ops on
valid UTF-8, so the validated path is unchanged.

Fixes #2294
This commit is contained in:
Augusto Xavier
2026-06-16 12:13:03 -03:00
committed by GitHub
parent aec851cae1
commit 844cba2fd3
2 changed files with 22 additions and 0 deletions

View File

@@ -56,6 +56,27 @@ class ImportEncodingTest < ActiveSupport::TestCase
assert import.rows.any? { |row| row.name&.include?("Café") }, "Extended Latin characters should be preserved"
end
test "normalizes encoding on save even when validations are skipped" do
file_path = Rails.root.join("test/fixtures/files/imports/windows1250.csv")
csv_content = File.binread(file_path)
assert_equal Encoding::ASCII_8BIT, csv_content.encoding
import = @family.imports.create!(
type: "TransactionImport",
account: @account,
date_format: "%Y-%m-%d"
)
import.assign_attributes(raw_file_str: csv_content)
assert_nothing_raised do
import.save!(validate: false)
end
assert_equal Encoding::UTF_8, import.reload.raw_file_str.encoding
assert import.raw_file_str.valid_encoding?, "Converted string should be valid UTF-8"
end
test "handles UTF-8 files without modification" do
# Test that valid UTF-8 files are not modified
file_path = Rails.root.join("test/fixtures/files/imports/transactions.csv")