mirror of
https://github.com/we-promise/sure.git
synced 2026-06-06 11:19:02 +00:00
feat(imports): add SureImport session batches (#1785)
* feat(imports): add SureImport session batches Add first-class SureImport sessions for ordered multi-file NDJSON imports. Persist source mappings across chunks, make session/chunk processing idempotent, expose progress readback, and keep existing single-file import behavior compatible. Includes the devcontainer libvips runtime dependency needed by ActiveStorage variant tests. Addresses #1610. Related to #1458. * fix(imports): avoid scanner-like API key test data * test(imports): assert skipped balances are not persisted * fix(imports): harden session publish retries Validate expected import chunk sequences exactly before publish, and restore session state with error details when enqueueing the publish job fails. * fix(imports): close session retry edge cases Backfill expected chunk counts after client-session insert races and enqueue import-session jobs after the status transition commits. Persist a safe enqueue failure body so API readback does not expose raw queue errors. * fix(imports): address session publish review gaps Remove dead transaction external-id assignment, harden session publish retry/sync behavior, align session chunk status docs, and add regression coverage for partial retries and safe enqueue error readback. * fix(imports): include sessions in family reset Clear import sessions through the family reset job so chunk imports and source mappings do not survive a reset. Expose import session and source mapping counts in the reset status response and regenerated OpenAPI schema so polling reflects the full reset surface. * test(imports): cover split import mapping invariants * test(imports): cover session verification invariants * fix(imports): scope SureImport session reimports * Tighten SureImport session batching * fix(imports): export rule source ids for sessions * test(imports): stabilize rule id export assertion * test(imports): restore reset status session fixture
This commit is contained in:
ghost
308
test/controllers/api/v1/import_sessions_controller_test.rb
Normal file
308
test/controllers/api/v1/import_sessions_controller_test.rb
Normal file
@@ -0,0 +1,308 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "test_helper"
|
||||
|
||||
class Api::V1::ImportSessionsControllerTest < ActionDispatch::IntegrationTest
|
||||
include ActiveJob::TestHelper
|
||||
|
||||
setup do
|
||||
@user = users(:family_admin)
|
||||
@family = @user.family
|
||||
@api_key = api_keys(:active_key)
|
||||
@read_only_api_key = api_keys(:one)
|
||||
|
||||
Redis.new.del("api_rate_limit:#{@api_key.id}")
|
||||
Redis.new.del("api_rate_limit:#{@read_only_api_key.id}")
|
||||
end
|
||||
|
||||
test "creates an idempotent Sure import session" do
|
||||
assert_difference("ImportSession.count", 1) do
|
||||
post api_v1_import_sessions_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
client_session_id: "client-session-1",
|
||||
expected_chunks: 2
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :created
|
||||
first_id = JSON.parse(response.body).dig("data", "id")
|
||||
|
||||
assert_no_difference("ImportSession.count") do
|
||||
post api_v1_import_sessions_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
client_session_id: "client-session-1",
|
||||
expected_chunks: 2
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :created
|
||||
assert_equal first_id, JSON.parse(response.body).dig("data", "id")
|
||||
end
|
||||
|
||||
test "rejects unsupported import session types" do
|
||||
assert_no_difference("ImportSession.count") do
|
||||
post api_v1_import_sessions_url,
|
||||
params: { type: "TransactionImport" },
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
assert_equal "validation_failed", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "rejects malformed expected chunk counts" do
|
||||
assert_no_difference("ImportSession.count") do
|
||||
post api_v1_import_sessions_url,
|
||||
params: { type: "SureImport", expected_chunks: "2abc" },
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
assert_equal "validation_failed", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "requires authentication for session creation" do
|
||||
post api_v1_import_sessions_url, params: { type: "SureImport" }
|
||||
|
||||
assert_response :unauthorized
|
||||
assert_equal "unauthorized", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "uploads ordered chunks and publishes a full-fidelity transaction import" do
|
||||
session = build_import_session
|
||||
|
||||
post chunks_api_v1_import_session_url(session),
|
||||
params: {
|
||||
sequence: 1,
|
||||
client_chunk_id: "entities",
|
||||
raw_file_content: build_ndjson(entity_records)
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
|
||||
assert_response :created
|
||||
assert_equal 1, JSON.parse(response.body).dig("data", "chunks_count")
|
||||
|
||||
post chunks_api_v1_import_session_url(session),
|
||||
params: {
|
||||
sequence: 2,
|
||||
client_chunk_id: "transactions",
|
||||
raw_file_content: build_ndjson(transaction_records)
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
|
||||
assert_response :created
|
||||
|
||||
perform_enqueued_jobs do
|
||||
post publish_api_v1_import_session_url(session), headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :accepted
|
||||
session.reload
|
||||
assert session.complete?
|
||||
assert_equal 1, session.summary.dig("transactions", "created")
|
||||
|
||||
entry = @family.accounts.find_by!(name: "API Session Checking").entries.find_by!(name: "API Grocery Run")
|
||||
transaction = entry.entryable
|
||||
assert_equal "API Groceries", transaction.category.name
|
||||
assert_equal "API Market", transaction.merchant.name
|
||||
assert_equal [ "API Weekly" ], transaction.tags.map(&:name)
|
||||
end
|
||||
|
||||
test "rejects replayed chunk with different content" do
|
||||
session = build_import_session
|
||||
params = {
|
||||
sequence: 1,
|
||||
client_chunk_id: "entities",
|
||||
raw_file_content: build_ndjson(entity_records)
|
||||
}
|
||||
|
||||
post chunks_api_v1_import_session_url(session), params: params, headers: api_headers(@api_key)
|
||||
assert_response :created
|
||||
|
||||
post chunks_api_v1_import_session_url(session),
|
||||
params: params.merge(raw_file_content: build_ndjson(transaction_records)),
|
||||
headers: api_headers(@api_key)
|
||||
|
||||
assert_response :conflict
|
||||
assert_equal "import_session_conflict", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "requires chunk sequence" do
|
||||
session = build_import_session
|
||||
|
||||
post chunks_api_v1_import_session_url(session),
|
||||
params: { raw_file_content: build_ndjson(entity_records) },
|
||||
headers: api_headers(@api_key)
|
||||
|
||||
assert_response :bad_request
|
||||
assert_equal "bad_request", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "rejects malformed chunk sequence values" do
|
||||
session = build_import_session
|
||||
|
||||
post chunks_api_v1_import_session_url(session),
|
||||
params: { sequence: "1abc", raw_file_content: build_ndjson(entity_records) },
|
||||
headers: api_headers(@api_key)
|
||||
|
||||
assert_response :conflict
|
||||
assert_equal "import_session_conflict", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "shows import session with read scope" do
|
||||
session = build_import_session
|
||||
|
||||
get api_v1_import_session_url(session), headers: api_headers(@read_only_api_key)
|
||||
|
||||
assert_response :success
|
||||
data = JSON.parse(response.body)["data"]
|
||||
assert_equal session.id, data["id"]
|
||||
assert_equal "SureImport", data["type"]
|
||||
end
|
||||
|
||||
test "shows chunks in sequence order" do
|
||||
session = build_import_session
|
||||
session.imports.create!(
|
||||
family: @family,
|
||||
type: "SureImport",
|
||||
sequence: 2,
|
||||
checksum: Digest::SHA256.hexdigest("two")
|
||||
)
|
||||
session.imports.create!(
|
||||
family: @family,
|
||||
type: "SureImport",
|
||||
sequence: 1,
|
||||
checksum: Digest::SHA256.hexdigest("one")
|
||||
)
|
||||
|
||||
get api_v1_import_session_url(session), headers: api_headers(@api_key)
|
||||
|
||||
assert_response :success
|
||||
assert_equal [ 1, 2 ], JSON.parse(response.body).dig("data", "chunks").map { |chunk| chunk["sequence"] }
|
||||
end
|
||||
|
||||
test "requires write scope for session mutation" do
|
||||
assert_no_difference("ImportSession.count") do
|
||||
post api_v1_import_sessions_url,
|
||||
params: { type: "SureImport" },
|
||||
headers: api_headers(@read_only_api_key)
|
||||
end
|
||||
|
||||
assert_response :forbidden
|
||||
assert_equal "insufficient_scope", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "rejects publishing a session with no chunks" do
|
||||
session = @family.import_sessions.create!
|
||||
|
||||
post publish_api_v1_import_session_url(session), headers: api_headers(@api_key)
|
||||
|
||||
assert_response :conflict
|
||||
assert_equal "import_session_conflict", JSON.parse(response.body)["error"]
|
||||
end
|
||||
|
||||
test "returns stable error when publish cannot enqueue" do
|
||||
session = build_import_session
|
||||
session.attach_chunk!(
|
||||
sequence: 1,
|
||||
content: build_ndjson(entity_records),
|
||||
filename: "entities.ndjson",
|
||||
content_type: "application/x-ndjson"
|
||||
)
|
||||
session.attach_chunk!(
|
||||
sequence: 2,
|
||||
content: build_ndjson(transaction_records),
|
||||
filename: "transactions.ndjson",
|
||||
content_type: "application/x-ndjson"
|
||||
)
|
||||
|
||||
ImportSessionJob.stub(:perform_later, ->(_import_session) { raise StandardError, "redis://secret.local/0" }) do
|
||||
post publish_api_v1_import_session_url(session), headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :service_unavailable
|
||||
body = JSON.parse(response.body)
|
||||
assert_equal "import_enqueue_failed", body["error"]
|
||||
assert_equal "Import session could not be queued.", body["message"]
|
||||
assert_no_match(/secret/, response.body)
|
||||
end
|
||||
|
||||
test "does not expose another family's import session" do
|
||||
other_family = Family.create!(name: "Other Family", currency: "USD", locale: "en")
|
||||
other_session = other_family.import_sessions.create!
|
||||
|
||||
get api_v1_import_session_url(other_session), headers: api_headers(@api_key)
|
||||
|
||||
assert_response :not_found
|
||||
end
|
||||
|
||||
private
|
||||
def build_import_session
|
||||
@family.import_sessions.create!(expected_chunks: 2)
|
||||
end
|
||||
|
||||
def entity_records
|
||||
[
|
||||
{
|
||||
type: "Account",
|
||||
data: {
|
||||
id: "api-acct-1",
|
||||
name: "API Session Checking",
|
||||
balance: "100.00",
|
||||
currency: "USD",
|
||||
accountable_type: "Depository"
|
||||
}
|
||||
},
|
||||
{
|
||||
type: "Category",
|
||||
data: {
|
||||
id: "api-cat-1",
|
||||
name: "API Groceries",
|
||||
color: "#407706",
|
||||
classification: "expense"
|
||||
}
|
||||
},
|
||||
{
|
||||
type: "Merchant",
|
||||
data: {
|
||||
id: "api-merchant-1",
|
||||
name: "API Market"
|
||||
}
|
||||
},
|
||||
{
|
||||
type: "Tag",
|
||||
data: {
|
||||
id: "api-tag-1",
|
||||
name: "API Weekly"
|
||||
}
|
||||
}
|
||||
]
|
||||
end
|
||||
|
||||
def transaction_records
|
||||
[
|
||||
{
|
||||
type: "Transaction",
|
||||
data: {
|
||||
id: "api-txn-1",
|
||||
account_id: "api-acct-1",
|
||||
category_id: "api-cat-1",
|
||||
merchant_id: "api-merchant-1",
|
||||
tag_ids: [ "api-tag-1" ],
|
||||
date: "2024-01-15",
|
||||
amount: "-12.34",
|
||||
currency: "USD",
|
||||
name: "API Grocery Run"
|
||||
}
|
||||
}
|
||||
]
|
||||
end
|
||||
|
||||
def build_ndjson(records)
|
||||
records.map(&:to_json).join("\n")
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user