Files
sure/app/controllers/api/v1/import_sessions_controller.rb
ghost 6e04c6927d feat(imports): add SureImport session batches (#1785)
* feat(imports): add SureImport session batches

Add first-class SureImport sessions for ordered multi-file NDJSON imports.

Persist source mappings across chunks, make session/chunk processing idempotent, expose progress readback, and keep existing single-file import behavior compatible.

Includes the devcontainer libvips runtime dependency needed by ActiveStorage variant tests.

Addresses #1610.

Related to #1458.

* fix(imports): avoid scanner-like API key test data

* test(imports): assert skipped balances are not persisted

* fix(imports): harden session publish retries

Validate expected import chunk sequences exactly before publish, and restore session state with error details when enqueueing the publish job fails.

* fix(imports): close session retry edge cases

Backfill expected chunk counts after client-session insert races and enqueue import-session jobs after the status transition commits. Persist a safe enqueue failure body so API readback does not expose raw queue errors.

* fix(imports): address session publish review gaps

Remove dead transaction external-id assignment, harden session publish retry/sync behavior, align session chunk status docs, and add regression coverage for partial retries and safe enqueue error readback.

* fix(imports): include sessions in family reset

Clear import sessions through the family reset job so chunk imports and source mappings do not survive a reset.

Expose import session and source mapping counts in the reset status response and regenerated OpenAPI schema so polling reflects the full reset surface.

* test(imports): cover split import mapping invariants

* test(imports): cover session verification invariants

* fix(imports): scope SureImport session reimports

* Tighten SureImport session batching

* fix(imports): export rule source ids for sessions

* test(imports): stabilize rule id export assertion

* test(imports): restore reset status session fixture
2026-06-04 11:48:44 +02:00

196 lines
6.1 KiB
Ruby

# frozen_string_literal: true
class Api::V1::ImportSessionsController < Api::V1::BaseController
before_action :ensure_read_scope, only: [ :show ]
before_action :ensure_write_scope, only: [ :create, :create_chunk, :publish ]
before_action :set_import_session, only: [ :show, :create_chunk, :publish ]
def create
@import_session = ImportSession.create_or_find_for!(
family: Current.family,
import_type: params[:type].to_s,
client_session_id: params[:client_session_id].presence,
expected_chunks: expected_chunks_param
)
render_import_session(status: :created)
rescue ImportSession::ConflictError => e
render_import_session_conflict(e.message)
rescue ActiveRecord::RecordInvalid => e
render_error(
"validation_failed",
"Import session could not be created",
:unprocessable_entity,
errors: e.record.errors.full_messages
)
end
def show
render_import_session
end
def create_chunk
content, filename, content_type = sure_import_upload_attributes
return unless content
@import_session.attach_chunk!(
sequence: sequence_param,
client_chunk_id: params[:client_chunk_id].presence,
content: content,
filename: filename,
content_type: content_type
)
@import_session.reload
render_import_session(status: :created)
rescue ImportSession::ConflictError => e
render_import_session_conflict(e.message)
rescue ActiveRecord::RecordInvalid => e
render_error(
"validation_failed",
"Import chunk could not be created",
:unprocessable_entity,
errors: e.record.errors.full_messages
)
end
def publish
@import_session.publish_later
@import_session.reload
render_import_session(status: :accepted)
rescue Import::MaxRowCountExceededError
render_error("max_row_count_exceeded", "Import session has too many rows to publish.", :unprocessable_entity)
rescue ImportSession::EnqueueError
render_error("import_enqueue_failed", "Import session could not be queued.", :service_unavailable)
rescue ImportSession::ConflictError => e
render_import_session_conflict(e.message)
end
private
def set_import_session
@import_session = Current.family.import_sessions.find(params[:id])
end
def ensure_read_scope
authorize_scope!(:read)
end
def ensure_write_scope
authorize_scope!(:write)
end
def expected_chunks_param
return if params[:expected_chunks].blank?
params[:expected_chunks]
end
def sequence_param
raise ActionController::ParameterMissing.new(:sequence) if params[:sequence].blank?
params[:sequence]
end
def sure_import_upload_attributes
if params[:file].present?
sure_import_file_upload_attributes(params[:file])
elsif params[:raw_file_content].present?
sure_import_raw_content_attributes(params[:raw_file_content].to_s)
else
render_error("missing_content", "Provide a Sure NDJSON file or raw_file_content.", :unprocessable_entity)
nil
end
end
def sure_import_file_upload_attributes(file)
if file.size > SureImport.max_ndjson_size
render_error(
"file_too_large",
"File is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB.",
:unprocessable_entity
)
return
end
extension = File.extname(file.original_filename.to_s).downcase
unless SureImport::ALLOWED_NDJSON_CONTENT_TYPES.include?(file.content_type) || extension.in?(%w[.ndjson .json])
render_error("invalid_file_type", "Invalid file type. Please upload a Sure NDJSON file.", :unprocessable_entity)
return
end
sure_import_validated_attributes(
content: file.read,
filename: file.original_filename.presence || "sure-import.ndjson",
content_type: file.content_type.presence || "application/x-ndjson"
)
end
def sure_import_raw_content_attributes(content)
if content.bytesize > SureImport.max_ndjson_size
render_error(
"content_too_large",
"Content is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB.",
:unprocessable_entity
)
return
end
sure_import_validated_attributes(
content: content,
filename: "sure-import.ndjson",
content_type: "application/x-ndjson"
)
end
def sure_import_validated_attributes(content:, filename:, content_type:)
unless SureImport.valid_ndjson_first_line?(content)
render_error("invalid_ndjson", "Invalid Sure NDJSON content.", :unprocessable_entity)
return
end
[ content, filename, content_type ]
end
def render_import_session_conflict(message)
render_error("import_session_conflict", message, :conflict)
end
def render_import_session(status: :ok)
chunks = @import_session.imports.ordered_by_sequence.map do |import|
{
id: import.id,
sequence: import.sequence,
client_chunk_id: import.client_chunk_id,
status: import.status,
rows_count: import.rows_count,
summary: import.summary || {},
error: import.error_details.presence,
created_at: import.created_at,
updated_at: import.updated_at
}
end
render json: {
data: {
id: @import_session.id,
type: @import_session.import_type,
status: @import_session.status,
client_session_id: @import_session.client_session_id,
expected_chunks: @import_session.expected_chunks,
chunks_count: chunks.size,
summary: @import_session.summary || {},
error: @import_session.error_details.presence,
created_at: @import_session.created_at,
updated_at: @import_session.updated_at,
chunks: chunks
}
}, status: status
end
def render_error(error, message, status, errors: nil)
payload = { error: error, message: message }
payload[:errors] = errors if errors
render json: payload, status: status
end
end