From da42423475b9eb17c3fbfe7a7ac9c22b5ad04bf0 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Fri, 1 May 2026 14:56:18 -0600 Subject: [PATCH] feat(api): accept Sure NDJSON imports (#1601) * feat(api): accept Sure NDJSON imports * fix(api): preserve uploaded Sure imports on publish errors * fix(api): reset preserved Sure imports after enqueue failure * fix(api): tighten Sure import upload handling * test(api): align import API key fixtures * docs(api): document import publish failure IDs --- app/controllers/api/v1/imports_controller.rb | 146 ++++++++++ docs/api/openapi.yaml | 199 +++++++++++++- spec/requests/api/v1/imports_spec.rb | 88 ++++-- spec/swagger_helper.rb | 17 +- .../api/v1/imports_controller_test.rb | 254 +++++++++++++++++- 5 files changed, 669 insertions(+), 35 deletions(-) diff --git a/app/controllers/api/v1/imports_controller.rb b/app/controllers/api/v1/imports_controller.rb index e6b78f8bd..0243b13c4 100644 --- a/app/controllers/api/v1/imports_controller.rb +++ b/app/controllers/api/v1/imports_controller.rb @@ -50,6 +50,7 @@ class Api::V1::ImportsController < Api::V1::BaseController # 1. Determine type and validate type = params[:type].to_s type = "TransactionImport" unless Import::TYPES.include?(type) + return create_sure_import(family) if type == "SureImport" # 2. Build the import object with permitted config attributes @import = family.imports.build(import_config_params.merge(type: type)) @@ -158,6 +159,151 @@ class Api::V1::ImportsController < Api::V1::BaseController ) end + def create_sure_import(family) + content, filename, content_type = sure_import_upload_attributes + return unless content + + begin + @import = persist_sure_import!(family, content, filename, content_type) + rescue ActiveRecord::RecordInvalid => e + render json: { + error: "validation_failed", + message: "Import could not be created", + errors: e.record&.errors&.full_messages || @import&.errors&.full_messages || [] + }, status: :unprocessable_entity + return + rescue StandardError => e + Rails.logger.error "Sure import creation failed: #{e.message}" + render json: { + error: "internal_server_error", + message: "Import could not be created" + }, status: :internal_server_error + return + end + + begin + @import.publish_later if @import.publishable? && params[:publish] == "true" + rescue Import::MaxRowCountExceededError + render json: { + error: "max_row_count_exceeded", + message: "Import was uploaded but has too many rows to publish automatically.", + import_id: @import.id + }, status: :unprocessable_entity + return + rescue StandardError => e + Rails.logger.error "Sure import publish failed for import #{@import.id}: #{e.message}" + restore_pending_sure_import_after_publish_failure + render json: { + error: "publish_failed", + message: "Import was uploaded but could not be queued for processing.", + import_id: @import.id + }, status: :internal_server_error + return + end + + render :show, status: :created + end + + def persist_sure_import!(family, content, filename, content_type) + import = nil + import = family.imports.create!(type: "SureImport") + import.ndjson_file.attach( + io: StringIO.new(content), + filename: filename, + content_type: content_type + ) + import.sync_ndjson_rows_count! + import + rescue StandardError => e + clean_up_failed_sure_import(import) + raise + end + + def restore_pending_sure_import_after_publish_failure + # Import#publish_later flips status to importing before enqueueing the job. + @import.update_column(:status, "pending") if @import&.persisted? && @import.importing? + end + + def clean_up_failed_sure_import(import) + return unless import + + begin + import.ndjson_file.purge if import.ndjson_file.attached? + rescue StandardError => e + Rails.logger.warn "Failed to purge Sure import attachment #{import.id}: #{e.message}" + ensure + import.destroy if import.persisted? + end + end + + def sure_import_upload_attributes + if params[:file].present? + sure_import_file_upload_attributes(params[:file]) + elsif params[:raw_file_content].present? + sure_import_raw_content_attributes(params[:raw_file_content].to_s) + else + render json: { + error: "missing_content", + message: "Provide a Sure NDJSON file or raw_file_content." + }, status: :unprocessable_entity + nil + end + end + + def sure_import_file_upload_attributes(file) + if file.size > SureImport::MAX_NDJSON_SIZE + render json: { + error: "file_too_large", + message: "File is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB." + }, status: :unprocessable_entity + return + end + + extension = File.extname(file.original_filename.to_s).downcase + unless SureImport::ALLOWED_NDJSON_CONTENT_TYPES.include?(file.content_type) || extension.in?(%w[.ndjson .json]) + render json: { + error: "invalid_file_type", + message: "Invalid file type. Please upload a Sure NDJSON file." + }, status: :unprocessable_entity + return + end + + content = file.read + sure_import_validated_attributes( + content: content, + filename: file.original_filename.presence || "sure-import.ndjson", + content_type: file.content_type.presence || "application/x-ndjson" + ) + end + + def sure_import_raw_content_attributes(content) + if content.bytesize > SureImport::MAX_NDJSON_SIZE + render json: { + error: "content_too_large", + message: "Content is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB." + }, status: :unprocessable_entity + return + end + + sure_import_validated_attributes( + content: content, + filename: "sure-import.ndjson", + content_type: "application/x-ndjson" + ) + end + + def sure_import_validated_attributes(content:, filename:, content_type:) + unless SureImport.valid_ndjson_first_line?(content) + render json: { + error: "invalid_ndjson", + message: "Invalid Sure NDJSON content." + }, status: :unprocessable_entity + return + end + + [ content, filename, content_type ] + end + def safe_page_param page = params[:page].to_i page > 0 ? page : 1 diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 345f81e70..a16a8c582 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -61,6 +61,22 @@ components: nullable: true description: Validation error messages (alternative to details used by trades, valuations, etc.) + ErrorResponseWithImportId: + type: object + required: + - error + - import_id + properties: + error: + type: string + message: + type: string + nullable: true + import_id: + type: string + format: uuid + description: Import ID preserved for retry or inspection after upload succeeds + but publish fails MfaRequiredResponse: type: object required: @@ -888,6 +904,7 @@ components: - MintImport - CategoryImport - RuleImport + - SureImport status: type: string enum: @@ -934,6 +951,7 @@ components: - MintImport - CategoryImport - RuleImport + - SureImport status: type: string enum: @@ -2273,6 +2291,7 @@ paths: - MintImport - CategoryImport - RuleImport + - SureImport responses: '200': description: imports filtered by type @@ -2282,7 +2301,8 @@ paths: "$ref": "#/components/schemas/ImportCollection" post: summary: Create import - description: Create a new import from raw CSV content. + description: Create a new import from raw CSV content, inline Sure NDJSON content, + or an uploaded Sure NDJSON file. tags: - Imports security: @@ -2300,7 +2320,15 @@ paths: content: application/json: schema: - "$ref": "#/components/schemas/ErrorResponse" + oneOf: + - "$ref": "#/components/schemas/ErrorResponse" + - "$ref": "#/components/schemas/ErrorResponseWithImportId" + '500': + description: import uploaded but publish enqueue failed + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorResponseWithImportId" requestBody: content: application/json: @@ -2309,7 +2337,8 @@ paths: properties: raw_file_content: type: string - description: The raw CSV content as a string + description: Raw CSV or Sure NDJSON content as a string. Required + for SureImport unless a multipart file is uploaded. type: type: string enum: @@ -2319,6 +2348,7 @@ paths: - MintImport - CategoryImport - RuleImport + - SureImport description: Import type (defaults to TransactionImport) account_id: type: string @@ -2330,25 +2360,51 @@ paths: if configuration is valid date_col_label: type: string - description: Header name for the date column + description: CSV imports only. Header name for the date column amount_col_label: type: string - description: Header name for the amount column + description: CSV imports only. Header name for the amount column name_col_label: type: string - description: Header name for the transaction name column + description: CSV imports only. Header name for the transaction name + column category_col_label: type: string - description: Header name for the category column + description: CSV imports only. Header name for the category column tags_col_label: type: string - description: Header name for the tags column + description: CSV imports only. Header name for the tags column notes_col_label: type: string - description: Header name for the notes column + description: CSV imports only. Header name for the notes column + account_col_label: + type: string + description: CSV imports only. Header name for the account column + when importing rows across multiple accounts + qty_col_label: + type: string + description: CSV trade imports only. Header name for the quantity + column + ticker_col_label: + type: string + description: CSV trade imports only. Header name for the ticker + column + price_col_label: + type: string + description: CSV trade imports only. Header name for the price column + entity_type_col_label: + type: string + description: CSV imports only. Header name for the entity type column + currency_col_label: + type: string + description: CSV imports only. Header name for the currency column + exchange_operating_mic_col_label: + type: string + description: CSV trade imports only. Header name for the exchange + operating MIC column date_format: type: string - description: Date format pattern (e.g., "%m/%d/%Y") + description: CSV imports only. Date format pattern (e.g., "%m/%d/%Y") number_format: type: string enum: @@ -2356,20 +2412,135 @@ paths: - 1.234,56 - 1 234,56 - '1,234' - description: Number format for parsing amounts + description: CSV imports only. Number format for parsing amounts signage_convention: type: string enum: - inflows_positive - inflows_negative - description: How to interpret positive/negative amounts + description: CSV imports only. How to interpret positive/negative + amounts col_sep: type: string enum: - "," - ";" - description: Column separator - required: true + description: CSV imports only. Column separator + amount_type_strategy: + type: string + enum: + - signed_amount + - custom_column + description: CSV imports only. Amount parsing strategy + amount_type_inflow_value: + type: string + description: CSV imports only. Column value that marks an amount + as an inflow when using custom_column strategy + multipart/form-data: + schema: + type: object + properties: + raw_file_content: + type: string + description: Raw CSV or Sure NDJSON content as a string. Required + for SureImport unless a multipart file is uploaded. + type: + type: string + enum: + - TransactionImport + - TradeImport + - AccountImport + - MintImport + - CategoryImport + - RuleImport + - SureImport + description: Import type (defaults to TransactionImport) + account_id: + type: string + format: uuid + description: Account ID to import into + publish: + type: string + description: Set to "true" to automatically queue for processing + if configuration is valid + date_col_label: + type: string + description: CSV imports only. Header name for the date column + amount_col_label: + type: string + description: CSV imports only. Header name for the amount column + name_col_label: + type: string + description: CSV imports only. Header name for the transaction name + column + category_col_label: + type: string + description: CSV imports only. Header name for the category column + tags_col_label: + type: string + description: CSV imports only. Header name for the tags column + notes_col_label: + type: string + description: CSV imports only. Header name for the notes column + account_col_label: + type: string + description: CSV imports only. Header name for the account column + when importing rows across multiple accounts + qty_col_label: + type: string + description: CSV trade imports only. Header name for the quantity + column + ticker_col_label: + type: string + description: CSV trade imports only. Header name for the ticker + column + price_col_label: + type: string + description: CSV trade imports only. Header name for the price column + entity_type_col_label: + type: string + description: CSV imports only. Header name for the entity type column + currency_col_label: + type: string + description: CSV imports only. Header name for the currency column + exchange_operating_mic_col_label: + type: string + description: CSV trade imports only. Header name for the exchange + operating MIC column + date_format: + type: string + description: CSV imports only. Date format pattern (e.g., "%m/%d/%Y") + number_format: + type: string + enum: + - '1,234.56' + - 1.234,56 + - 1 234,56 + - '1,234' + description: CSV imports only. Number format for parsing amounts + signage_convention: + type: string + enum: + - inflows_positive + - inflows_negative + description: CSV imports only. How to interpret positive/negative + amounts + col_sep: + type: string + enum: + - "," + - ";" + description: CSV imports only. Column separator + amount_type_strategy: + type: string + enum: + - signed_amount + - custom_column + description: CSV imports only. Amount parsing strategy + amount_type_inflow_value: + type: string + description: CSV imports only. Column value that marks an amount + as an inflow when using custom_column strategy "/api/v1/imports/{id}": parameters: - name: id diff --git a/spec/requests/api/v1/imports_spec.rb b/spec/requests/api/v1/imports_spec.rb index 44580e747..220ee1f89 100644 --- a/spec/requests/api/v1/imports_spec.rb +++ b/spec/requests/api/v1/imports_spec.rb @@ -76,7 +76,7 @@ RSpec.describe 'API V1 Imports', type: :request do schema: { type: :string, enum: %w[pending complete importing reverting revert_failed failed] } parameter name: :type, in: :query, required: false, description: 'Filter by import type', - schema: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport] } + schema: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] } response '200', 'imports listed' do schema '$ref' => '#/components/schemas/ImportCollection' @@ -102,22 +102,22 @@ RSpec.describe 'API V1 Imports', type: :request do end post 'Create import' do - description 'Create a new import from raw CSV content.' + description 'Create a new import from raw CSV content, inline Sure NDJSON content, or an uploaded Sure NDJSON file.' tags 'Imports' security [ { apiKeyAuth: [] } ] - consumes 'application/json' + consumes 'application/json', 'multipart/form-data' produces 'application/json' - parameter name: :body, in: :body, required: true, schema: { + parameter name: :body, in: :body, required: false, schema: { type: :object, properties: { raw_file_content: { type: :string, - description: 'The raw CSV content as a string' + description: 'Raw CSV or Sure NDJSON content as a string. Required for SureImport unless a multipart file is uploaded.' }, type: { type: :string, - enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport], + enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport], description: 'Import type (defaults to TransactionImport)' }, account_id: { @@ -131,46 +131,83 @@ RSpec.describe 'API V1 Imports', type: :request do }, date_col_label: { type: :string, - description: 'Header name for the date column' + description: 'CSV imports only. Header name for the date column' }, amount_col_label: { type: :string, - description: 'Header name for the amount column' + description: 'CSV imports only. Header name for the amount column' }, name_col_label: { type: :string, - description: 'Header name for the transaction name column' + description: 'CSV imports only. Header name for the transaction name column' }, category_col_label: { type: :string, - description: 'Header name for the category column' + description: 'CSV imports only. Header name for the category column' }, tags_col_label: { type: :string, - description: 'Header name for the tags column' + description: 'CSV imports only. Header name for the tags column' }, notes_col_label: { type: :string, - description: 'Header name for the notes column' + description: 'CSV imports only. Header name for the notes column' + }, + account_col_label: { + type: :string, + description: 'CSV imports only. Header name for the account column when importing rows across multiple accounts' + }, + qty_col_label: { + type: :string, + description: 'CSV trade imports only. Header name for the quantity column' + }, + ticker_col_label: { + type: :string, + description: 'CSV trade imports only. Header name for the ticker column' + }, + price_col_label: { + type: :string, + description: 'CSV trade imports only. Header name for the price column' + }, + entity_type_col_label: { + type: :string, + description: 'CSV imports only. Header name for the entity type column' + }, + currency_col_label: { + type: :string, + description: 'CSV imports only. Header name for the currency column' + }, + exchange_operating_mic_col_label: { + type: :string, + description: 'CSV trade imports only. Header name for the exchange operating MIC column' }, date_format: { type: :string, - description: 'Date format pattern (e.g., "%m/%d/%Y")' + description: 'CSV imports only. Date format pattern (e.g., "%m/%d/%Y")' }, number_format: { type: :string, enum: [ '1,234.56', '1.234,56', '1 234,56', '1,234' ], - description: 'Number format for parsing amounts' + description: 'CSV imports only. Number format for parsing amounts' }, signage_convention: { type: :string, enum: %w[inflows_positive inflows_negative], - description: 'How to interpret positive/negative amounts' + description: 'CSV imports only. How to interpret positive/negative amounts' }, col_sep: { type: :string, enum: [ ',', ';' ], - description: 'Column separator' + description: 'CSV imports only. Column separator' + }, + amount_type_strategy: { + type: :string, + enum: %w[signed_amount custom_column], + description: 'CSV imports only. Amount parsing strategy' + }, + amount_type_inflow_value: { + type: :string, + description: 'CSV imports only. Column value that marks an amount as an inflow when using custom_column strategy' } } } @@ -193,7 +230,10 @@ RSpec.describe 'API V1 Imports', type: :request do end response '422', 'validation error - file too large' do - schema '$ref' => '#/components/schemas/ErrorResponse' + schema oneOf: [ + { '$ref' => '#/components/schemas/ErrorResponse' }, + { '$ref' => '#/components/schemas/ErrorResponseWithImportId' } + ] let(:body) do { @@ -204,6 +244,20 @@ RSpec.describe 'API V1 Imports', type: :request do run_test! end + + response '500', 'import uploaded but publish enqueue failed' do + schema '$ref' => '#/components/schemas/ErrorResponseWithImportId' + + let(:body) do + { + raw_file_content: { type: 'Account', data: { id: 'account_1', name: 'Checking' } }.to_json, + type: 'SureImport', + publish: 'true' + } + end + + run_test! + end end end diff --git a/spec/swagger_helper.rb b/spec/swagger_helper.rb index cd94929a0..031472c9e 100644 --- a/spec/swagger_helper.rb +++ b/spec/swagger_helper.rb @@ -64,6 +64,19 @@ RSpec.configure do |config| } } }, + ErrorResponseWithImportId: { + type: :object, + required: %w[error import_id], + properties: { + error: { type: :string }, + message: { type: :string, nullable: true }, + import_id: { + type: :string, + format: :uuid, + description: 'Import ID preserved for retry or inspection after upload succeeds but publish fails' + } + } + }, MfaRequiredResponse: { type: :object, required: %w[error mfa_required], @@ -524,7 +537,7 @@ RSpec.configure do |config| required: %w[id type status created_at updated_at], properties: { id: { type: :string, format: :uuid }, - type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport] }, + type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] }, status: { type: :string, enum: %w[pending complete importing reverting revert_failed failed] }, created_at: { type: :string, format: :'date-time' }, updated_at: { type: :string, format: :'date-time' }, @@ -538,7 +551,7 @@ RSpec.configure do |config| required: %w[id type status created_at updated_at], properties: { id: { type: :string, format: :uuid }, - type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport] }, + type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] }, status: { type: :string, enum: %w[pending complete importing reverting revert_failed failed] }, created_at: { type: :string, format: :'date-time' }, updated_at: { type: :string, format: :'date-time' }, diff --git a/test/controllers/api/v1/imports_controller_test.rb b/test/controllers/api/v1/imports_controller_test.rb index fb5ac9bc2..5e1099158 100644 --- a/test/controllers/api/v1/imports_controller_test.rb +++ b/test/controllers/api/v1/imports_controller_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "test_helper" class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest @@ -13,7 +15,8 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest user: @user, name: "Test Read-Write Key", scopes: [ "read_write" ], - display_key: "test_rw_#{SecureRandom.hex(8)}" + display_key: "test_rw_#{SecureRandom.hex(8)}", + source: "web" ) @read_only_api_key = ApiKey.create!( @@ -131,6 +134,253 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest assert_equal '[{"action_type":"set_transaction_category","value":"Groceries"}]', row.actions end + test "should create Sure import with raw NDJSON content" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + + assert_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content + }, + headers: api_headers(@api_key) + end + + assert_response :created + + json_response = JSON.parse(response.body) + import = Import.find(json_response["data"]["id"]) + + assert_instance_of SureImport, import + assert import.ndjson_file.attached? + assert_equal 1, import.rows_count + assert_equal "pending", import.status + end + + test "should require authentication for Sure import" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content + } + end + + assert_response :unauthorized + end + + test "should reject Sure import with read-only API key" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :forbidden + json_response = JSON.parse(response.body) + assert_equal "insufficient_scope", json_response["error"] + end + + test "should create Sure import with uploaded NDJSON file" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + valid_file = Rack::Test::UploadedFile.new( + StringIO.new(ndjson_content), + "application/x-ndjson", + original_filename: "sure-backup.ndjson" + ) + + assert_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + file: valid_file + }, + headers: api_headers(@api_key) + end + + assert_response :created + + import = Import.find(JSON.parse(response.body)["data"]["id"]) + assert_instance_of SureImport, import + assert import.ndjson_file.attached? + assert_equal 1, import.rows_count + end + + test "should reject Sure import with no file or raw content" do + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport" + }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "missing_content", json_response["error"] + end + + test "should reject Sure import uploaded file exceeding max size" do + test_limit = 1.kilobyte + large_file = Rack::Test::UploadedFile.new( + StringIO.new("x" * (test_limit + 1)), + "application/x-ndjson", + original_filename: "large.ndjson" + ) + + original_value = SureImport::MAX_NDJSON_SIZE + SureImport.send(:remove_const, :MAX_NDJSON_SIZE) + SureImport.const_set(:MAX_NDJSON_SIZE, test_limit) + + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + file: large_file + }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "file_too_large", json_response["error"] + ensure + SureImport.send(:remove_const, :MAX_NDJSON_SIZE) + SureImport.const_set(:MAX_NDJSON_SIZE, original_value) + end + + test "should reject Sure import uploaded file with invalid type" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + invalid_file = Rack::Test::UploadedFile.new( + StringIO.new(ndjson_content), + "application/pdf", + original_filename: "sure-backup.pdf" + ) + + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + file: invalid_file + }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "invalid_file_type", json_response["error"] + end + + test "should clean up Sure import if row sync fails" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + SureImport.any_instance.stubs(:sync_ndjson_rows_count!).raises(StandardError, "sync failed") + + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content + }, + headers: api_headers(@api_key) + end + + assert_response :internal_server_error + json_response = JSON.parse(response.body) + assert_equal "internal_server_error", json_response["error"] + end + + test "should clean up Sure import if row sync validation fails" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + invalid_import = SureImport.new + invalid_import.errors.add(:base, "invalid rows") + SureImport.any_instance.stubs(:sync_ndjson_rows_count!).raises(ActiveRecord::RecordInvalid.new(invalid_import)) + + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content + }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "validation_failed", json_response["error"] + assert_includes json_response["errors"], "invalid rows" + end + + test "should preserve Sure import if publish queueing fails" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + ImportJob.stubs(:perform_later).raises(StandardError, "queue offline") + + assert_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content, + publish: "true" + }, + headers: api_headers(@api_key) + end + + assert_response :internal_server_error + json_response = JSON.parse(response.body) + assert_equal "publish_failed", json_response["error"] + + import = Import.find(json_response["import_id"]) + assert_instance_of SureImport, import + assert import.ndjson_file.attached? + assert_equal 1, import.rows_count + assert_equal "pending", import.status + end + + test "should preserve Sure import if auto publish exceeds row count" do + ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json + SureImport.any_instance.stubs(:publish_later).raises(Import::MaxRowCountExceededError) + + assert_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content, + publish: "true" + }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "max_row_count_exceeded", json_response["error"] + + import = Import.find(json_response["import_id"]) + assert_instance_of SureImport, import + assert import.ndjson_file.attached? + assert_equal 1, import.rows_count + end + + test "should reject invalid Sure import NDJSON content" do + assert_no_difference("Import.count") do + post api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: "not ndjson" + }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "invalid_ndjson", json_response["error"] + end + test "should create import and auto-publish when configured and requested" do csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction" @@ -257,6 +507,6 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest private def api_headers(api_key) - { "X-Api-Key" => api_key.display_key } + { "X-Api-Key" => api_key.plain_key } end end