mirror of
https://github.com/we-promise/sure.git
synced 2026-05-07 12:54:04 +00:00
feat(api): accept Sure NDJSON imports (#1601)
* feat(api): accept Sure NDJSON imports * fix(api): preserve uploaded Sure imports on publish errors * fix(api): reset preserved Sure imports after enqueue failure * fix(api): tighten Sure import upload handling * test(api): align import API key fixtures * docs(api): document import publish failure IDs
This commit is contained in:
ghost
@@ -50,6 +50,7 @@ class Api::V1::ImportsController < Api::V1::BaseController
|
||||
# 1. Determine type and validate
|
||||
type = params[:type].to_s
|
||||
type = "TransactionImport" unless Import::TYPES.include?(type)
|
||||
return create_sure_import(family) if type == "SureImport"
|
||||
|
||||
# 2. Build the import object with permitted config attributes
|
||||
@import = family.imports.build(import_config_params.merge(type: type))
|
||||
@@ -158,6 +159,151 @@ class Api::V1::ImportsController < Api::V1::BaseController
|
||||
)
|
||||
end
|
||||
|
||||
def create_sure_import(family)
|
||||
content, filename, content_type = sure_import_upload_attributes
|
||||
return unless content
|
||||
|
||||
begin
|
||||
@import = persist_sure_import!(family, content, filename, content_type)
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
render json: {
|
||||
error: "validation_failed",
|
||||
message: "Import could not be created",
|
||||
errors: e.record&.errors&.full_messages || @import&.errors&.full_messages || []
|
||||
}, status: :unprocessable_entity
|
||||
return
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "Sure import creation failed: #{e.message}"
|
||||
render json: {
|
||||
error: "internal_server_error",
|
||||
message: "Import could not be created"
|
||||
}, status: :internal_server_error
|
||||
return
|
||||
end
|
||||
|
||||
begin
|
||||
@import.publish_later if @import.publishable? && params[:publish] == "true"
|
||||
rescue Import::MaxRowCountExceededError
|
||||
render json: {
|
||||
error: "max_row_count_exceeded",
|
||||
message: "Import was uploaded but has too many rows to publish automatically.",
|
||||
import_id: @import.id
|
||||
}, status: :unprocessable_entity
|
||||
return
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "Sure import publish failed for import #{@import.id}: #{e.message}"
|
||||
restore_pending_sure_import_after_publish_failure
|
||||
render json: {
|
||||
error: "publish_failed",
|
||||
message: "Import was uploaded but could not be queued for processing.",
|
||||
import_id: @import.id
|
||||
}, status: :internal_server_error
|
||||
return
|
||||
end
|
||||
|
||||
render :show, status: :created
|
||||
end
|
||||
|
||||
def persist_sure_import!(family, content, filename, content_type)
|
||||
import = nil
|
||||
import = family.imports.create!(type: "SureImport")
|
||||
import.ndjson_file.attach(
|
||||
io: StringIO.new(content),
|
||||
filename: filename,
|
||||
content_type: content_type
|
||||
)
|
||||
import.sync_ndjson_rows_count!
|
||||
import
|
||||
rescue StandardError => e
|
||||
clean_up_failed_sure_import(import)
|
||||
raise
|
||||
end
|
||||
|
||||
def restore_pending_sure_import_after_publish_failure
|
||||
# Import#publish_later flips status to importing before enqueueing the job.
|
||||
@import.update_column(:status, "pending") if @import&.persisted? && @import.importing?
|
||||
end
|
||||
|
||||
def clean_up_failed_sure_import(import)
|
||||
return unless import
|
||||
|
||||
begin
|
||||
import.ndjson_file.purge if import.ndjson_file.attached?
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "Failed to purge Sure import attachment #{import.id}: #{e.message}"
|
||||
ensure
|
||||
import.destroy if import.persisted?
|
||||
end
|
||||
end
|
||||
|
||||
def sure_import_upload_attributes
|
||||
if params[:file].present?
|
||||
sure_import_file_upload_attributes(params[:file])
|
||||
elsif params[:raw_file_content].present?
|
||||
sure_import_raw_content_attributes(params[:raw_file_content].to_s)
|
||||
else
|
||||
render json: {
|
||||
error: "missing_content",
|
||||
message: "Provide a Sure NDJSON file or raw_file_content."
|
||||
}, status: :unprocessable_entity
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def sure_import_file_upload_attributes(file)
|
||||
if file.size > SureImport::MAX_NDJSON_SIZE
|
||||
render json: {
|
||||
error: "file_too_large",
|
||||
message: "File is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB."
|
||||
}, status: :unprocessable_entity
|
||||
return
|
||||
end
|
||||
|
||||
extension = File.extname(file.original_filename.to_s).downcase
|
||||
unless SureImport::ALLOWED_NDJSON_CONTENT_TYPES.include?(file.content_type) || extension.in?(%w[.ndjson .json])
|
||||
render json: {
|
||||
error: "invalid_file_type",
|
||||
message: "Invalid file type. Please upload a Sure NDJSON file."
|
||||
}, status: :unprocessable_entity
|
||||
return
|
||||
end
|
||||
|
||||
content = file.read
|
||||
sure_import_validated_attributes(
|
||||
content: content,
|
||||
filename: file.original_filename.presence || "sure-import.ndjson",
|
||||
content_type: file.content_type.presence || "application/x-ndjson"
|
||||
)
|
||||
end
|
||||
|
||||
def sure_import_raw_content_attributes(content)
|
||||
if content.bytesize > SureImport::MAX_NDJSON_SIZE
|
||||
render json: {
|
||||
error: "content_too_large",
|
||||
message: "Content is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB."
|
||||
}, status: :unprocessable_entity
|
||||
return
|
||||
end
|
||||
|
||||
sure_import_validated_attributes(
|
||||
content: content,
|
||||
filename: "sure-import.ndjson",
|
||||
content_type: "application/x-ndjson"
|
||||
)
|
||||
end
|
||||
|
||||
def sure_import_validated_attributes(content:, filename:, content_type:)
|
||||
unless SureImport.valid_ndjson_first_line?(content)
|
||||
render json: {
|
||||
error: "invalid_ndjson",
|
||||
message: "Invalid Sure NDJSON content."
|
||||
}, status: :unprocessable_entity
|
||||
return
|
||||
end
|
||||
|
||||
[ content, filename, content_type ]
|
||||
end
|
||||
|
||||
def safe_page_param
|
||||
page = params[:page].to_i
|
||||
page > 0 ? page : 1
|
||||
|
||||
@@ -61,6 +61,22 @@ components:
|
||||
nullable: true
|
||||
description: Validation error messages (alternative to details used by trades,
|
||||
valuations, etc.)
|
||||
ErrorResponseWithImportId:
|
||||
type: object
|
||||
required:
|
||||
- error
|
||||
- import_id
|
||||
properties:
|
||||
error:
|
||||
type: string
|
||||
message:
|
||||
type: string
|
||||
nullable: true
|
||||
import_id:
|
||||
type: string
|
||||
format: uuid
|
||||
description: Import ID preserved for retry or inspection after upload succeeds
|
||||
but publish fails
|
||||
MfaRequiredResponse:
|
||||
type: object
|
||||
required:
|
||||
@@ -888,6 +904,7 @@ components:
|
||||
- MintImport
|
||||
- CategoryImport
|
||||
- RuleImport
|
||||
- SureImport
|
||||
status:
|
||||
type: string
|
||||
enum:
|
||||
@@ -934,6 +951,7 @@ components:
|
||||
- MintImport
|
||||
- CategoryImport
|
||||
- RuleImport
|
||||
- SureImport
|
||||
status:
|
||||
type: string
|
||||
enum:
|
||||
@@ -2273,6 +2291,7 @@ paths:
|
||||
- MintImport
|
||||
- CategoryImport
|
||||
- RuleImport
|
||||
- SureImport
|
||||
responses:
|
||||
'200':
|
||||
description: imports filtered by type
|
||||
@@ -2282,7 +2301,8 @@ paths:
|
||||
"$ref": "#/components/schemas/ImportCollection"
|
||||
post:
|
||||
summary: Create import
|
||||
description: Create a new import from raw CSV content.
|
||||
description: Create a new import from raw CSV content, inline Sure NDJSON content,
|
||||
or an uploaded Sure NDJSON file.
|
||||
tags:
|
||||
- Imports
|
||||
security:
|
||||
@@ -2300,7 +2320,15 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
"$ref": "#/components/schemas/ErrorResponse"
|
||||
oneOf:
|
||||
- "$ref": "#/components/schemas/ErrorResponse"
|
||||
- "$ref": "#/components/schemas/ErrorResponseWithImportId"
|
||||
'500':
|
||||
description: import uploaded but publish enqueue failed
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
"$ref": "#/components/schemas/ErrorResponseWithImportId"
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
@@ -2309,7 +2337,8 @@ paths:
|
||||
properties:
|
||||
raw_file_content:
|
||||
type: string
|
||||
description: The raw CSV content as a string
|
||||
description: Raw CSV or Sure NDJSON content as a string. Required
|
||||
for SureImport unless a multipart file is uploaded.
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
@@ -2319,6 +2348,7 @@ paths:
|
||||
- MintImport
|
||||
- CategoryImport
|
||||
- RuleImport
|
||||
- SureImport
|
||||
description: Import type (defaults to TransactionImport)
|
||||
account_id:
|
||||
type: string
|
||||
@@ -2330,25 +2360,51 @@ paths:
|
||||
if configuration is valid
|
||||
date_col_label:
|
||||
type: string
|
||||
description: Header name for the date column
|
||||
description: CSV imports only. Header name for the date column
|
||||
amount_col_label:
|
||||
type: string
|
||||
description: Header name for the amount column
|
||||
description: CSV imports only. Header name for the amount column
|
||||
name_col_label:
|
||||
type: string
|
||||
description: Header name for the transaction name column
|
||||
description: CSV imports only. Header name for the transaction name
|
||||
column
|
||||
category_col_label:
|
||||
type: string
|
||||
description: Header name for the category column
|
||||
description: CSV imports only. Header name for the category column
|
||||
tags_col_label:
|
||||
type: string
|
||||
description: Header name for the tags column
|
||||
description: CSV imports only. Header name for the tags column
|
||||
notes_col_label:
|
||||
type: string
|
||||
description: Header name for the notes column
|
||||
description: CSV imports only. Header name for the notes column
|
||||
account_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the account column
|
||||
when importing rows across multiple accounts
|
||||
qty_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the quantity
|
||||
column
|
||||
ticker_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the ticker
|
||||
column
|
||||
price_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the price column
|
||||
entity_type_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the entity type column
|
||||
currency_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the currency column
|
||||
exchange_operating_mic_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the exchange
|
||||
operating MIC column
|
||||
date_format:
|
||||
type: string
|
||||
description: Date format pattern (e.g., "%m/%d/%Y")
|
||||
description: CSV imports only. Date format pattern (e.g., "%m/%d/%Y")
|
||||
number_format:
|
||||
type: string
|
||||
enum:
|
||||
@@ -2356,20 +2412,135 @@ paths:
|
||||
- 1.234,56
|
||||
- 1 234,56
|
||||
- '1,234'
|
||||
description: Number format for parsing amounts
|
||||
description: CSV imports only. Number format for parsing amounts
|
||||
signage_convention:
|
||||
type: string
|
||||
enum:
|
||||
- inflows_positive
|
||||
- inflows_negative
|
||||
description: How to interpret positive/negative amounts
|
||||
description: CSV imports only. How to interpret positive/negative
|
||||
amounts
|
||||
col_sep:
|
||||
type: string
|
||||
enum:
|
||||
- ","
|
||||
- ";"
|
||||
description: Column separator
|
||||
required: true
|
||||
description: CSV imports only. Column separator
|
||||
amount_type_strategy:
|
||||
type: string
|
||||
enum:
|
||||
- signed_amount
|
||||
- custom_column
|
||||
description: CSV imports only. Amount parsing strategy
|
||||
amount_type_inflow_value:
|
||||
type: string
|
||||
description: CSV imports only. Column value that marks an amount
|
||||
as an inflow when using custom_column strategy
|
||||
multipart/form-data:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
raw_file_content:
|
||||
type: string
|
||||
description: Raw CSV or Sure NDJSON content as a string. Required
|
||||
for SureImport unless a multipart file is uploaded.
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- TransactionImport
|
||||
- TradeImport
|
||||
- AccountImport
|
||||
- MintImport
|
||||
- CategoryImport
|
||||
- RuleImport
|
||||
- SureImport
|
||||
description: Import type (defaults to TransactionImport)
|
||||
account_id:
|
||||
type: string
|
||||
format: uuid
|
||||
description: Account ID to import into
|
||||
publish:
|
||||
type: string
|
||||
description: Set to "true" to automatically queue for processing
|
||||
if configuration is valid
|
||||
date_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the date column
|
||||
amount_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the amount column
|
||||
name_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the transaction name
|
||||
column
|
||||
category_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the category column
|
||||
tags_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the tags column
|
||||
notes_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the notes column
|
||||
account_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the account column
|
||||
when importing rows across multiple accounts
|
||||
qty_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the quantity
|
||||
column
|
||||
ticker_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the ticker
|
||||
column
|
||||
price_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the price column
|
||||
entity_type_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the entity type column
|
||||
currency_col_label:
|
||||
type: string
|
||||
description: CSV imports only. Header name for the currency column
|
||||
exchange_operating_mic_col_label:
|
||||
type: string
|
||||
description: CSV trade imports only. Header name for the exchange
|
||||
operating MIC column
|
||||
date_format:
|
||||
type: string
|
||||
description: CSV imports only. Date format pattern (e.g., "%m/%d/%Y")
|
||||
number_format:
|
||||
type: string
|
||||
enum:
|
||||
- '1,234.56'
|
||||
- 1.234,56
|
||||
- 1 234,56
|
||||
- '1,234'
|
||||
description: CSV imports only. Number format for parsing amounts
|
||||
signage_convention:
|
||||
type: string
|
||||
enum:
|
||||
- inflows_positive
|
||||
- inflows_negative
|
||||
description: CSV imports only. How to interpret positive/negative
|
||||
amounts
|
||||
col_sep:
|
||||
type: string
|
||||
enum:
|
||||
- ","
|
||||
- ";"
|
||||
description: CSV imports only. Column separator
|
||||
amount_type_strategy:
|
||||
type: string
|
||||
enum:
|
||||
- signed_amount
|
||||
- custom_column
|
||||
description: CSV imports only. Amount parsing strategy
|
||||
amount_type_inflow_value:
|
||||
type: string
|
||||
description: CSV imports only. Column value that marks an amount
|
||||
as an inflow when using custom_column strategy
|
||||
"/api/v1/imports/{id}":
|
||||
parameters:
|
||||
- name: id
|
||||
|
||||
@@ -76,7 +76,7 @@ RSpec.describe 'API V1 Imports', type: :request do
|
||||
schema: { type: :string, enum: %w[pending complete importing reverting revert_failed failed] }
|
||||
parameter name: :type, in: :query, required: false,
|
||||
description: 'Filter by import type',
|
||||
schema: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport] }
|
||||
schema: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] }
|
||||
|
||||
response '200', 'imports listed' do
|
||||
schema '$ref' => '#/components/schemas/ImportCollection'
|
||||
@@ -102,22 +102,22 @@ RSpec.describe 'API V1 Imports', type: :request do
|
||||
end
|
||||
|
||||
post 'Create import' do
|
||||
description 'Create a new import from raw CSV content.'
|
||||
description 'Create a new import from raw CSV content, inline Sure NDJSON content, or an uploaded Sure NDJSON file.'
|
||||
tags 'Imports'
|
||||
security [ { apiKeyAuth: [] } ]
|
||||
consumes 'application/json'
|
||||
consumes 'application/json', 'multipart/form-data'
|
||||
produces 'application/json'
|
||||
|
||||
parameter name: :body, in: :body, required: true, schema: {
|
||||
parameter name: :body, in: :body, required: false, schema: {
|
||||
type: :object,
|
||||
properties: {
|
||||
raw_file_content: {
|
||||
type: :string,
|
||||
description: 'The raw CSV content as a string'
|
||||
description: 'Raw CSV or Sure NDJSON content as a string. Required for SureImport unless a multipart file is uploaded.'
|
||||
},
|
||||
type: {
|
||||
type: :string,
|
||||
enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport],
|
||||
enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport],
|
||||
description: 'Import type (defaults to TransactionImport)'
|
||||
},
|
||||
account_id: {
|
||||
@@ -131,46 +131,83 @@ RSpec.describe 'API V1 Imports', type: :request do
|
||||
},
|
||||
date_col_label: {
|
||||
type: :string,
|
||||
description: 'Header name for the date column'
|
||||
description: 'CSV imports only. Header name for the date column'
|
||||
},
|
||||
amount_col_label: {
|
||||
type: :string,
|
||||
description: 'Header name for the amount column'
|
||||
description: 'CSV imports only. Header name for the amount column'
|
||||
},
|
||||
name_col_label: {
|
||||
type: :string,
|
||||
description: 'Header name for the transaction name column'
|
||||
description: 'CSV imports only. Header name for the transaction name column'
|
||||
},
|
||||
category_col_label: {
|
||||
type: :string,
|
||||
description: 'Header name for the category column'
|
||||
description: 'CSV imports only. Header name for the category column'
|
||||
},
|
||||
tags_col_label: {
|
||||
type: :string,
|
||||
description: 'Header name for the tags column'
|
||||
description: 'CSV imports only. Header name for the tags column'
|
||||
},
|
||||
notes_col_label: {
|
||||
type: :string,
|
||||
description: 'Header name for the notes column'
|
||||
description: 'CSV imports only. Header name for the notes column'
|
||||
},
|
||||
account_col_label: {
|
||||
type: :string,
|
||||
description: 'CSV imports only. Header name for the account column when importing rows across multiple accounts'
|
||||
},
|
||||
qty_col_label: {
|
||||
type: :string,
|
||||
description: 'CSV trade imports only. Header name for the quantity column'
|
||||
},
|
||||
ticker_col_label: {
|
||||
type: :string,
|
||||
description: 'CSV trade imports only. Header name for the ticker column'
|
||||
},
|
||||
price_col_label: {
|
||||
type: :string,
|
||||
description: 'CSV trade imports only. Header name for the price column'
|
||||
},
|
||||
entity_type_col_label: {
|
||||
type: :string,
|
||||
description: 'CSV imports only. Header name for the entity type column'
|
||||
},
|
||||
currency_col_label: {
|
||||
type: :string,
|
||||
description: 'CSV imports only. Header name for the currency column'
|
||||
},
|
||||
exchange_operating_mic_col_label: {
|
||||
type: :string,
|
||||
description: 'CSV trade imports only. Header name for the exchange operating MIC column'
|
||||
},
|
||||
date_format: {
|
||||
type: :string,
|
||||
description: 'Date format pattern (e.g., "%m/%d/%Y")'
|
||||
description: 'CSV imports only. Date format pattern (e.g., "%m/%d/%Y")'
|
||||
},
|
||||
number_format: {
|
||||
type: :string,
|
||||
enum: [ '1,234.56', '1.234,56', '1 234,56', '1,234' ],
|
||||
description: 'Number format for parsing amounts'
|
||||
description: 'CSV imports only. Number format for parsing amounts'
|
||||
},
|
||||
signage_convention: {
|
||||
type: :string,
|
||||
enum: %w[inflows_positive inflows_negative],
|
||||
description: 'How to interpret positive/negative amounts'
|
||||
description: 'CSV imports only. How to interpret positive/negative amounts'
|
||||
},
|
||||
col_sep: {
|
||||
type: :string,
|
||||
enum: [ ',', ';' ],
|
||||
description: 'Column separator'
|
||||
description: 'CSV imports only. Column separator'
|
||||
},
|
||||
amount_type_strategy: {
|
||||
type: :string,
|
||||
enum: %w[signed_amount custom_column],
|
||||
description: 'CSV imports only. Amount parsing strategy'
|
||||
},
|
||||
amount_type_inflow_value: {
|
||||
type: :string,
|
||||
description: 'CSV imports only. Column value that marks an amount as an inflow when using custom_column strategy'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -193,7 +230,10 @@ RSpec.describe 'API V1 Imports', type: :request do
|
||||
end
|
||||
|
||||
response '422', 'validation error - file too large' do
|
||||
schema '$ref' => '#/components/schemas/ErrorResponse'
|
||||
schema oneOf: [
|
||||
{ '$ref' => '#/components/schemas/ErrorResponse' },
|
||||
{ '$ref' => '#/components/schemas/ErrorResponseWithImportId' }
|
||||
]
|
||||
|
||||
let(:body) do
|
||||
{
|
||||
@@ -204,6 +244,20 @@ RSpec.describe 'API V1 Imports', type: :request do
|
||||
|
||||
run_test!
|
||||
end
|
||||
|
||||
response '500', 'import uploaded but publish enqueue failed' do
|
||||
schema '$ref' => '#/components/schemas/ErrorResponseWithImportId'
|
||||
|
||||
let(:body) do
|
||||
{
|
||||
raw_file_content: { type: 'Account', data: { id: 'account_1', name: 'Checking' } }.to_json,
|
||||
type: 'SureImport',
|
||||
publish: 'true'
|
||||
}
|
||||
end
|
||||
|
||||
run_test!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -64,6 +64,19 @@ RSpec.configure do |config|
|
||||
}
|
||||
}
|
||||
},
|
||||
ErrorResponseWithImportId: {
|
||||
type: :object,
|
||||
required: %w[error import_id],
|
||||
properties: {
|
||||
error: { type: :string },
|
||||
message: { type: :string, nullable: true },
|
||||
import_id: {
|
||||
type: :string,
|
||||
format: :uuid,
|
||||
description: 'Import ID preserved for retry or inspection after upload succeeds but publish fails'
|
||||
}
|
||||
}
|
||||
},
|
||||
MfaRequiredResponse: {
|
||||
type: :object,
|
||||
required: %w[error mfa_required],
|
||||
@@ -524,7 +537,7 @@ RSpec.configure do |config|
|
||||
required: %w[id type status created_at updated_at],
|
||||
properties: {
|
||||
id: { type: :string, format: :uuid },
|
||||
type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport] },
|
||||
type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] },
|
||||
status: { type: :string, enum: %w[pending complete importing reverting revert_failed failed] },
|
||||
created_at: { type: :string, format: :'date-time' },
|
||||
updated_at: { type: :string, format: :'date-time' },
|
||||
@@ -538,7 +551,7 @@ RSpec.configure do |config|
|
||||
required: %w[id type status created_at updated_at],
|
||||
properties: {
|
||||
id: { type: :string, format: :uuid },
|
||||
type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport] },
|
||||
type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] },
|
||||
status: { type: :string, enum: %w[pending complete importing reverting revert_failed failed] },
|
||||
created_at: { type: :string, format: :'date-time' },
|
||||
updated_at: { type: :string, format: :'date-time' },
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "test_helper"
|
||||
|
||||
class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest
|
||||
@@ -13,7 +15,8 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest
|
||||
user: @user,
|
||||
name: "Test Read-Write Key",
|
||||
scopes: [ "read_write" ],
|
||||
display_key: "test_rw_#{SecureRandom.hex(8)}"
|
||||
display_key: "test_rw_#{SecureRandom.hex(8)}",
|
||||
source: "web"
|
||||
)
|
||||
|
||||
@read_only_api_key = ApiKey.create!(
|
||||
@@ -131,6 +134,253 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest
|
||||
assert_equal '[{"action_type":"set_transaction_category","value":"Groceries"}]', row.actions
|
||||
end
|
||||
|
||||
test "should create Sure import with raw NDJSON content" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
|
||||
assert_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: ndjson_content
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :created
|
||||
|
||||
json_response = JSON.parse(response.body)
|
||||
import = Import.find(json_response["data"]["id"])
|
||||
|
||||
assert_instance_of SureImport, import
|
||||
assert import.ndjson_file.attached?
|
||||
assert_equal 1, import.rows_count
|
||||
assert_equal "pending", import.status
|
||||
end
|
||||
|
||||
test "should require authentication for Sure import" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: ndjson_content
|
||||
}
|
||||
end
|
||||
|
||||
assert_response :unauthorized
|
||||
end
|
||||
|
||||
test "should reject Sure import with read-only API key" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: ndjson_content
|
||||
},
|
||||
headers: api_headers(@read_only_api_key)
|
||||
end
|
||||
|
||||
assert_response :forbidden
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "insufficient_scope", json_response["error"]
|
||||
end
|
||||
|
||||
test "should create Sure import with uploaded NDJSON file" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
valid_file = Rack::Test::UploadedFile.new(
|
||||
StringIO.new(ndjson_content),
|
||||
"application/x-ndjson",
|
||||
original_filename: "sure-backup.ndjson"
|
||||
)
|
||||
|
||||
assert_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
file: valid_file
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :created
|
||||
|
||||
import = Import.find(JSON.parse(response.body)["data"]["id"])
|
||||
assert_instance_of SureImport, import
|
||||
assert import.ndjson_file.attached?
|
||||
assert_equal 1, import.rows_count
|
||||
end
|
||||
|
||||
test "should reject Sure import with no file or raw content" do
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport"
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "missing_content", json_response["error"]
|
||||
end
|
||||
|
||||
test "should reject Sure import uploaded file exceeding max size" do
|
||||
test_limit = 1.kilobyte
|
||||
large_file = Rack::Test::UploadedFile.new(
|
||||
StringIO.new("x" * (test_limit + 1)),
|
||||
"application/x-ndjson",
|
||||
original_filename: "large.ndjson"
|
||||
)
|
||||
|
||||
original_value = SureImport::MAX_NDJSON_SIZE
|
||||
SureImport.send(:remove_const, :MAX_NDJSON_SIZE)
|
||||
SureImport.const_set(:MAX_NDJSON_SIZE, test_limit)
|
||||
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
file: large_file
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "file_too_large", json_response["error"]
|
||||
ensure
|
||||
SureImport.send(:remove_const, :MAX_NDJSON_SIZE)
|
||||
SureImport.const_set(:MAX_NDJSON_SIZE, original_value)
|
||||
end
|
||||
|
||||
test "should reject Sure import uploaded file with invalid type" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
invalid_file = Rack::Test::UploadedFile.new(
|
||||
StringIO.new(ndjson_content),
|
||||
"application/pdf",
|
||||
original_filename: "sure-backup.pdf"
|
||||
)
|
||||
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
file: invalid_file
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "invalid_file_type", json_response["error"]
|
||||
end
|
||||
|
||||
test "should clean up Sure import if row sync fails" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
SureImport.any_instance.stubs(:sync_ndjson_rows_count!).raises(StandardError, "sync failed")
|
||||
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: ndjson_content
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :internal_server_error
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "internal_server_error", json_response["error"]
|
||||
end
|
||||
|
||||
test "should clean up Sure import if row sync validation fails" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
invalid_import = SureImport.new
|
||||
invalid_import.errors.add(:base, "invalid rows")
|
||||
SureImport.any_instance.stubs(:sync_ndjson_rows_count!).raises(ActiveRecord::RecordInvalid.new(invalid_import))
|
||||
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: ndjson_content
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "validation_failed", json_response["error"]
|
||||
assert_includes json_response["errors"], "invalid rows"
|
||||
end
|
||||
|
||||
test "should preserve Sure import if publish queueing fails" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
ImportJob.stubs(:perform_later).raises(StandardError, "queue offline")
|
||||
|
||||
assert_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: ndjson_content,
|
||||
publish: "true"
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :internal_server_error
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "publish_failed", json_response["error"]
|
||||
|
||||
import = Import.find(json_response["import_id"])
|
||||
assert_instance_of SureImport, import
|
||||
assert import.ndjson_file.attached?
|
||||
assert_equal 1, import.rows_count
|
||||
assert_equal "pending", import.status
|
||||
end
|
||||
|
||||
test "should preserve Sure import if auto publish exceeds row count" do
|
||||
ndjson_content = { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json
|
||||
SureImport.any_instance.stubs(:publish_later).raises(Import::MaxRowCountExceededError)
|
||||
|
||||
assert_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: ndjson_content,
|
||||
publish: "true"
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "max_row_count_exceeded", json_response["error"]
|
||||
|
||||
import = Import.find(json_response["import_id"])
|
||||
assert_instance_of SureImport, import
|
||||
assert import.ndjson_file.attached?
|
||||
assert_equal 1, import.rows_count
|
||||
end
|
||||
|
||||
test "should reject invalid Sure import NDJSON content" do
|
||||
assert_no_difference("Import.count") do
|
||||
post api_v1_imports_url,
|
||||
params: {
|
||||
type: "SureImport",
|
||||
raw_file_content: "not ndjson"
|
||||
},
|
||||
headers: api_headers(@api_key)
|
||||
end
|
||||
|
||||
assert_response :unprocessable_entity
|
||||
json_response = JSON.parse(response.body)
|
||||
assert_equal "invalid_ndjson", json_response["error"]
|
||||
end
|
||||
|
||||
test "should create import and auto-publish when configured and requested" do
|
||||
csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction"
|
||||
|
||||
@@ -257,6 +507,6 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest
|
||||
private
|
||||
|
||||
def api_headers(api_key)
|
||||
{ "X-Api-Key" => api_key.display_key }
|
||||
{ "X-Api-Key" => api_key.plain_key }
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user