feat(merchants): add raw data import (csv) for merchants (#1992)

* feat(merchants): add csv import endpoint for merchants

* docs: update endpoint docs

* fix(merchant): recommended ai fixes
This commit is contained in:
Blaž Dular
2026-06-06 16:33:32 +02:00
committed by GitHub
parent d88d6e9e58
commit 94422955f8
18 changed files with 512 additions and 37 deletions

View File

@@ -2,29 +2,14 @@
module Api
module V1
# API v1 endpoint for merchants
# Provides read-only access to family and provider merchants
#
# @example List all merchants
# GET /api/v1/merchants
#
# @example Get a specific merchant
# GET /api/v1/merchants/:id
#
class MerchantsController < BaseController
before_action -> { authorize_scope!(:read) }
before_action -> { authorize_scope!(:read) }, only: [ :index, :show ]
before_action -> { authorize_scope!(:write) }, only: [ :create ]
# List all merchants available to the family
#
# Returns both family-owned merchants and provider merchants
# that are assigned to the family's transactions.
#
# @return [Array<Hash>] JSON array of merchant objects
def index
family = current_resource_owner.family
user = current_resource_owner
# Single query with OR conditions - more efficient than Ruby deduplication
family_merchant_ids = family.merchants.select(:id)
accessible_account_ids = family.accounts.accessible_by(user).select(:id)
provider_merchant_ids = Transaction.joins(:entry)
@@ -44,13 +29,6 @@ module Api
render json: { error: "Failed to fetch merchants" }, status: :internal_server_error
end
# Get a specific merchant by ID
#
# Returns a merchant if it belongs to the family or is assigned
# to any of the family's transactions.
#
# @param id [String] The merchant ID
# @return [Hash] JSON merchant object or error
def show
family = current_resource_owner.family
user = current_resource_owner
@@ -71,12 +49,83 @@ module Api
render json: { error: "Failed to fetch merchant" }, status: :internal_server_error
end
def create
family = current_resource_owner.family
unless params[:file].present?
return render json: { error: "missing_file", message: "Please provide a CSV file." },
status: :unprocessable_entity
end
file = params[:file]
if file.size > Import::MAX_CSV_SIZE
return render json: {
error: "file_too_large",
message: "File is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB."
}, status: :unprocessable_entity
end
unless Import::ALLOWED_CSV_MIME_TYPES.include?(file.content_type)
return render json: {
error: "invalid_file_type",
message: "Invalid file type. Please upload a CSV file."
}, status: :unprocessable_entity
end
csv = Import.parse_csv_str(file.read)
name_header = normalized_header(csv.headers, "name")
unless name_header
return render json: {
error: "missing_column",
message: "CSV must include a 'name' column."
}, status: :unprocessable_entity
end
color_header = normalized_header(csv.headers, "color")
website_url_header = normalized_header(csv.headers, "website_url", "website url", "website")
imported = []
skipped = []
csv.each do |row|
name = row[name_header].to_s.strip
next if name.blank?
merchant = family.merchants.find_or_initialize_by(name: name)
if merchant.persisted?
skipped << { name: name, reason: "already_exists" }
next
end
merchant.color = row[color_header].to_s.strip.presence if color_header
merchant.website_url = row[website_url_header].to_s.strip.presence if website_url_header
if merchant.save
imported << merchant
else
skipped << { name: name, errors: merchant.errors.full_messages }
end
end
render json: {
imported: imported.count,
skipped: skipped.count,
merchants: imported.map { |m| merchant_json(m) }
}, status: :created
rescue CSV::MalformedCSVError => e
render json: { error: "invalid_csv", message: "CSV could not be parsed: #{e.message}" },
status: :unprocessable_entity
rescue StandardError => e
Rails.logger.error("API Merchants Import Error: #{e.message}")
render json: { error: "internal_server_error", message: "An unexpected error occurred" },
status: :internal_server_error
end
private
# Serialize a merchant to JSON format
#
# @param merchant [Merchant] The merchant to serialize
# @return [Hash] JSON-serializable hash
def merchant_json(merchant)
{
id: merchant.id,
@@ -86,6 +135,19 @@ module Api
updated_at: merchant.updated_at
}
end
def normalized_header(headers, *candidates)
normalized_map = headers.to_h { |h| [ normalize(h), h ] }
candidates.each do |candidate|
header = normalized_map[normalize(candidate)]
return header if header.present?
end
nil
end
def normalize(str)
str.to_s.strip.downcase.gsub(/\*/, "").gsub(/[\s_-]+/, "_")
end
end
end
end

View File

@@ -111,7 +111,8 @@ class ImportsController < ApplicationController
if !@import.uploaded?
redirect_to import_upload_path(@import), alert: t("imports.show.finalize_upload")
elsif !@import.publishable?
redirect_to import_confirm_path(@import), alert: t("imports.show.finalize_mappings")
next_path = @import.mapping_steps.empty? ? import_clean_path(@import) : import_confirm_path(@import)
redirect_to next_path, alert: t("imports.show.finalize_mappings")
end
end

View File

@@ -25,7 +25,9 @@ module ImportsHelper
entity_type: I18n.t("imports.column_labels.entity_type"),
category_parent: I18n.t("imports.column_labels.category_parent"),
category_color: I18n.t("imports.column_labels.category_color"),
category_icon: I18n.t("imports.column_labels.category_icon")
category_icon: I18n.t("imports.column_labels.category_icon"),
merchant_color: I18n.t("imports.column_labels.merchant_color"),
merchant_website: I18n.t("imports.column_labels.merchant_website")
}[key]
end
@@ -80,7 +82,7 @@ module ImportsHelper
private
def permitted_import_types
%w[transaction_import trade_import account_import mint_import actual_import category_import rule_import]
%w[transaction_import trade_import account_import mint_import actual_import category_import rule_import merchant_import]
end
DryRunResource = Struct.new(:label, :icon, :text_class, :bg_class, keyword_init: true)

View File

@@ -10,7 +10,7 @@ class Import < ApplicationRecord
DOCUMENT_TYPES = %w[bank_statement credit_card_statement investment_statement financial_document contract other].freeze
TYPES = %w[TransactionImport TradeImport AccountImport MintImport ActualImport CategoryImport RuleImport PdfImport QifImport SureImport].freeze
TYPES = %w[TransactionImport TradeImport AccountImport MintImport ActualImport CategoryImport RuleImport MerchantImport PdfImport QifImport SureImport].freeze
SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative]
SEPARATORS = [ [ "Comma (,)", "," ], [ "Semicolon (;)", ";" ] ].freeze

View File

@@ -0,0 +1,110 @@
class MerchantImport < Import
def import!
transaction do
rows.each do |row|
merchant_name = row.name.to_s.strip
next if merchant_name.blank?
merchant = family.merchants.find_or_initialize_by(name: merchant_name)
next unless merchant.new_record?
merchant.color = row.merchant_color.presence || FamilyMerchant::COLORS.sample
merchant.website_url = row.merchant_website.presence
merchant.save!
end
end
end
def column_keys
%i[name merchant_color merchant_website]
end
def required_column_keys
%i[name]
end
def mapping_steps
[]
end
def dry_run
{ merchants: rows_count }
end
def csv_template
template = <<-CSV
name*,color,website_url
Coffee Shop,#e99537,https://coffeeshop.com
Pizza Palace,#4da568,https://pizzapalace.com
Bookstore,,
CSV
CSV.parse(template, headers: true)
end
def generate_rows_from_csv
rows.destroy_all
validate_required_headers!
name_header = header_for("name")
color_header = header_for("color")
website_header = header_for("website_url", "website url", "website")
csv_rows.each.with_index(1) do |row, index|
rows.create!(
source_row_number: index,
name: row[name_header].to_s.strip,
merchant_color: row[color_header].to_s.strip,
merchant_website: row[website_header].to_s.strip,
currency: default_currency
)
end
end
private
def validate_required_headers!
missing_headers = required_column_keys.map(&:to_s).reject { |key| header_for(key).present? }
return if missing_headers.empty?
errors.add(:base, :missing_columns, columns: missing_headers.join(", "))
raise ActiveRecord::RecordInvalid.new(self)
end
def header_for(*candidates)
candidates.each do |candidate|
normalized = normalize_header(candidate)
header = normalized_headers[normalized]
return header if header.present?
end
nil
end
def normalized_headers
@normalized_headers ||= begin
result = {}
duplicates = []
csv_headers.each do |header|
key = normalize_header(header)
if result.key?(key)
duplicates << header
else
result[key] = header
end
end
if duplicates.any?
errors.add(:base, :duplicate_columns, columns: duplicates.join(", "))
raise ActiveRecord::RecordInvalid.new(self)
end
result
end
end
def normalize_header(header)
header.to_s.strip.downcase.gsub(/\*/, "").gsub(/[\s-]+/, "_")
end
end

View File

@@ -8,6 +8,12 @@
frame: :modal,
icon: "combine") %>
<% end %>
<%= render DS::Link.new(
text: t(".import"),
variant: "outline",
icon: "upload",
href: new_import_path(type: "MerchantImport")
) %>
<%= render DS::Link.new(
text: t(".new"),
variant: "primary",

View File

@@ -0,0 +1,14 @@
<%# locals: (import:) %>
<div class="space-y-4">
<p class="text-sm text-secondary"><%= t("import.configurations.merchant_import.description") %></p>
<%= styled_form_with model: import,
url: import_configuration_path(import),
scope: :import,
method: :patch,
class: "space-y-3" do |form| %>
<p class="text-sm text-secondary"><%= t("import.configurations.merchant_import.instructions") %></p>
<%= form.submit t("import.configurations.merchant_import.button_label"), disabled: import.complete? %>
<% end %>
</div>

View File

@@ -225,6 +225,16 @@
enabled: true %>
<% end %>
<% if params[:type].nil? || params[:type] == "MerchantImport" %>
<%= render "imports/import_option",
type: "MerchantImport",
icon_name: "store",
icon_bg_class: "bg-orange-500/5",
icon_text_class: "text-orange-500",
label: t(".import_merchants"),
enabled: true %>
<% end %>
<% if params[:type].nil? || params[:type] == "RuleImport" %>
<%= render "imports/import_option",
type: "RuleImport",

View File

@@ -0,0 +1,8 @@
---
en:
activerecord:
errors:
models:
merchant_import:
missing_columns: "Missing required columns: %{columns}"
duplicate_columns: "Duplicate column names after normalization: %{columns}"

View File

@@ -110,6 +110,10 @@ en:
description: Upload a simple CSV file (like the one we generate when you
export your data). We'll automatically map the columns for you.
instructions: Select continue to parse your CSV and move on to the clean step.
merchant_import:
button_label: Continue
description: Upload a CSV file with your merchants. We'll automatically map the columns for you.
instructions: Select continue to parse your CSV and move on to the clean step.
mint_import:
date_format_label: Date format
actual_import:
@@ -250,6 +254,8 @@ en:
category_parent: "Parent category"
category_color: "Color"
category_icon: "Lucide icon"
merchant_color: "Color"
merchant_website: "Website URL"
update:
account_saved: "Account saved."
invalid_account: "Account not found."
@@ -305,6 +311,7 @@ en:
qif_import: "QIF import"
category_import: "Category import"
rule_import: "Rule import"
merchant_import: "Merchant import"
pdf_import: "PDF import"
document_import: "Document import"
sure_import: "Sure import"
@@ -338,6 +345,7 @@ en:
qif_import: "QIF"
category_import: "Category"
rule_import: "Rule"
merchant_import: "Merchant"
pdf_import: "PDF"
document_import: "Document"
sure_import: "Sure"
@@ -362,6 +370,7 @@ en:
import_ynab: Import from YNAB
import_accounts: Import accounts
import_categories: Import categories
import_merchants: Import merchants
import_mint: Import from Mint
import_actual: Import from Actual Budget
import_portfolio: Import investments

View File

@@ -16,6 +16,7 @@ en:
index:
empty: No merchants yet
new: New merchant
import: Import merchants
merge: Merge merchants
title: Merchants
family_title: "%{moniker} merchants"

View File

@@ -515,7 +515,7 @@ Rails.application.routes.draw do
resources :budgets, only: [ :index, :show ]
resources :budget_categories, only: [ :index, :show ]
resources :categories, only: [ :index, :show, :create ]
resources :merchants, only: [ :index, :show ]
resources :merchants, only: [ :index, :show, :create ]
resources :rules, only: [ :index, :show ]
resources :rule_runs, only: [ :index, :show ]
resources :securities, only: [ :index, :show ]

View File

@@ -0,0 +1,6 @@
class AddMerchantColumnsToImportRows < ActiveRecord::Migration[7.2]
def change
add_column :import_rows, :merchant_color, :string
add_column :import_rows, :merchant_website, :string
end
end

2
db/schema.rb generated
View File

@@ -978,6 +978,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_31_213000) do
t.text "conditions"
t.text "actions"
t.integer "source_row_number", null: false
t.string "merchant_color"
t.string "merchant_website"
t.index ["import_id", "source_row_number"], name: "index_import_rows_on_import_id_and_source_row_number", unique: true
t.index ["import_id"], name: "index_import_rows_on_import_id"
t.check_constraint "source_row_number > 0", name: "chk_import_rows_source_row_number_positive"

View File

@@ -1,6 +1,6 @@
# Merchants API
The Merchants API allows external applications to retrieve merchants within Sure. Merchants represent payees or vendors associated with transactions.
The Merchants API allows external applications to retrieve and bulk-import merchants within Sure. Merchants represent payees or vendors associated with transactions.
## Generated OpenAPI specification
@@ -21,7 +21,10 @@ The Merchants API allows external applications to retrieve merchants within Sure
## Authentication requirements
All merchant endpoints require an OAuth2 access token or API key that grants the `read` scope.
| Endpoint | Required scope |
| --- | --- |
| `GET` endpoints | `read` |
| `POST /api/v1/merchants` (CSV import) | `write` |
## Available endpoints
@@ -29,6 +32,7 @@ All merchant endpoints require an OAuth2 access token or API key that grants the
| --- | --- | --- |
| `GET /api/v1/merchants` | `read` | List all merchants available to the family. |
| `GET /api/v1/merchants/{id}` | `read` | Retrieve a single merchant by ID. |
| `POST /api/v1/merchants` | `write` | Bulk-import merchants from a CSV file. |
Refer to the generated [`openapi.yaml`](openapi.yaml) for request/response schemas, reusable components, and security definitions.
@@ -104,6 +108,84 @@ When creating or updating transactions, you can assign a merchant using the `mer
}
```
## Importing merchants via CSV
`POST /api/v1/merchants` accepts a `multipart/form-data` upload and bulk-creates `FamilyMerchant` records. Existing merchants with the same name are skipped (no update, no error).
### Request
```http
POST /api/v1/merchants
Content-Type: multipart/form-data
X-Api-Key: <write-scoped-key>
file=@merchants.csv
```
### CSV format
| Column | Required | Description |
| --- | --- | --- |
| `name` | Yes | Merchant name. Rows with a blank name are skipped. |
| `color` | No | Hex colour code (e.g. `#e99537`). Defaults to a random palette colour. |
| `website_url` | No | Merchant website. Aliases accepted: `website url`, `website`. |
The header row is required. Column names are matched case-insensitively and extra spaces, underscores, and asterisks are ignored (e.g. `Name*`, `Website URL`, and `website_url` all match).
Example CSV:
```csv
name,color,website_url
Coffee Shop,#e99537,https://coffeeshop.com
Pizza Palace,#4da568,https://pizzapalace.com
Bookstore,,
```
### Response — 201 Created
```json
{
"imported": 2,
"skipped": 1,
"merchants": [
{
"id": "550e8400-e29b-41d4-a716-446655440001",
"name": "Coffee Shop",
"type": "FamilyMerchant",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-15T10:30:00Z"
},
{
"id": "550e8400-e29b-41d4-a716-446655440002",
"name": "Pizza Palace",
"type": "FamilyMerchant",
"created_at": "2024-01-15T10:30:00Z",
"updated_at": "2024-01-15T10:30:00Z"
}
]
}
```
`skipped` counts rows where a merchant with that name already exists for the family.
### Error responses for CSV import
| HTTP status | `error` value | Cause |
| --- | --- | --- |
| `401` | `unauthorized` | Missing or invalid API key. |
| `403` | `forbidden` | API key lacks the `write` scope. |
| `422` | `missing_file` | No `file` parameter supplied. |
| `422` | `file_too_large` | File exceeds 10 MB. |
| `422` | `invalid_file_type` | File is not a recognised CSV MIME type. |
| `422` | `missing_column` | CSV has no `name` column. |
| `422` | `invalid_csv` | CSV is malformed or cannot be parsed. |
## Importing merchants via the web UI
Merchants can also be imported through the built-in multi-step import flow at **Settings → Imports → New Import → Raw Data → Import merchants**. The flow supports the same CSV format as the API endpoint (upload → configure → clean → publish).
A shortcut button ("Import merchants") is also available directly on the **Merchants** page.
## Error responses
Errors conform to the shared `ErrorResponse` schema in the OpenAPI document:

View File

@@ -47,6 +47,43 @@ RSpec.describe 'API V1 Merchants', type: :request do
run_test!
end
end
post 'Import merchants from CSV' do
tags 'Merchants'
security [ { apiKeyAuth: [] } ]
consumes 'multipart/form-data'
produces 'application/json'
parameter name: :file, in: :formData, type: :file, required: true,
description: 'CSV file with columns: name* (required), color, website_url'
response '201', 'merchants imported' do
schema '$ref' => '#/components/schemas/MerchantImportResult'
let(:file) do
Rack::Test::UploadedFile.new(
StringIO.new("name,color,website_url\nCoffee Shop,#e99537,https://coffeeshop.com"),
'text/csv',
true,
original_filename: 'merchants.csv'
)
end
run_test!
end
response '401', 'unauthorized' do
schema '$ref' => '#/components/schemas/ErrorResponse'
let(:'X-Api-Key') { nil }
run_test!
end
response '422', 'missing file or invalid CSV' do
schema '$ref' => '#/components/schemas/ErrorResponse'
let(:file) { nil }
run_test!
end
end
end
path '/api/v1/merchants/{id}' do

View File

@@ -571,6 +571,15 @@ RSpec.configure do |config|
updated_at: { type: :string, format: :'date-time' }
}
},
MerchantImportResult: {
type: :object,
required: %w[imported skipped merchants],
properties: {
imported: { type: :integer, description: 'Number of merchants successfully created' },
skipped: { type: :integer, description: 'Number of rows skipped (duplicates or invalid)' },
merchants: { type: :array, items: { '$ref' => '#/components/schemas/MerchantDetail' } }
}
},
Tag: {
type: :object,
required: %w[id name color],

View File

@@ -7,10 +7,11 @@ class Api::V1::MerchantsControllerTest < ActionDispatch::IntegrationTest
@user = users(:family_admin)
@other_family_user = users(:empty)
# Verify cross-family isolation setup is correct
assert_not_equal @user.family_id, @other_family_user.family_id,
"Test setup error: @other_family_user must belong to a different family"
@user.api_keys.active.destroy_all
@oauth_app = Doorkeeper::Application.create!(
name: "Test App",
redirect_uri: "https://example.com/callback",
@@ -52,7 +53,6 @@ class Api::V1::MerchantsControllerTest < ActionDispatch::IntegrationTest
end
test "index does not return merchants from other families" do
# Create a merchant in another family
other_merchant = @other_family_user.family.merchants.create!(name: "Other Merchant")
get api_v1_merchants_url, headers: auth_headers
@@ -96,9 +96,125 @@ class Api::V1::MerchantsControllerTest < ActionDispatch::IntegrationTest
assert_response :not_found
end
# Create (CSV import) action tests
test "create requires authentication" do
post api_v1_merchants_url, params: { file: csv_file("name\nNew Merchant") }
assert_response :unauthorized
end
test "create rejects read-only api key" do
post api_v1_merchants_url,
params: { file: csv_file("name\nNew Merchant") },
headers: api_headers(read_only_api_key)
assert_response :forbidden
end
test "create imports merchants from csv" do
csv_content = "name,color,website_url\nImported Merchant,#ff0000,https://example.com\nAnother Merchant,,"
assert_difference "@user.family.merchants.count", 2 do
post api_v1_merchants_url,
params: { file: csv_file(csv_content) },
headers: api_headers(read_write_api_key)
end
assert_response :created
body = JSON.parse(response.body)
assert_equal 2, body["imported"]
assert_equal 0, body["skipped"]
assert_equal 2, body["merchants"].length
imported = body["merchants"].find { |m| m["name"] == "Imported Merchant" }
assert imported.present?
assert imported["id"].present?
assert_equal "FamilyMerchant", imported["type"]
end
test "create skips duplicate merchant names" do
csv_content = "name\n#{@merchant.name}\nBrand New Merchant"
assert_difference "@user.family.merchants.count", 1 do
post api_v1_merchants_url,
params: { file: csv_file(csv_content) },
headers: api_headers(read_write_api_key)
end
assert_response :created
body = JSON.parse(response.body)
assert_equal 1, body["imported"]
assert_equal 1, body["skipped"]
end
test "create returns 422 when file is missing" do
post api_v1_merchants_url, headers: api_headers(read_write_api_key)
assert_response :unprocessable_entity
body = JSON.parse(response.body)
assert_equal "missing_file", body["error"]
end
test "create returns 422 when csv is missing name column" do
csv_content = "color,website_url\n#ff0000,https://example.com"
post api_v1_merchants_url,
params: { file: csv_file(csv_content) },
headers: api_headers(read_write_api_key)
assert_response :unprocessable_entity
body = JSON.parse(response.body)
assert_equal "missing_column", body["error"]
end
test "create returns 422 for invalid file type" do
file = Rack::Test::UploadedFile.new(
StringIO.new("not a csv"),
"application/pdf",
true,
original_filename: "merchants.pdf"
)
post api_v1_merchants_url,
params: { file: file },
headers: api_headers(read_write_api_key)
assert_response :unprocessable_entity
body = JSON.parse(response.body)
assert_equal "invalid_file_type", body["error"]
end
private
def auth_headers
{ "Authorization" => "Bearer #{@access_token.token}" }
end
def read_write_api_key
@read_write_api_key ||= ApiKey.create!(
user: @user,
name: "Test RW Key",
key: ApiKey.generate_secure_key,
scopes: %w[read_write],
source: "web"
)
end
def read_only_api_key
@read_only_api_key ||= ApiKey.create!(
user: @user,
name: "Test RO Key",
key: ApiKey.generate_secure_key,
scopes: %w[read],
source: "mobile"
)
end
def api_headers(api_key)
{ "X-Api-Key" => api_key.plain_key }
end
def csv_file(content, filename: "merchants.csv")
uploaded_file(filename: filename, content_type: "text/csv", content: content)
end
end