diff --git a/app/controllers/api/v1/imports_controller.rb b/app/controllers/api/v1/imports_controller.rb new file mode 100644 index 000000000..2b6a5a5af --- /dev/null +++ b/app/controllers/api/v1/imports_controller.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +class Api::V1::ImportsController < Api::V1::BaseController + include Pagy::Backend + + # Ensure proper scope authorization + before_action :ensure_read_scope, only: [ :index, :show ] + before_action :ensure_write_scope, only: [ :create ] + before_action :set_import, only: [ :show ] + + def index + family = current_resource_owner.family + imports_query = family.imports.ordered + + # Apply filters + if params[:status].present? + imports_query = imports_query.where(status: params[:status]) + end + + if params[:type].present? + imports_query = imports_query.where(type: params[:type]) + end + + # Pagination + @pagy, @imports = pagy( + imports_query, + page: safe_page_param, + limit: safe_per_page_param + ) + + @per_page = safe_per_page_param + + render :index + + rescue StandardError => e + Rails.logger.error "ImportsController#index error: #{e.message}" + render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error + end + + def show + render :show + rescue StandardError => e + Rails.logger.error "ImportsController#show error: #{e.message}" + render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error + end + + def create + family = current_resource_owner.family + + # 1. Determine type and validate + type = params[:type].to_s + type = "TransactionImport" unless Import::TYPES.include?(type) + + # 2. Build the import object with permitted config attributes + @import = family.imports.build(import_config_params) + @import.type = type + @import.account_id = params[:account_id] if params[:account_id].present? + + # 3. Attach the uploaded file if present (with validation) + if params[:file].present? + file = params[:file] + + if file.size > Import::MAX_CSV_SIZE + return render json: { + error: "file_too_large", + message: "File is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + }, status: :unprocessable_entity + end + + unless Import::ALLOWED_MIME_TYPES.include?(file.content_type) + return render json: { + error: "invalid_file_type", + message: "Invalid file type. Please upload a CSV file." + }, status: :unprocessable_entity + end + + @import.raw_file_str = file.read + elsif params[:raw_file_content].present? + if params[:raw_file_content].bytesize > Import::MAX_CSV_SIZE + return render json: { + error: "content_too_large", + message: "Content is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + }, status: :unprocessable_entity + end + + @import.raw_file_str = params[:raw_file_content] + end + + # 4. Save and Process + if @import.save + # Generate rows if file content was provided + if @import.uploaded? + begin + @import.generate_rows_from_csv + @import.reload + rescue StandardError => e + Rails.logger.error "Row generation failed for import #{@import.id}: #{e.message}" + end + end + + # If the import is configured (has rows), we can try to auto-publish or just leave it as pending + # For API simplicity, if enough info is provided, we might want to trigger processing + + if @import.configured? && params[:publish] == "true" + @import.publish_later + end + + render :show, status: :created + else + render json: { + error: "validation_failed", + message: "Import could not be created", + errors: @import.errors.full_messages + }, status: :unprocessable_entity + end + + rescue StandardError => e + Rails.logger.error "ImportsController#create error: #{e.message}" + render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error + end + + private + + def set_import + @import = current_resource_owner.family.imports.includes(:rows).find(params[:id]) + rescue ActiveRecord::RecordNotFound + render json: { error: "not_found", message: "Import not found" }, status: :not_found + end + + def ensure_read_scope + authorize_scope!(:read) + end + + def ensure_write_scope + authorize_scope!(:write) + end + + def import_config_params + params.permit( + :date_col_label, + :amount_col_label, + :name_col_label, + :category_col_label, + :tags_col_label, + :notes_col_label, + :account_col_label, + :qty_col_label, + :ticker_col_label, + :price_col_label, + :entity_type_col_label, + :currency_col_label, + :exchange_operating_mic_col_label, + :date_format, + :number_format, + :signage_convention, + :col_sep, + :amount_type_strategy, + :amount_type_inflow_value + ) + end + + def safe_page_param + page = params[:page].to_i + page > 0 ? page : 1 + end + + def safe_per_page_param + per_page = params[:per_page].to_i + (1..100).include?(per_page) ? per_page : 25 + end +end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 7b6040743..50891d323 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -26,14 +26,38 @@ class ImportsController < ApplicationController end def create + type = params.dig(:import, :type).to_s + type = "TransactionImport" unless Import::TYPES.include?(type) + account = Current.family.accounts.find_by(id: params.dig(:import, :account_id)) import = Current.family.imports.create!( - type: import_params[:type], + type: type, account: account, date_format: Current.family.date_format, ) - redirect_to import_upload_path(import) + if import_params[:csv_file].present? + file = import_params[:csv_file] + + if file.size > Import::MAX_CSV_SIZE + import.destroy + redirect_to new_import_path, alert: "File is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + return + end + + unless Import::ALLOWED_MIME_TYPES.include?(file.content_type) + import.destroy + redirect_to new_import_path, alert: "Invalid file type. Please upload a CSV file." + return + end + + # Stream reading is not fully applicable here as we store the raw string in the DB, + # but we have validated size beforehand to prevent memory exhaustion from massive files. + import.update!(raw_file_str: file.read) + redirect_to import_configuration_path(import), notice: "CSV uploaded successfully." + else + redirect_to import_upload_path(import) + end end def show @@ -70,6 +94,6 @@ class ImportsController < ApplicationController end def import_params - params.require(:import).permit(:type) + params.require(:import).permit(:csv_file) end end diff --git a/app/javascript/controllers/drag_and_drop_import_controller.js b/app/javascript/controllers/drag_and_drop_import_controller.js new file mode 100644 index 000000000..8b8332ac2 --- /dev/null +++ b/app/javascript/controllers/drag_and_drop_import_controller.js @@ -0,0 +1,65 @@ +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["input", "form", "overlay"] + + dragDepth = 0 + + connect() { + this.boundDragOver = this.dragOver.bind(this) + this.boundDragEnter = this.dragEnter.bind(this) + this.boundDragLeave = this.dragLeave.bind(this) + this.boundDrop = this.drop.bind(this) + + // Listen on the document to catch drags anywhere + document.addEventListener("dragover", this.boundDragOver) + document.addEventListener("dragenter", this.boundDragEnter) + document.addEventListener("dragleave", this.boundDragLeave) + document.addEventListener("drop", this.boundDrop) + } + + disconnect() { + document.removeEventListener("dragover", this.boundDragOver) + document.removeEventListener("dragenter", this.boundDragEnter) + document.removeEventListener("dragleave", this.boundDragLeave) + document.removeEventListener("drop", this.boundDrop) + } + + dragEnter(event) { + event.preventDefault() + this.dragDepth++ + if (this.dragDepth === 1) { + this.overlayTarget.classList.remove("hidden") + } + } + + dragOver(event) { + event.preventDefault() + } + + dragLeave(event) { + event.preventDefault() + this.dragDepth-- + if (this.dragDepth <= 0) { + this.dragDepth = 0 + this.overlayTarget.classList.add("hidden") + } + } + + drop(event) { + event.preventDefault() + this.dragDepth = 0 + this.overlayTarget.classList.add("hidden") + + if (event.dataTransfer.files.length > 0) { + const file = event.dataTransfer.files[0] + // Simple validation + if (file.type === "text/csv" || file.name.toLowerCase().endsWith(".csv")) { + this.inputTarget.files = event.dataTransfer.files + this.formTarget.requestSubmit() + } else { + alert("Please upload a valid CSV file.") + } + } + } +} diff --git a/app/models/account_import.rb b/app/models/account_import.rb index 0ffdcec4c..f5b790a09 100644 --- a/app/models/account_import.rb +++ b/app/models/account_import.rb @@ -42,7 +42,7 @@ class AccountImport < Import def dry_run { - accounts: rows.count + accounts: rows_count } end diff --git a/app/models/category_import.rb b/app/models/category_import.rb index a862bb77d..59f8095f4 100644 --- a/app/models/category_import.rb +++ b/app/models/category_import.rb @@ -42,7 +42,7 @@ class CategoryImport < Import end def dry_run - { categories: rows.count } + { categories: rows_count } end def csv_template diff --git a/app/models/import.rb b/app/models/import.rb index 536209371..05b426573 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -2,6 +2,9 @@ class Import < ApplicationRecord MaxRowCountExceededError = Class.new(StandardError) MappingError = Class.new(StandardError) + MAX_CSV_SIZE = 10.megabytes + ALLOWED_MIME_TYPES = %w[text/csv text/plain application/vnd.ms-excel application/csv].freeze + TYPES = %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport].freeze SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative] SEPARATORS = [ [ "Comma (,)", "," ], [ "Semicolon (;)", ";" ] ].freeze @@ -36,6 +39,7 @@ class Import < ApplicationRecord validates :col_sep, inclusion: { in: SEPARATORS.map(&:last) } validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }, allow_nil: true validates :number_format, presence: true, inclusion: { in: NUMBER_FORMATS.keys } + validate :account_belongs_to_family has_many :rows, dependent: :destroy has_many :mappings, dependent: :destroy @@ -110,7 +114,7 @@ class Import < ApplicationRecord def dry_run mappings = { - transactions: rows.count, + transactions: rows_count, categories: Import::CategoryMapping.for_import(self).creational.count, tags: Import::TagMapping.for_import(self).creational.count } @@ -152,6 +156,7 @@ class Import < ApplicationRecord end rows.insert_all!(mapped_rows) + update_column(:rows_count, rows.count) end def sync_mappings @@ -181,7 +186,7 @@ class Import < ApplicationRecord end def configured? - uploaded? && rows.any? + uploaded? && rows_count > 0 end def cleaned? @@ -232,7 +237,7 @@ class Import < ApplicationRecord private def row_count_exceeded? - rows.count > max_row_count + rows_count > max_row_count end def import! @@ -288,4 +293,11 @@ class Import < ApplicationRecord def set_default_number_format self.number_format ||= "1,234.56" # Default to US/UK format end + + def account_belongs_to_family + return if account.nil? + return if account.family_id == family_id + + errors.add(:account, "must belong to your family") + end end diff --git a/app/models/import/row.rb b/app/models/import/row.rb index ef16d26e5..26525b6f4 100644 --- a/app/models/import/row.rb +++ b/app/models/import/row.rb @@ -1,5 +1,5 @@ class Import::Row < ApplicationRecord - belongs_to :import + belongs_to :import, counter_cache: true validates :amount, numericality: true, allow_blank: true validates :currency, presence: true diff --git a/app/models/mint_import.rb b/app/models/mint_import.rb index d62da471d..932353d8a 100644 --- a/app/models/mint_import.rb +++ b/app/models/mint_import.rb @@ -18,6 +18,7 @@ class MintImport < Import end rows.insert_all!(mapped_rows) + update_column(:rows_count, rows.count) end def import! diff --git a/app/models/rule_import.rb b/app/models/rule_import.rb index e214481aa..d2a2d07ca 100644 --- a/app/models/rule_import.rb +++ b/app/models/rule_import.rb @@ -20,7 +20,7 @@ class RuleImport < Import end def dry_run - { rules: rows.count } + { rules: rows_count } end def csv_template diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index 99e7eb205..40387dfbf 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -54,7 +54,7 @@ class TradeImport < Import end def dry_run - mappings = { transactions: rows.count } + mappings = { transactions: rows_count } mappings.merge( accounts: Import::AccountMapping.for_import(self).creational.count diff --git a/app/views/api/v1/imports/index.json.jbuilder b/app/views/api/v1/imports/index.json.jbuilder new file mode 100644 index 000000000..eff1c6414 --- /dev/null +++ b/app/views/api/v1/imports/index.json.jbuilder @@ -0,0 +1,21 @@ +json.data do + json.array! @imports do |import| + json.id import.id + json.type import.type + json.status import.status + json.created_at import.created_at + json.updated_at import.updated_at + json.account_id import.account_id + json.rows_count import.rows_count + json.error import.error if import.error.present? + end +end + +json.meta do + json.current_page @pagy.page + json.next_page @pagy.next + json.prev_page @pagy.prev + json.total_pages @pagy.pages + json.total_count @pagy.count + json.per_page @per_page +end diff --git a/app/views/api/v1/imports/show.json.jbuilder b/app/views/api/v1/imports/show.json.jbuilder new file mode 100644 index 000000000..18509062e --- /dev/null +++ b/app/views/api/v1/imports/show.json.jbuilder @@ -0,0 +1,30 @@ +json.data do + json.id @import.id + json.type @import.type + json.status @import.status + json.created_at @import.created_at + json.updated_at @import.updated_at + json.account_id @import.account_id + json.error @import.error if @import.error.present? + + json.configuration do + json.date_col_label @import.date_col_label + json.amount_col_label @import.amount_col_label + json.name_col_label @import.name_col_label + json.category_col_label @import.category_col_label + json.tags_col_label @import.tags_col_label + json.notes_col_label @import.notes_col_label + json.account_col_label @import.account_col_label + json.date_format @import.date_format + json.number_format @import.number_format + json.signage_convention @import.signage_convention + end + + json.stats do + json.rows_count @import.rows_count + json.valid_rows_count @import.rows.select(&:valid?).count if @import.rows.loaded? + end + + # Only show a subset of rows for preview if needed, or link to a separate rows endpoint + # json.sample_rows @import.rows.limit(5) +end diff --git a/app/views/import/uploads/show.html.erb b/app/views/import/uploads/show.html.erb index 1115382ec..7227ff352 100644 --- a/app/views/import/uploads/show.html.erb +++ b/app/views/import/uploads/show.html.erb @@ -4,7 +4,10 @@ <%= content_for :previous_path, imports_path %> -