feat(api): expose import status details (#1599)

* feat(api): expose import status details

* fix(api): reuse import status validation counts

* fix(api): cache Sure import status reads

* fix(imports): invalidate cached Sure import blobs

* docs(api): split import status schemas

* fix(api): refine import status detail contract

app/models/import.rb

@@ -258,6 +258,10 @@ class Import < ApplicationRecord
     uploaded? && rows_count > 0
   end
 
+  def configured_for_status_detail?
+    configured?
+  end
+
   def cleaned?
     configured? && rows.all?(&:valid?)
   end
@@ -266,6 +270,23 @@ class Import < ApplicationRecord
     cleaned? && mappings.all?(&:valid?)
   end
 
+  def cleaned_from_validation_stats?(invalid_rows_count:)
+    configured? && invalid_rows_count.zero?
+  end
+
+  def publishable_from_validation_stats?(invalid_rows_count:)
+    cleaned_from_validation_stats?(invalid_rows_count: invalid_rows_count) && mappings.all?(&:valid?)
+  end
+
+  def mapping_status_counts
+    mappable_ids = mappings.pluck(:mappable_id)
+
+    {
+      mappings_count: mappable_ids.size,
+      unassigned_mappings_count: mappable_ids.count(&:nil?)
+    }
+  end
+
   def revertable?
     complete? || revert_failed?
   end

app/models/pdf_import.rb

@@ -154,6 +154,14 @@ class PdfImport < Import
     account.present? && statement_with_transactions? && cleaned? && mappings.all?(&:valid?)
   end
 
+  def cleaned_from_validation_stats?(invalid_rows_count:)
+    account.present? && statement_with_transactions? && super
+  end
+
+  def publishable_from_validation_stats?(invalid_rows_count:)
+    account.present? && statement_with_transactions? && super
+  end
+
   def column_keys
     %i[date amount name category notes]
   end

app/models/qif_import.rb

@@ -73,6 +73,10 @@ class QifImport < Import
     account.present? && super
   end
 
+  def publishable_from_validation_stats?(invalid_rows_count:)
+    account.present? && super
+  end
+
   # Returns true if import! will move the opening anchor back to cover transactions
   # that predate the current anchor date. Used to show a notice in the confirm step.
   def will_adjust_opening_anchor?

app/models/sure_import.rb

@@ -112,6 +112,14 @@ class SureImport < Import
     cleaned? && dry_run.values.sum.positive?
   end
 
+  def cleaned_from_validation_stats?(invalid_rows_count:)
+    configured? && invalid_rows_count.zero?
+  end
+
+  def publishable_from_validation_stats?(invalid_rows_count:)
+    cleaned_from_validation_stats?(invalid_rows_count: invalid_rows_count) && dry_run.values.sum.positive?
+  end
+
   def max_row_count
     100_000
   end
@@ -127,6 +135,11 @@ class SureImport < Import
   private
 
     def ndjson_blob_string
-      ndjson_file.download.force_encoding(Encoding::UTF_8)
+      blob_id = ndjson_file.blob&.id
+
+      return @ndjson_blob_string if defined?(@ndjson_blob_string) && @ndjson_blob_id == blob_id
+
+      @ndjson_blob_id = blob_id
+      @ndjson_blob_string = ndjson_file.download.force_encoding(Encoding::UTF_8)
     end
 end

app/views/api/v1/imports/_status_detail.json.jbuilder

@@ -0,0 +1,22 @@
+uploaded = local_assigns[:uploaded]
+uploaded = import.uploaded? if uploaded.nil?
+configured = local_assigns[:configured]
+configured = import.configured_for_status_detail? if configured.nil?
+
+json.uploaded uploaded
+json.configured configured
+json.terminal import.complete? || import.failed? || import.revert_failed?
+
+if include_validation_stats
+  valid_rows_count = local_assigns.fetch(:valid_rows_count)
+  invalid_rows_count = local_assigns.fetch(:invalid_rows_count)
+
+  cleaned = local_assigns[:cleaned]
+  publishable = local_assigns[:publishable]
+  cleaned = import.cleaned_from_validation_stats?(invalid_rows_count: invalid_rows_count) if cleaned.nil?
+  publishable = import.publishable_from_validation_stats?(invalid_rows_count: invalid_rows_count) if publishable.nil?
+
+  json.cleaned cleaned
+  json.publishable publishable
+  json.revertable import.revertable?
+end

app/views/api/v1/imports/index.json.jbuilder

@@ -8,6 +8,9 @@ json.data do
     json.account_id import.account_id
     json.rows_count import.rows_count
     json.error import.error if import.error.present?
+    json.status_detail do
+      json.partial! "status_detail", import: import, include_validation_stats: false
+    end
   end
 end
 

app/views/api/v1/imports/show.json.jbuilder

@@ -1,3 +1,10 @@
+rows = @import.rows.to_a
+valid_rows_count = rows.count(&:valid?)
+invalid_rows_count = rows.length - valid_rows_count
+cleaned = @import.cleaned_from_validation_stats?(invalid_rows_count: invalid_rows_count)
+publishable = @import.publishable_from_validation_stats?(invalid_rows_count: invalid_rows_count)
+mapping_counts = @import.mapping_status_counts
+
 json.data do
   json.id @import.id
   json.type @import.type
@@ -6,6 +13,15 @@ json.data do
   json.updated_at @import.updated_at
   json.account_id @import.account_id
   json.error @import.error if @import.error.present?
+  json.status_detail do
+    json.partial! "status_detail",
+                  import: @import,
+                  include_validation_stats: true,
+                  valid_rows_count: valid_rows_count,
+                  invalid_rows_count: invalid_rows_count,
+                  cleaned: cleaned,
+                  publishable: publishable
+  end
 
   json.configuration do
     json.date_col_label @import.date_col_label
@@ -22,7 +38,10 @@ json.data do
 
   json.stats do
     json.rows_count @import.rows_count
-    json.valid_rows_count @import.rows.select(&:valid?).count if @import.rows.loaded?
+    json.valid_rows_count valid_rows_count
+    json.invalid_rows_count invalid_rows_count
+    json.mappings_count mapping_counts[:mappings_count]
+    json.unassigned_mappings_count mapping_counts[:unassigned_mappings_count]
   end
 
   # Only show a subset of rows for preview if needed, or link to a separate rows endpoint