Fix TwelveData retry flow through Sync lifecycle

Co-authored-by: jjmata <187772+jjmata@users.noreply.github.com>
Agent-Logs-Url: https://github.com/we-promise/sure/sessions/ea7a06f4-8344-4a00-877a-eda8cdfdc16a

app/jobs/sync_job.rb

@@ -6,7 +6,10 @@ class SyncJob < ApplicationJob
   # to ensure the minute window has passed, then increase exponentially
   retry_on Provider::TwelveData::RateLimitError,
            wait: ->(executions) { [ 70 * (2 ** (executions - 1)), 600 ].min },
-           attempts: 5
+           attempts: 5 do |job, error|
+             sync = job.arguments.first
+             sync.fail_for_retry_exhaustion!(error.message)
+           end
 
   # Accept a runtime-only flag to influence sync behavior without persisting config
   def perform(sync, balances_only: false)

app/models/sync.rb

@@ -89,6 +89,9 @@ class Sync < ApplicationRecord
 
       begin
         syncable.perform_sync(self)
+      rescue Provider::TwelveData::RateLimitError
+        reset_for_retry!
+        raise
       rescue => e
         fail!
         update(error: e.message)
@@ -99,6 +102,17 @@ class Sync < ApplicationRecord
     end
   end
 
+  def fail_for_retry_exhaustion!(error_message)
+    Sync.transaction do
+      lock!
+      start! if may_start?
+      fail! if may_fail?
+      update!(error: error_message)
+    end
+
+    parent&.finalize_if_all_children_finalized
+  end
+
   # Finalizes the current sync AND parent (if it exists)
   def finalize_if_all_children_finalized
     Sync.transaction do
@@ -106,6 +120,7 @@ class Sync < ApplicationRecord
 
       # If this is the "parent" and there are still children running, don't finalize.
       return unless all_children_finalized?
+      return if pending?
 
       if syncing?
         if has_failed_children?
@@ -147,6 +162,18 @@ class Sync < ApplicationRecord
   end
 
   private
+    def reset_for_retry!
+      update_columns(
+        status: "pending",
+        error: nil,
+        pending_at: Time.current,
+        syncing_at: nil,
+        failed_at: nil,
+        completed_at: nil,
+        updated_at: Time.current
+      )
+    end
+
     def log_status_change
       Rails.logger.info("changing from #{aasm.from_state} to #{aasm.to_state} (event: #{aasm.current_event})")
     end

test/models/sync_test.rb

@@ -43,6 +43,38 @@ class SyncTest < ActiveSupport::TestCase
     assert_equal "test sync error", sync.error
   end
 
+  test "re-raises TwelveData rate limit errors and resets sync to pending" do
+    syncable = accounts(:depository)
+    sync = Sync.create!(syncable: syncable)
+    error = Provider::TwelveData::RateLimitError.new("rate limited")
+
+    syncable.expects(:perform_sync).with(sync).raises(error)
+    syncable.expects(:perform_post_sync).never
+    syncable.expects(:broadcast_sync_complete).never
+
+    assert_raises(Provider::TwelveData::RateLimitError) do
+      sync.perform
+    end
+
+    sync.reload
+    assert_equal "pending", sync.status
+    assert_nil sync.error
+    assert_nil sync.syncing_at
+    assert_nil sync.failed_at
+    assert_nil sync.completed_at
+  end
+
+  test "fail_for_retry_exhaustion! marks pending sync as failed" do
+    sync = Sync.create!(syncable: accounts(:depository))
+
+    sync.fail_for_retry_exhaustion!("rate limited too many times")
+
+    sync.reload
+    assert_equal "failed", sync.status
+    assert_equal "rate limited too many times", sync.error
+    assert_not_nil sync.failed_at
+  end
+
   test "can run nested syncs that alert the parent when complete" do
     family = families(:dylan_family)
     plaid_item = plaid_items(:one)