Add scheduled job to sync all accounts every 24 hours (#330)

* Add scheduled job to sync all accounts every 30 minutes

Signed-off-by: Nikhil Badyal <nikhill773384@gmail.com>

* Change job queue from default to scheduled

Signed-off-by: Juan José Mata <juanjo.mata@gmail.com>

* Flatten job into single directory

* Every 30 minutes is a bit much and will trigger Sentry warnings

* Locking and logging improvements

* Add support for extra Sidekiq goodies

---------

Signed-off-by: Nikhil Badyal <nikhill773384@gmail.com>
Signed-off-by: Juan José Mata <juanjo.mata@gmail.com>
Co-authored-by: Nikhil Badyal <nikhill773384@gmail.com>

Gemfile

@@ -33,6 +33,7 @@ gem "hotwire_combobox"
 # Background Jobs
 gem "sidekiq"
 gem "sidekiq-cron"
+gem "sidekiq-unique-jobs"
 
 # Monitoring
 gem "vernier"

Gemfile.lock

@@ -603,6 +603,10 @@ GEM
       fugit (~> 1.8, >= 1.11.1)
       globalid (>= 1.0.1)
       sidekiq (>= 6.5.0)
+    sidekiq-unique-jobs (8.0.11)
+      concurrent-ruby (~> 1.0, >= 1.0.5)
+      sidekiq (>= 7.0.0, < 9.0.0)
+      thor (>= 1.0, < 3.0)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -759,6 +763,7 @@ DEPENDENCIES
   sentry-sidekiq
   sidekiq
   sidekiq-cron
+  sidekiq-unique-jobs
   simplecov
   skylight
   stackprof

app/jobs/sync_all_job.rb

@@ -0,0 +1,14 @@
+class SyncAllJob < ApplicationJob
+  queue_as :scheduled
+  sidekiq_options lock: :until_executed, on_conflict: :log
+
+  def perform
+    Rails.logger.info("Starting sync for all families")
+    Family.find_each do |family|
+      family.sync_later
+    rescue => e
+      Rails.logger.error("Failed to sync family #{family.id}: #{e.message}")
+    end
+    Rails.logger.info("Completed sync for all families")
+  end
+end

config/schedule.yml

@@ -18,3 +18,9 @@ run_security_health_checks:
   class: "SecurityHealthCheckJob"
   queue: "scheduled"
   description: "Runs security health checks to detect issues with security data"
+
+sync_all_accounts:
+  cron: "22 2 * * *" # every 24 hours at 2:22am
+  class: "SyncAllJob"
+  queue: "scheduled"
+  description: "Syncs all accounts for all families"