Jonathan/appeals 24987 (#19003)

* APPEALS-24987 Added redis-mutex to gemfile

* APPEALS-24987 added RedisMutex init and lock

* removed "on" lock in EPE

* added comments, retrying lock using "id"

* removing lock on "id"

* deleting extra line

* saving progress

* removing debugging lines

* changed mutex logic to use "with_lock"

* updated mutex lock key

---------

Co-authored-by: Jonathan Tsang <tsang_jonathan@bah.com>

app/models/end_product_establishment.rb

@@ -211,24 +211,28 @@ def cancel_unused_end_product!
   end
 
   def sync!
-    # There is no need to sync end_product_status if the status
-    # is already inactive since an EP can never leave that state
-    return true unless status_active?
-
-    fail EstablishedEndProductNotFound, id unless result
-
-    # load contentions now, in case "source" needs them.
-    # this VBMS call is slow and will cause the transaction below
-    # to timeout in some cases.
-    contentions unless result.status_type_code == EndProduct::STATUSES.key("Canceled")
+    RedisMutex.with_lock("EndProductEstablishment:#{id}", block: 60, expire: 100) do    # key => "EndProductEstablishment:id"
+      # There is no need to sync end_product_status if the status
+      # is already inactive since an EP can never leave that state
+      return true unless status_active?
+
+      fail EstablishedEndProductNotFound, id unless result
+
+      # load contentions now, in case "source" needs them.
+      # this VBMS call is slow and will cause the transaction below
+      # to timeout in some cases.
+      contentions unless result.status_type_code == EndProduct::STATUSES.key("Canceled")
+
+      transaction do
+        update!(synced_status: result.status_type_code)
+        status_cancelled? ? handle_cancelled_ep! : sync_source!
+        close_request_issues_with_no_decision!
+      end
 
-    transaction do
-      update!(synced_status: result.status_type_code)
-      status_cancelled? ? handle_cancelled_ep! : sync_source!
-      close_request_issues_with_no_decision!
+      save_updated_end_product_code!
     end
-
-    save_updated_end_product_code!
+  rescue RedisMutex::LockError
+    Rails.logger.error('failed to acquire lock! EPE sync is being called by another process. Please try again later.')
   rescue EstablishedEndProductNotFound, AppealRepository::AppealNotValidToReopen => error
     raise error
   rescue StandardError => error