From 33ba9ef3f654aab7f919ec55e2c71cd4e579ed5f Mon Sep 17 00:00:00 2001
From: Reouven Mimoun <reouvenm@cymulate.com>
Date: Tue, 17 Mar 2026 13:15:49 +0200
Subject: [PATCH] fix(webhooks): add scheduled task to unlock stale webhook
 request locks

When a worker process crashes or is killed (common in Kubernetes with
rolling updates, OOM kills, etc.), locks held on webhook_requests are
never released. Unlike QueuedMessage which has TidyQueuedMessagesTask
to clean up stale locks, WebhookRequest had no equivalent mechanism,
causing webhook delivery to be permanently blocked.

This adds:
- A `with_stale_lock` scope on WebhookRequest (locks older than 1 hour)
- A TidyWebhookRequestsTask scheduled task that runs hourly to unlock
  stale webhook requests so they can be retried

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 app/lib/worker/process.rb                        |  1 +
 app/models/webhook_request.rb                    |  2 ++
 .../tidy_webhook_requests_task.rb                | 16 ++++++++++++++++
 3 files changed, 19 insertions(+)
 create mode 100644 app/scheduled_tasks/tidy_webhook_requests_task.rb

diff --git a/app/lib/worker/process.rb b/app/lib/worker/process.rb
index e2058bc0a..5b27e83ec 100644
--- a/app/lib/worker/process.rb
+++ b/app/lib/worker/process.rb
@@ -42,6 +42,7 @@ class Process
       PruneWebhookRequestsScheduledTask,
       SendNotificationsScheduledTask,
       TidyQueuedMessagesTask,
+      TidyWebhookRequestsTask,
     ].freeze
 
     # @param [Integer] thread_count The number of worker threads to run in this process
diff --git a/app/models/webhook_request.rb b/app/models/webhook_request.rb
index dfbf63e57..895e0cd18 100644
--- a/app/models/webhook_request.rb
+++ b/app/models/webhook_request.rb
@@ -36,6 +36,8 @@ class WebhookRequest < ApplicationRecord
 
   serialize :payload, type: Hash
 
+  scope :with_stale_lock, -> { where("locked_at IS NOT NULL AND locked_at < ?", 1.hour.ago) }
+
   class << self
 
     def trigger(server, event, payload = {})
diff --git a/app/scheduled_tasks/tidy_webhook_requests_task.rb b/app/scheduled_tasks/tidy_webhook_requests_task.rb
new file mode 100644
index 000000000..54c0b6389
--- /dev/null
+++ b/app/scheduled_tasks/tidy_webhook_requests_task.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class TidyWebhookRequestsTask < ApplicationScheduledTask
+
+  def call
+    WebhookRequest.with_stale_lock.find_each do |request|
+      logger.info "unlocking stale webhook request #{request.id} (locked at #{request.locked_at} by #{request.locked_by})"
+      request.unlock
+    end
+  end
+
+  def self.next_run_after
+    quarter_to_each_hour
+  end
+
+end