Do not raise delivery failure on 4xx errors, increase stoplight threshold (#7541)

* Do not raise delivery failure on 4xx errors, increase stoplight threshold

Stoplight failure threshold from 3 to 10
Status code 429 will raise a failure/get retried

* Oops

app/workers/activitypub/delivery_worker.rb

@@ -3,6 +3,9 @@
 class ActivityPub::DeliveryWorker
   include Sidekiq::Worker
 
+  STOPLIGHT_FAILURE_THRESHOLD = 10
+  STOPLIGHT_COOLDOWN = 60
+
   sidekiq_options queue: 'push', retry: 16, dead: false
 
   HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
@@ -31,17 +34,23 @@ class ActivityPub::DeliveryWorker
   def perform_request
     light = Stoplight(@inbox_url) do
       build_request.perform do |response|
-        raise Mastodon::UnexpectedResponseError, response unless response_successful?(response)
+        raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
       end
     end
 
-    light.run
+    light.with_threshold(STOPLIGHT_FAILURE_THRESHOLD)
+         .with_cool_off_time(STOPLIGHT_COOLDOWN)
+         .run
   end
 
   def response_successful?(response)
     response.code > 199 && response.code < 300
   end
 
+  def response_error_unsalvageable?(response)
+    response.code > 399 && response.code < 500 && response.code != 429
+  end
+
   def failure_tracker
     @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url)
   end