rabbit: raise failure-timeout

The failure timeout of 30s is far too low. Essentially it means that a
failed node is considered ready after 30s. Given that any start or
stop operation takes considerably more than 30s. We should only expire
failures after around 30 minutes to prevent flapping services.

chef/cookbooks/rabbitmq/attributes/default.rb

@@ -42,13 +42,14 @@
 default[:rabbitmq][:ha][:op][:start][:timeout] = "300s"
 default[:rabbitmq][:ha][:op][:promote][:timeout] = "180s"
 default[:rabbitmq][:ha][:op][:monitor][:interval] = "10s"
-default[:rabbitmq][:ha][:clustered_op][:start][:timeout] = "360s"
-default[:rabbitmq][:ha][:clustered_op][:stop][:timeout] = "120s"
-default[:rabbitmq][:ha][:clustered_op][:promote][:timeout] = "120s"
-default[:rabbitmq][:ha][:clustered_op][:demote][:timeout] = "120s"
+default[:rabbitmq][:ha][:clustered_op][:start][:timeout] = "540s"
+default[:rabbitmq][:ha][:clustered_op][:stop][:timeout] = "180s"
+default[:rabbitmq][:ha][:clustered_op][:promote][:timeout] = "180s"
+default[:rabbitmq][:ha][:clustered_op][:demote][:timeout] = "180s"
 default[:rabbitmq][:ha][:clustered_op][:notify][:timeout] = "180s"
 default[:rabbitmq][:ha][:clustered_op][:monitor] = [
-  { interval: "30s" }, { interval: "27s", role: "Master" }
+  { interval: "60s", timeout: "90s" },
+  { interval: "57s", timeout: "90s", role: "Master" }
 ]
 
 default[:rabbitmq][:hipe_compile] = false

chef/cookbooks/rabbitmq/recipes/ha_cluster.rb

@@ -122,7 +122,7 @@
   op node[:rabbitmq][:ha][:clustered_op]
   meta ({
     "migration-threshold" => "10",
-    "failure-timeout" => "30s",
+    "failure-timeout" => "1800s",
     "resource-stickiness" => "100"
   })
   action :update