Fixed the web socket reconnection handling to use exponential retry b…

…ack off.

README.md

@@ -2,6 +2,10 @@
 
 ## Development
 
+Its recommended to setup a virtual environment, e.g. `virtualenv venv`.
+
+Then do `pip install requirements.txt`
+
 You will need the following env vars set. In Prod, these would be passed in via Beep Boop.
 
 `export BEEPBOOP_TOKEN=foo`
@@ -10,4 +14,5 @@ You will need the following env vars set. In Prod, these would be passed in via
 
 `export BEEPBOOP_RESOURCER=ws://localhost:9000/ws` -- recommend using the [Beep Boop dev-console](https://github.com/BeepBoopHQ/dev-console) and setting this value to it.
 
-Run `python ./examples/simple.py` which registers listeners as a consuming app might.
+Run `python ./examples/simple.py` which registers listeners as a consuming app might.  Note you may need to set `export PYTHONPATH=.` so that
+it can properly import the beepboop module.

beepboop.py

@@ -12,8 +12,8 @@
 logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s', level=log_level)
 logger = logging.getLogger(__name__)
 
-sys.path.append('/Users/skud/projects/src/github.com/BeepBoopHQ/websocket-client')
 import websocket
+import random
 
 
 class BeepBoop(object):
@@ -43,10 +43,15 @@ def start(self):
     def handlers(self, handler_funcs_dict):
         self.handler_funcs = handler_funcs_dict
 
+
     def _connect(self):
-        self.iter += 1
-        print ('iteration: ' + str(self.iter))
-        self.ws_app.run_forever()
+        # while loop makes sure we retry to connect on server down or network failure
+        while True:
+            self.ws_app.run_forever()
+            self.iter += 1
+            logging.debug('reconnecting attempt: ' + str(self.iter))
+            expBackoffSleep(self.iter, 32)
+
 
     def on_message(self, ws, message):
         if self.handler_funcs['on_message']:
@@ -60,14 +65,11 @@ def on_close(self, ws):
         if self.handler_funcs['on_close']:
             self.handler_funcs['on_close'](ws)
 
-        # must be connected to the resourcer so we need to keep "retrying".
-        # NOTE: websocket lib should handle cleanup but i'm seeing leaking
-        time.sleep(1)
-        self._connect()
-
     def on_open(self, ws):
         self.ws_conn = ws
         self._authorize()
+        # reset to 0 since we've reopened a connection
+        self.iter = 0 
         if self.handler_funcs['on_open']:
             self.handler_funcs['on_open'](ws)
 
@@ -88,3 +90,10 @@ def _getprop(self, param, env_var):
             exit()
 
         return v
+
+ # Use binary exponential backoff to desynchronize client requests.
+ # As described by: https://cloud.google.com/storage/docs/exponential-backoff
+def expBackoffSleep(n, max_backoff_time):
+    time_to_sleep = min(random.random() * (2**n), max_backoff_time)
+    logging.debug('time to sleep: ' + str(time_to_sleep))
+    time.sleep(time_to_sleep)

examples/simple.py

@@ -2,7 +2,6 @@
 import sys
 import os
 import pprint
-sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
 
 import beepboop
 

requirements.txt

@@ -1 +1,3 @@
-websocket-client>=0.22.0
+six==1.10.0
+websocket-client==0.35.0
+wheel==0.24.0