Skip to content
This repository has been archived by the owner on Feb 8, 2019. It is now read-only.

A ton of addedd stuff #78

Open
wants to merge 22 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
0214390
updating README
May 10, 2017
463992f
add optionable send of notification for resolved incidents
May 10, 2017
40ab5ad
add HipChat notification method
May 10, 2017
c471874
add better notification logging and result keeping
May 10, 2017
dba9d0c
add unacknowledge backend code
May 10, 2017
2359793
update requirements.txt for hipchat, django-debug-toolbar and securit…
May 10, 2017
d45c1af
add incident endpoint for API as an alias to create_event
May 10, 2017
30a1442
add django-debug-toolbar to settings and urls if DEBUG==True
May 10, 2017
1b2eaea
add better error management for hipchat and default settings.py config
May 10, 2017
b6fe592
fix email notifier to be able to use custom SMTP servers; add default…
May 10, 2017
8b7d61c
modify user notif_method template for better visibility
May 10, 2017
116cf33
front-end: add cosmetic glyphicons to menus; add confirmation before …
May 10, 2017
bde6341
add optional TLS for SMTP email notifier
May 11, 2017
1b54188
add UNACK support in incidents.py
May 12, 2017
fb706dd
notification tasks return result for easier debugging with Flower
May 12, 2017
aae607b
add more verbose errors from incidents.py and send resolve notifications
May 12, 2017
bab0a4a
add escalate_to_service: the possibility to escalate(basically send, …
May 12, 2017
db2c490
fix extra space in openduty/models.py
May 12, 2017
3dd4786
add the option to notify groups of people in escalation rules, events…
May 12, 2017
77f1382
add the option to search by incident_key with autosuggest for the lat…
May 12, 2017
1644473
add example systemd service files
May 12, 2017
5dddc86
add example nginx conf for reverse rpoxy to gunicorn
May 12, 2017
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Has been tested with Nagios, works well for us. Any Pagerduty Notifier using the
[Icinga2 config](https://github.com/deathowl/OpenDuty-Icinga2) for openduty integration

#Notifications
XMPP, email, SMS, Phone(Thanks Twilio for being awesome!), and Push notifications(thanks Pushover also),and Slack are supported at the moment.
XMPP, email, SMS, Phone(Thanks Twilio for being awesome!), Push notifications(thanks Pushover, Prowl as well!)and Slack, HipChat, Rocket.chat are supported at the moment.
#Current status
Openduty is in Beta status, it can be considered stable at the moment, however major structural changes can appear anytime (not affecting the API, or the Notifier structure)

Expand Down Expand Up @@ -79,7 +79,24 @@ python manage.py runserver
```
now, you can start hacking on it.

# After models you've changed your models please run:
# Running as a service with systemd
*OpenDuty can be ran as a service with the help of gunicorn and systemd*
```
cp -r systemd/gunicorn.service.* /etc/systemd/system/

cp -r systemd/celery.service* /etc/systemd/system/

// EDIT VARIABLES IN *.service.d/main.conf TO REFLECT YOUR ENV
vi /etc/systemd/system/gunicorn.service.d/main.conf
vi /etc/systemd/system/celery.service.d/main.conf

systemctl daemon-reload
sudo systemctl start gunicorn
sudo systemctl enable gunicorn

```

# After you've changed your models please run:
```
./manage.py schemamigration openduty --auto
./manage.py schemamigration notification --auto
Expand Down
15 changes: 15 additions & 0 deletions nginx.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
server {
listen 80;
server_name _;
access_log /var/log/openduty/nginx/access.log;
error_log /var/log/openduty/nginx/error.log;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://localhost:8080;
}
}


42 changes: 32 additions & 10 deletions notification/helper.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
__author__ = 'deathowl'

from datetime import datetime, timedelta

from django.db import transaction
from notification.tasks import send_notifications
from openduty.escalation_helper import get_escalation_for_service
from openduty.models import Incident
from django.utils import timezone
from notification.models import ScheduledNotification
from django.conf import settings
Expand All @@ -28,21 +29,41 @@ def notify_user_about_incident(incident, user, delay=None, preparedmsg = None):
@staticmethod
def generate_notifications_for_incident(incident):
now = timezone.make_aware(datetime.now(), timezone.get_current_timezone())
duty_officers = get_escalation_for_service(incident.service_key)

duty_officers = []
# if incident has been escalated, notify according to the escalated service's escalation rule
if hasattr(incident, 'service_to_escalate_to') and incident.service_to_escalate_to is not None:
print "escalation rule in place to " + incident.service_to_escalate_to.name
duty_officers = get_escalation_for_service(incident.service_to_escalate_to)
with transaction.atomic():
incident.description = "[escalated] " + incident.description
incident.save()
else:
duty_officers = get_escalation_for_service(incident.service_key)
current_time = now

notifications = []
group_index = {}
user_method_index = {}

for officer_index, duty_officer in enumerate(duty_officers):
escalation_time = incident.service_key.escalate_after * (officer_index + 1)
if incident.event_type == Incident.RESOLVE and not duty_officer.profile.send_resolve_enabled:
print "Skipping notification for %s because type is RESOLVE and user %s has send_resolve_enabled OFF" % (incident.incident_key, duty_officer.username)
continue
index = 0
if hasattr(duty_officer ,'came_from_group' ):
if not duty_officer.came_from_group in group_index:
group_index[duty_officer.came_from_group] = officer_index
index = group_index[duty_officer.came_from_group]
else:
index = officer_index
escalation_time = incident.service_key.escalate_after * (index + 1)
escalate_at = current_time + timedelta(minutes=escalation_time)

user_method_index[duty_officer.username] = 0
methods = duty_officer.notification_methods.order_by('position').all()
method_index = 0

for method in methods:
notification_time = incident.service_key.retry * method_index + incident.service_key.escalate_after * officer_index
notification_time = incident.service_key.retry * user_method_index[duty_officer.username] + incident.service_key.escalate_after * index
notify_at = current_time + timedelta(minutes=notification_time)
if notify_at < escalate_at:
notification = ScheduledNotification()
Expand All @@ -51,14 +72,14 @@ def generate_notifications_for_incident(incident):
notification.notifier = method.method
notification.send_at = notify_at
uri = settings.BASE_URL + "/incidents/details/" + str(incident.id)
notification.message = "A Service is experiencing a problem: " + incident.incident_key + " " + incident.description + ". Handle at: " + uri + " Details: " + incident.details
notification.message = incident.description + ". Handle at: " + uri + " Details: " + incident.details

notifications.append(notification)

print "Notify %s at %s with method: %s" % (duty_officer.username, notify_at, notification.notifier)
print "[%s] Notify %s about %s at %s with method: %s" % (notification.incident.event_type, duty_officer.username, notification.incident.incident_key, notify_at, notification.notifier)
else:
break
method_index += 1
user_method_index[duty_officer.username] += 1

# todo: error handling

Expand Down Expand Up @@ -90,7 +111,8 @@ def generate_notifications_for_user(incident, user, delay=None, preparedmsg = No
else:
notification.message = preparedmsg
notifications.append(notification)
print "Notify %s at %s with method: %s" % (user.username, notify_at, notification.notifier)
if notification.incident:
print "[%s] Notify %s at %s with method: %s" % (notification.incident.event_type, user.username, notify_at, notification.notifier)
method_index += 1

# todo: error handling
Expand Down
3 changes: 2 additions & 1 deletion notification/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ class UserNotificationMethod(models.Model):
METHOD_SLACK = 'slack'
METHOD_PROWL = 'prowl'
METHOD_ROCKET = 'rocket'
METHOD_HIPCHAT = 'hipchat'


methods = [METHOD_XMPP, METHOD_PUSHOVER, METHOD_EMAIL, METHOD_TWILIO_SMS, METHOD_TWILIO_CALL, METHOD_SLACK, METHOD_PROWL, METHOD_ROCKET]
methods = [METHOD_XMPP, METHOD_PUSHOVER, METHOD_EMAIL, METHOD_TWILIO_SMS, METHOD_TWILIO_CALL, METHOD_SLACK, METHOD_PROWL, METHOD_ROCKET, METHOD_HIPCHAT]

user = models.ForeignKey(User, related_name='notification_methods')
position = models.IntegerField()
Expand Down
14 changes: 9 additions & 5 deletions notification/notifier/email.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ def __init__(self, config):
self.__config = config
def notify(self, notification):

gmail_user = self.__config['user']
gmail_pwd = self.__config['password']
host = self.__config['host']
port = self.__config['port']
user = self.__config['user']
password = self.__config['password']
truncate_length = int(self.__config.get('max_subject_length', 100))
FROM = self.__config['user']
TO = [notification.user_to_notify.email]
Expand All @@ -25,10 +27,12 @@ def notify(self, notification):
message = """\From: %s\nTo: %s\nSubject: %s\n\n%s
""" % (FROM, ", ".join(TO), SUBJECT, TEXT)
try:
server = smtplib.SMTP("smtp.gmail.com", 587)
server.starttls()
server = smtplib.SMTP(host, int(port))
if self.__config['tls']:
server.starttls()
server.ehlo()
server.login(gmail_user, gmail_pwd)
if user and password:
server.login(user, password)
server.sendmail(FROM, TO, message)
server.close()
print 'successfully sent the mail'
Expand Down
68 changes: 68 additions & 0 deletions notification/notifier/hipchat.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
from hypchat import HypChat
import json
import requests

class HipchatNotifier:

def __init__(self, config):
self.__config = config

def start(self):
if not self.__config['token'] or not self.__config['endpoint']:
print "HipChat configuration is missing %s" % self.__config
raise
elif self.__config['endpoint']:
return HypChat(self.__config['token'], endpoint=self.__config['endpoint'])
else:
return HypChat(self.__config['token'])

def notify(self, notification):
hc = self.start()
try:
description = notification.incident.description
details = notification.incident.details
except :
description = notification.message
details = ""
try:
message = description + " " + details
colour = "yellow"
if "CRITICAL" in message:
colour = "red"
elif "UNKNOWN" in message:
colour = "gray"
if notification.user_to_notify.profile.hipchat_room_name:
print "Notifying HipChat via API v2"
response = hc.get_room(notification.user_to_notify.profile.hipchat_room_name).notification(message, color=colour, notify="True", format="html")
elif notification.user_to_notify.profile.hipchat_room_url:
print "Notifying HipChat via a simple POST"
headers = {"content-type": "application/json"}
hip_msg = '{"color": "' + colour + '", "message": "' + message + '", "notify": true, "message_format": "html"}'
response = requests.post(notification.user_to_notify.profile.hipchat_room_url,headers=headers,data=hip_msg)
print response.content
else:
print "HipChat message send failed"
return
print "HipChat message sent"
except Exception, e:
try:
resp = json.loads(str(e))
print "Failed to send HipChat message %s " % (e, str(resp['error']['code']) + " " + resp['error']['message'])
raise
except ValueError, e2:
print "Failed to send HipChat message and failed to get it's error %s ; %s" % (e, e2)
raise

def get_all_rooms(self):
try:
hc = self.start()
if not hc:
return [""]
else:
rooms = hc.rooms().contents()
names = []
for room in rooms:
names.append(room['name'])
return names
except:
return [""]
18 changes: 11 additions & 7 deletions notification/tasks.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from __future__ import absolute_import

from notification.notifier.rocket import RocketNotifier
from openduty.celery import app
from notification.notifier.pushover import PushoverNotifier
from notification.notifier.xmpp import XmppNotifier
Expand All @@ -9,14 +8,15 @@
from notification.notifier.twilio_call import TwilioCallNotifier
from notification.notifier.slack import SlackNotifier
from notification.notifier.prowl import ProwlNotifier

from notification.notifier.rocket import RocketNotifier
from notification.notifier.hipchat import HipchatNotifier
from notification.models import ScheduledNotification, UserNotificationMethod
from django.conf import settings
from django.utils import timezone

from openduty.models import EventLog

@app.task(ignore_result=True)
@app.task(ignore_result=False)
def send_notifications(notification_id):
try:
notification = ScheduledNotification.objects.get(id = notification_id)
Expand All @@ -36,6 +36,9 @@ def send_notifications(notification_id):
notifier = ProwlNotifier(settings.PROWL_SETTINGS)
elif notification.notifier == UserNotificationMethod.METHOD_ROCKET:
notifier = RocketNotifier()
elif notification.notifier == UserNotificationMethod.METHOD_HIPCHAT:
notifier = HipchatNotifier(settings.HIPCHAT_SETTINGS)

notifier.notify(notification)
# Log successful notification
logmessage = EventLog()
Expand All @@ -44,24 +47,25 @@ def send_notifications(notification_id):
logmessage.incident_key = notification.incident
logmessage.user = notification.user_to_notify
logmessage.action = 'notified'
logmessage.data = "Notification sent to %s about %s service" % (notification.user_to_notify, logmessage.service_key, )
logmessage.data = "Notification sent to %s about %s service via %s" % (notification.user_to_notify, logmessage.service_key, notification.notifier, )
logmessage.occurred_at = timezone.now()
logmessage.save()
return (logmessage.data)
if notification.notifier != UserNotificationMethod.METHOD_TWILIO_CALL:
# In case of a twilio call, we need the object for TWiml generation
notification.delete()
except ScheduledNotification.DoesNotExist:
pass #Incident was resolved. NOP.
except:
except Exception, e:
# Log successful notification
logmessage = EventLog()
if notification.incident:
logmessage.service_key = notification.incident.service_key
logmessage.incident_key = notification.incident
logmessage.user = notification.user_to_notify
logmessage.action = 'notification_failed'
logmessage.data = "Sending notification failed to %s about %s service" % (notification.user_to_notify, logmessage.service_key, )
logmessage.data = "Sending notification failed to %s about %s service because %s" % (notification.user_to_notify, logmessage.service_key, e,)
logmessage.occurred_at = timezone.now()
logmessage.save()
return (logmessage.data)
raise

24 changes: 19 additions & 5 deletions openduty/escalation.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from django.contrib.auth.decorators import login_required

from .models import Calendar, User, SchedulePolicy, SchedulePolicyRule
from django.contrib.auth.models import Group
from django.http import Http404
from django.views.decorators.http import require_http_methods
from django.db import IntegrityError
Expand Down Expand Up @@ -32,12 +33,16 @@ def new(request):
users = User.objects.all()
except User.DoesNotExist:
users = None
try:
groups = Group.objects.all()
except Group.DoesNotExist:
groups = None
try:
calendars = Calendar.objects.all()
except Calendar.DoesNotExist:
calendars = None

return TemplateResponse(request, 'escalation/edit.html', {'calendars': calendars, 'users': users})
return TemplateResponse(request, 'escalation/edit.html', {'calendars': calendars, 'groups': groups, 'users': users})

@login_required()
def edit(request, id):
Expand All @@ -51,13 +56,17 @@ def edit(request, id):
calendars = Calendar.objects.all()
except Calendar.DoesNotExist:
calendars = None
try:
groups = Group.objects.all()
except Group.DoesNotExist:
groups = None
try:
users = User.objects.all()
except User.DoesNotExist:
users = None

return TemplateResponse(request, 'escalation/edit.html', {'item': policy, 'elements': elements,
'calendars': calendars, 'users': users})
'calendars': calendars, 'groups': groups, 'users': users})
except Calendar.DoesNotExist:
raise Http404

Expand Down Expand Up @@ -91,16 +100,21 @@ def save(request):
parts = item.split("|")
rule.escalate_after = 0 # HACK!
rule.position = idx + 1
rule.schedule = None
rule.user_id = None
rule.group_id = None
if parts[0] == "user":
rule.user_id = User.objects.get(id=parts[1])
rule.schedule = None
if parts[0] == "calendar":
elif parts[0] == "calendar":
rule.schedule = Calendar.objects.get(id=parts[1])
rule.user_id = None
elif parts[0] == "group":
rule.group_id = Group.objects.get(id=parts[1])

try:
rule.save()
except IntegrityError:
return HttpResponseRedirect(reverse('openduty.escalation.edit', None, [str(request.POST['id'])]))


return HttpResponseRedirect('/policies/')

Loading