Skip to content

Commit

Permalink
include current datetime in log files for memmon
Browse files Browse the repository at this point in the history
  • Loading branch information
@erral
erral committed Feb 1, 2017
1 parent 22e2f12 commit a656d4f
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 43 deletions.
34 changes: 24 additions & 10 deletions superlance/memmon.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
import os
import sys
import time
import datetime
from collections import namedtuple
from superlance.compat import maxint
from superlance.compat import xmlrpclib
Expand Down Expand Up @@ -147,7 +148,8 @@ def runforever(self, test=False):
if self.any is not None:
status.append('Checking any=%s' % self.any)

self.stderr.write('\n'.join(status) + '\n')
self.stderr.write(datetime.datetime.now().isoformat() + ' ' +
'\n'.join(status) + '\n')

infos = self.rpc.supervisor.getAllProcessInfo()

Expand All @@ -170,19 +172,28 @@ def runforever(self, test=False):

for n in name, pname:
if n in self.programs:
self.stderr.write('RSS of %s is %s\n' % (pname, rss))
if rss > self.programs[name]:
self.stderr.write('%s RSS of %s is %s\n' %
(datetime.datetime.now().isoformat(),
pname,
rss))
if rss > self.programs[name]:
self.restart(pname, rss)
continue

if group in self.groups:
self.stderr.write('RSS of %s is %s\n' % (pname, rss))
self.stderr.write('%s RSS of %s is %s\n' %
(datetime.datetime.now().isoformat(),
pname,
rss))
if rss > self.groups[group]:
self.restart(pname, rss)
continue

if self.any is not None:
self.stderr.write('RSS of %s is %s\n' % (pname, rss))
self.stderr.write('%s RSS of %s is %s\n' %
(datetime.datetime.now().isoformat(),
pname,
rss))
if rss > self.any:
self.restart(pname, rss)
continue
Expand All @@ -195,12 +206,14 @@ def runforever(self, test=False):
def restart(self, name, rss):
info = self.rpc.supervisor.getProcessInfo(name)
uptime = info['now'] - info['start'] #uptime in seconds
self.stderr.write('Restarting %s\n' % name)
self.stderr.write('%s Restarting %s\n' %
(datetime.datetime.now().isoformat(), name))
memmonId = self.memmonName and " [%s]" % self.memmonName or ""
try:
self.rpc.supervisor.stopProcess(name)
except xmlrpclib.Fault as e:
msg = ('Failed to stop process %s (RSS %s), exiting: %s' %
(name, rss, e))
msg = ('%s Failed to stop process %s (RSS %s), exiting: %s' %
(datetime.datetime.now().isoformat(), name, rss, e))
self.stderr.write(str(msg))
if self.email:
subject = self.format_subject(
Expand All @@ -212,8 +225,9 @@ def restart(self, name, rss):
try:
self.rpc.supervisor.startProcess(name)
except xmlrpclib.Fault as e:
msg = ('Failed to start process %s after stopping it, '
'exiting: %s' % (name, e))
msg = ('%s Failed to start process %s after stopping it, '
'exiting: %s' %
(datetime.datetime.now().isoformat(), name, e))
self.stderr.write(str(msg))
if self.email:
subject = self.format_subject(
Expand Down
65 changes: 32 additions & 33 deletions superlance/tests/memmon_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,13 @@ def test_runforever_tick_programs(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 8)
self.assertEqual(lines[0], 'Checking programs bar=0, baz_01=0, foo=0')
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[2], 'Restarting foo:foo')
self.assertEqual(lines[3], 'RSS of bar:bar is 2265088')
self.assertEqual(lines[4], 'Restarting bar:bar')
self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088')
self.assertEqual(lines[6], 'Restarting baz:baz_01')
self.assertIn('Checking programs bar=0, baz_01=0, foo=0', lines[0])
self.assertIn('RSS of foo:foo is 2264064', lines[1])
self.assertIn('Restarting foo:foo', lines[2])
self.assertIn('RSS of bar:bar is 2265088', lines[3])
self.assertIn('Restarting bar:bar', lines[4])
self.assertIn('RSS of baz:baz_01 is 2265088', lines[5])
self.assertIn('Restarting baz:baz_01', lines[6])
self.assertEqual(lines[7], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -80,9 +80,9 @@ def test_runforever_tick_groups(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 4)
self.assertEqual(lines[0], 'Checking groups foo=0')
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[2], 'Restarting foo:foo')
self.assertIn('Checking groups foo=0', lines[0])
self.assertIn('RSS of foo:foo is 2264064', lines[1])
self.assertIn('Restarting foo:foo', lines[2])
self.assertEqual(lines[3], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -102,13 +102,13 @@ def test_runforever_tick_any(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 8)
self.assertEqual(lines[0], 'Checking any=0')
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[2], 'Restarting foo:foo')
self.assertEqual(lines[3], 'RSS of bar:bar is 2265088')
self.assertEqual(lines[4], 'Restarting bar:bar')
self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088')
self.assertEqual(lines[6], 'Restarting baz:baz_01')
self.assertIn('Checking any=0', lines[0])
self.assertIn('RSS of foo:foo is 2264064', lines[1])
self.assertIn('Restarting foo:foo', lines[2])
self.assertIn('RSS of bar:bar is 2265088', lines[3])
self.assertIn('Restarting bar:bar', lines[4])
self.assertIn('RSS of baz:baz_01 is 2265088', lines[5])
self.assertIn('Restarting baz:baz_01', lines[6])
self.assertEqual(lines[7], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -123,12 +123,12 @@ def test_runforever_tick_programs_and_groups(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 7)
self.assertEqual(lines[0], 'Checking programs baz_01=0')
self.assertEqual(lines[1], 'Checking groups foo=0')
self.assertEqual(lines[2], 'RSS of foo:foo is 2264064')
self.assertEqual(lines[3], 'Restarting foo:foo')
self.assertEqual(lines[4], 'RSS of baz:baz_01 is 2265088')
self.assertEqual(lines[5], 'Restarting baz:baz_01')
self.assertIn('Checking programs baz_01=0', lines[0])
self.assertIn('Checking groups foo=0', lines[1])
self.assertIn('RSS of foo:foo is 2264064', lines[2])
self.assertIn('Restarting foo:foo', lines[3])
self.assertIn('RSS of baz:baz_01 is 2265088', lines[4])
self.assertIn('Restarting baz:baz_01', lines[5])
self.assertEqual(lines[6], '')
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
Expand All @@ -148,8 +148,8 @@ def test_runforever_tick_programs_norestart(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 3)
self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint)
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertIn('Checking programs foo=%s' % maxint, lines[0])
self.assertIn('RSS of foo:foo is 2264064', lines[1])
self.assertEqual(lines[2], '')
self.assertEqual(memmon.mailed, False)

Expand All @@ -163,8 +163,8 @@ def test_stopprocess_fault_tick_programs_norestart(self):
memmon.runforever(test=True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 3)
self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint)
self.assertEqual(lines[1], 'RSS of foo:foo is 2264064')
self.assertIn('Checking programs foo=%s' % maxint, lines[0])
self.assertIn('RSS of foo:foo is 2264064', lines[1])
self.assertEqual(lines[2], '')
self.assertEqual(memmon.mailed, False)

Expand Down Expand Up @@ -192,17 +192,17 @@ def test_stopprocess_fails_to_stop(self):
self.assertRaises(xmlrpclib.Fault, memmon.runforever, True)
lines = memmon.stderr.getvalue().split('\n')
self.assertEqual(len(lines), 4)
self.assertEqual(lines[0], 'Checking programs BAD_NAME=%s' % 0)
self.assertEqual(lines[1], 'RSS of BAD_NAME:BAD_NAME is 2264064')
self.assertEqual(lines[2], 'Restarting BAD_NAME:BAD_NAME')
self.assertTrue(lines[3].startswith('Failed'))
self.assertIn('Checking programs BAD_NAME=%s' % 0, lines[0])
self.assertIn('RSS of BAD_NAME:BAD_NAME is 2264064', lines[1])
self.assertIn('Restarting BAD_NAME:BAD_NAME', lines[2])
self.assertIn('Failed', lines[3])
mailed = memmon.mailed.split('\n')
self.assertEqual(len(mailed), 4)
self.assertEqual(mailed[0], 'To: [email protected]')
self.assertEqual(mailed[1],
'Subject: memmon: failed to stop process BAD_NAME:BAD_NAME, exiting')
self.assertEqual(mailed[2], '')
self.assertTrue(mailed[3].startswith('Failed'))
self.assertIn('Failed', mailed[3])

def test_subject_no_name(self):
"""set the name to None to check if subject formats to:
Expand Down Expand Up @@ -392,4 +392,3 @@ def test_argparser(self):

if __name__ == '__main__':
unittest.main()

0 comments on commit a656d4f

Please sign in to comment.