apel · tofu-rocketry · Feb 28, 2024 · Jan 6, 2023 · Jan 6, 2023 · Jan 6, 2023
diff --git a/monitoring/__init__.py b/monitoring/__init__.py
@@ -0,0 +1,2 @@
+import pymysql
+pymysql.install_as_MySQLdb()
diff --git a/monitoring/availability/templates/status.html b/monitoring/availability/templates/status.html
@@ -0,0 +1 @@
+<html>{{ message }}</html>
diff --git a/monitoring/availability/urls.py b/monitoring/availability/urls.py
@@ -1,6 +1,6 @@
 from django.conf.urls import url
 
-import views
+from monitoring.availability import views
 
 urlpatterns = [
     url(r'^$', views.status),

diff --git a/monitoring/availability/views.py b/monitoring/availability/views.py
@@ -1,15 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import time
-
 from rest_framework.decorators import api_view
 from rest_framework.response import Response
 
 
 @api_view()
 def status(requst):
-    if int(time.time()) % 2:
-        return Response("Everything OK")
-    else:
-        return Response("Everything NOT ok.")
+    return Response({"message": "OK"}, status=200, template_name="status.html")
diff --git a/monitoring/publishing/models.py b/monitoring/publishing/models.py
@@ -6,28 +6,60 @@
 
 class GridSite(models.Model):
     fetched = models.DateTimeField(auto_now=True)
-    name = models.CharField(max_length=255, primary_key=True)
+    SiteName = models.CharField(max_length=255, primary_key=True)
     updated = models.DateTimeField()
 
 
 class VSuperSummaries(models.Model):
     Site = models.CharField(max_length=255, primary_key=True)
     LatestPublish = models.DateTimeField()
+    Month = models.IntegerField()
+    Year = models.IntegerField()
+    RecordStart = models.DateTimeField()
+    RecordEnd = models.DateTimeField()
+    RecordCountPublished = models.IntegerField()
 
     class Meta:
         managed = False
         db_table = 'VSuperSummaries'
 
 
+class GridSiteSync(models.Model):
+    fetched = models.DateTimeField(auto_now=True)
+    SiteName = models.CharField(max_length=255)
+    YearMonth = models.CharField(max_length=255)
+    Year = models.IntegerField()
+    Month = models.IntegerField()
+    RecordStart = models.DateTimeField()
+    RecordEnd = models.DateTimeField()
+    RecordCountPublished = models.IntegerField()
+    RecordCountInDb = models.IntegerField()
+    SyncStatus = models.CharField(max_length=255)
+
+    class Meta:
+        # Descending order of Year and Month to display latest data first
+        ordering = ('SiteName', '-Year', '-Month')
+        unique_together = ('SiteName', 'YearMonth')
+
+
+class VSyncRecords(models.Model):
+    Site = models.CharField(max_length=255, primary_key=True)
+    RecordCountInDb = models.IntegerField()
+
+    class Meta:
+        managed = False
+        db_table = 'VSyncRecords'
+
+
 class CloudSite(models.Model):
     fetched = models.DateTimeField(auto_now=True)
-    name = models.CharField(max_length=255, primary_key=True)
-    vms = models.IntegerField(default=0)
-    script = models.CharField(max_length=255)
+    SiteName = models.CharField(max_length=255, primary_key=True)
+    Vms = models.IntegerField(default=0)
+    Script = models.CharField(max_length=255)
     updated = models.DateTimeField()
 
     class Meta:
-        ordering = ('name',)
+        ordering = ('SiteName',)
 
 
 class VAnonCloudRecord(models.Model):
@@ -46,3 +78,20 @@ def __str__(self):
                                                     self.CloudType,
                                                     self.UpdateTime,
                                                     self.VMs)
+
+
+class GridSiteSyncSubmitH(models.Model):
+    fetched = models.DateTimeField(auto_now=True)
+    SiteName = models.CharField(max_length=255)
+    YearMonth = models.CharField(max_length=255)
+    Year = models.IntegerField()
+    Month = models.IntegerField()
+    RecordStart = models.DateTimeField()
+    RecordEnd = models.DateTimeField()
+    RecordCountPublished = models.IntegerField()
+    RecordCountInDb = models.IntegerField()
+    SubmitHost = models.CharField(max_length=255)
+
+    class Meta:
+        ordering = ('SiteName',  '-Year', '-Month')
+        unique_together = ('SiteName', 'YearMonth', 'SubmitHost')
diff --git a/monitoring/publishing/serializers.py b/monitoring/publishing/serializers.py
@@ -1,6 +1,11 @@
 from rest_framework import serializers
 
-from models import CloudSite, GridSite
+from monitoring.publishing.models import (
+    CloudSite,
+    GridSite,
+    GridSiteSync,
+    GridSiteSyncSubmitH
+)
 
 
 class GridSiteSerializer(serializers.HyperlinkedModelSerializer):
@@ -11,7 +16,36 @@ class GridSiteSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = GridSite
-        fields = ('url', 'name', 'updated')
+        fields = (
+            'url',
+            'SiteName',
+            'updated'
+        )
+
+
+class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer):
+    # Override default format with None so that Python datetime is used as
+    # ouput format. Encoding will be determined by the renderer and can be
+    # formatted by a template filter.
+
+    class Meta:
+        model = GridSiteSync
+        fields = (
+            'url',
+            'SiteName',
+            'YearMonth',
+            'RecordStart',
+            'RecordEnd',
+            'RecordCountPublished',
+            'RecordCountInDb',
+            'SyncStatus'
+        )
+
+        # Sitename substitutes pk
+        lookup_field = 'SiteName'
+        extra_kwargs = {
+            'url': {'lookup_field': 'SiteName'}
+        }
 
 
 class CloudSiteSerializer(serializers.HyperlinkedModelSerializer):
@@ -22,4 +56,31 @@ class CloudSiteSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = CloudSite
-        fields = ('url', 'name', 'vms', 'script', 'updated')
+        fields = (
+            'url',
+            'SiteName',
+            'Vms',
+            'Script',
+            'updated'
+        )
+
+
+class GridSiteSyncSubmitHSerializer(serializers.HyperlinkedModelSerializer):
+    # Override default format with None so that Python datetime is used as
+    # ouput format. Encoding will be determined by the renderer and can be
+    # formatted by a template filter.
+
+    class Meta:
+        model = GridSiteSyncSubmitH
+        fields = (
+            'url',
+            'SiteName',
+            'YearMonth',
+            'RecordStart',
+            'RecordEnd',
+            'RecordCountPublished',
+            'RecordCountInDb',
+            'SubmitHost'
+        )
+
+        lookup_fields = ('SiteName', 'YearMonth')
diff --git a/monitoring/publishing/templates/cloudsites.html b/monitoring/publishing/templates/cloudsites.html
@@ -16,9 +16,9 @@ <h2>Sites publishing cloud accounting records from 2018-06-19 onwards</h2>
     </tr>
     {% for site in sites %}
     <tr>
-      <td>{{ site.name }}</td>
-      <td>{{ site.vms }}</td>
-      <td>{{ site.script }}</td>
+      <td>{{ site.SiteName }}</td>
+      <td>{{ site.Vms }}</td>
+      <td>{{ site.Script }}</td>
       <td>{{ site.updated|date:"Y-m-d H:i:s" }}</td>
     </tr>
     {% endfor %}

diff --git a/monitoring/publishing/templates/gridsites.html b/monitoring/publishing/templates/gridsites.html
@@ -17,15 +17,15 @@ <h3>APEL Publication Test</h3>
 <li>lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}</ul><hr>
 
 <table>
-  <tr><th colspan='5' class='tableheader'>{{ sites.0.name }}</td></tr>
+  <tr><th colspan='5' class='tableheader'>{{ sites.0.SiteName }}</td></tr>
   <tr>
     <th class='tableheader'>ExecutingSite</th>
     <th class='tableheader'>MeasurementDate</th>
     <th class='tableheader'>MeasurementTime</th>
     <th class='tableheader'>Publication <br> Status</th>
   </tr>
   <tr>
-    <td align='middle' class='tabletext'>{{ sites.0.name }}</td>
+    <td align='middle' class='tabletext'>{{ sites.0.SiteName }}</td>
     <td align='middle' class='tabletext'>{{ last_fetched|date:"Y-m-d" }}</td>
     <td align='middle' class='tabletext'>{{ last_fetched|date:"G:i:s" }}</td>
     <td align='middle' class='tabletext'>{{ stdout }}</td>

diff --git a/monitoring/publishing/templates/gridsync.html b/monitoring/publishing/templates/gridsync.html
@@ -0,0 +1,43 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+	<html xmlns="http://www.w3.org/1999/xhtml">
+		<head>
+			<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+      <title>APEL Publication Summary </title>
+      <link href="http://goc-accounting.grid-support.ac.uk/core/stylesheet.css" rel="stylesheet" type="text/css" />
+      <link href="http://goc-accounting.grid-support.ac.uk/core/style.css" rel="stylesheet" type="text/css" />
+		</head>
+<body>
+<h3>APEL Synchronisation Test</h3>
+<ul>
+ <li> A comparison is made between your local APEL database, and the data that you have published to the GOC.
+ <li> Major differences are flagged with FAIL.
+ <li> Information about APEL <a href='https://wiki.egi.eu/wiki/APEL'>APEL Wiki</a>
+ <li> Contact: apel-admins [at] stfc.ac.uk
+<li>lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}</ul><hr>
+
+<table>
+  <tr><th colspan='7' class='tableheader'> All sites </td></tr>
+  <tr>
+    <th class='tableheader'>Site Name</th>
+    <th class='tableheader'>Month</th>
+    <th class='tableheader'>Record Start</th>
+    <th class='tableheader'>Record End</th>
+    <th class='tableheader'>Record Count <br/> In Your Database</th>
+    <th class='tableheader'>Record Count <br/> What You Published</th>
+    <th class='tableheader'>Synchronisation <br/> Status</th>
+  </tr>
+
+  {% for record in records %}
+  <tr>
+    <td><a href="{% url 'gridsync_singlesite' SiteName=record.SiteName %}">{{ record.SiteName }}</td>
+    <td>{{ record.YearMonth }}</a> </td>    
+    <td>{{ record.RecordStart }}</td>
+    <td>{{ record.RecordEnd }}</td>
+    <td>{{ record.RecordCountPublished }}</td>
+    <td>{{ record.RecordCountInDb }}</td>
+    <td>{{ record.SyncStatus  }}</td>
+
+  </tr>
+  {% endfor %}
+
+</table>
diff --git a/monitoring/publishing/templates/gridsync_singlesite.html b/monitoring/publishing/templates/gridsync_singlesite.html
@@ -0,0 +1,41 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+	<html xmlns="http://www.w3.org/1999/xhtml">
+		<head>
+			<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+      <title>APEL Publication Summary </title>
+      <link href="http://goc-accounting.grid-support.ac.uk/core/stylesheet.css" rel="stylesheet" type="text/css" />
+      <link href="http://goc-accounting.grid-support.ac.uk/core/style.css" rel="stylesheet" type="text/css" />
+		</head>
+<body>
+<h3>APEL Synchronisation Test</h3>
+<ul>
+ <li> A comparison is made between your local APEL database, and the data that you have published to the GOC.
+ <li> Major differences are flagged with FAIL.
+ <li> Information about APEL <a href='https://wiki.egi.eu/wiki/APEL'>APEL Wiki</a>
+ <li> Contact: apel-admins [at] stfc.ac.uk
+<li>lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}</ul><hr>
+
+<table>
+  <tr><th colspan='6' class='tableheader'> {{ records.0.SiteName }} </td></tr>
+  <tr>
+    <th class='tableheader'>Month</th>
+    <th class='tableheader'>Record Start</th>
+    <th class='tableheader'>Record End</th>
+    <th class='tableheader'>Record Count <br/> In Your Database</th>
+    <th class='tableheader'>Record Count <br/> What You Published</th>
+    <th class='tableheader'>Synchronisation <br/> Status</th>
+  </tr>
+
+  {% for record in records %}
+  <tr>
+    <td><a href="{% url 'gridsync_submithost'  SiteName=record.SiteName YearMonth=record.YearMonth %}">{{ record.YearMonth }}</a> </td>    
+    <td>{{ record.RecordStart }}</td>
+    <td>{{ record.RecordEnd }}</td>
+    <td>{{ record.RecordCountPublished }}</td>
+    <td>{{ record.RecordCountInDb }}</td>
+    <td>{{ record.SyncStatus  }}</td>
+
+  </tr>
+  {% endfor %}
+
+</table>
diff --git a/monitoring/publishing/templates/gridsync_submithost.html b/monitoring/publishing/templates/gridsync_submithost.html
@@ -0,0 +1,43 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+	<html xmlns="http://www.w3.org/1999/xhtml">
+		<head>
+			<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+      <title>APEL Publication Summary </title>
+      <link href="http://goc-accounting.grid-support.ac.uk/core/stylesheet.css" rel="stylesheet" type="text/css" />
+      <link href="http://goc-accounting.grid-support.ac.uk/core/style.css" rel="stylesheet" type="text/css" />
+		</head>
+<body>
+<h3>APEL Synchronisation Test</h3>
+<ul>
+ <li> A comparison is made between your local APEL database, and the data that you have published to the GOC.
+ <li> Major differences are flagged with FAIL.
+ <li> Information about APEL <a href='https://wiki.egi.eu/wiki/APEL'>APEL Wiki</a>
+ <li> Contact: apel-admins [at] stfc.ac.uk
+<li>lastBuild : {{ last_fetched|date:"Y-m-d H:i:s.u"|slice:":22" }}</ul><hr>
+
+<table>
+  <tr><th colspan='6' class='tableheader'> {{submisthosts.0.SiteName}}, {{submisthosts.0.YearMonth}} </td></tr>
+  <tr>
+    <th class='tableheader'>Month</th>
+    <th class='tableheader'>SubmitHost</th>
+    <th class='tableheader'>RecordStart</th>
+    <th class='tableheader'>RecordEnd</th>
+    <th class='tableheader'>Record Count <br/> In Your Database</th>
+    <th class='tableheader'>Record Count <br/> What You Published</th>
+  </tr>
+
+  {% for host in submisthosts %}
+  <tr>
+
+    <td>{{ host.YearMonth }} </td>
+    <td>{{ host.SubmitHost }}</td>
+    <td>{{ host.RecordStart }}</td>
+    <td>{{ host.RecordEnd }}</td>
+    <td>{{ host.RecordCountPublished }}</td>
+    <td>{{ host.RecordCountInDb }}</td>
+
+
+  </tr>
+  {% endfor %}
+
+</table>
diff --git a/monitoring/publishing/urls.py b/monitoring/publishing/urls.py
@@ -2,13 +2,26 @@
 
 from rest_framework import routers
 
-import views
+from monitoring.publishing import views
+from django.urls import re_path
 
 router = routers.SimpleRouter()
 router.register(r'cloud', views.CloudSiteViewSet)
 router.register(r'grid', views.GridSiteViewSet)
-
+router.register(r'gridsync', views.GridSiteSyncViewSet)
+router.register(r'gridsync', views.GridSiteSyncSubmitHViewSet)
 
 urlpatterns = [
-    url(r'^', include(router.urls)),
+    re_path(
+        r'^gridsync/(?P<SiteName>[a-zA-Z0-9-]+)/$',
+        views.GridSiteSyncViewSet.as_view({'get': 'retrieve'}),
+        name='gridsync_singlesite'
+    ),
+    re_path(
+        r'^gridsync/(?P<SiteName>[a-zA-Z0-9-]+)/(?P<YearMonth>[0-9-]+)/$',
+        views.GridSiteSyncSubmitHViewSet.as_view({'get': 'retrieve'}),
+        name='gridsync_submithost'
+    ),
 ]
+
+urlpatterns += router.urls