Merge pull request #229 from datosgobar/210-catalog-indicators

210 catalog indicators

series_tiempo_ar_api/apps/management/migrations/0008_auto_20180320_1215.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-03-20 15:15
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', '0007_auto_20180319_1431'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='indicator',
+            name='type',
+            field=models.CharField(choices=[('catalog_new', 'Cat\xe1logos nuevos'), ('catalog_total', 'Cat\xe1logos totales'), ('catalog_updated', 'Cat\xe1logos actualizados'), ('catalog_not_updated', 'Cat\xe1logos no actualizados'), ('dataset_new', 'Datasets nuevos'), ('dataset_total', 'Datasets totales'), ('dataset_updated', 'Datasets actualizados'), ('dataset_not_updated', 'Datasets no actualizados'), ('dataset_indexable', 'Datasets indexables'), ('dataset_not_indexable', 'Datasets no indexables'), ('dataset_error', 'Datasets con errores'), ('distribution_new', 'Distribuciones nuevas'), ('distribution_total', 'Distribuciones totales'), ('distribution_updated', 'Distribuciones actualizadas'), ('distribution_not_updated', 'Distribuciones no actualizadas'), ('distribution_indexable', 'Distribuciones indexables'), ('distribution_not_indexable', 'Distribuciones no indexables'), ('distribution_error', 'Distribuciones con error'), ('field_new', 'Series nuevas'), ('field_total', 'Series totales'), ('field_updated', 'Series actualizadas'), ('field_not_updated', 'Series no actualizadas'), ('field_indexable', 'Series indexables'), ('field_not_indexable', 'Series no indexables'), ('field_error', 'Series con error')], max_length=100),
+        ),
+    ]

series_tiempo_ar_api/apps/management/models.py

@@ -170,6 +170,9 @@ class Indicator(models.Model):
     CATALOG_NEW = 'catalog_new'
     CATALOG_UPDATED = 'catalog_updated'
     CATALOG_TOTAL = 'catalog_total'
+
+    CATALOG_NOT_UPDATED = 'catalog_not_updated'
+
     DATASET_NEW = 'dataset_new'
     DATASET_UPDATED = 'dataset_updated'
     DATASET_TOTAL = 'dataset_total'
@@ -201,6 +204,7 @@ class Indicator(models.Model):
         (CATALOG_NEW, 'Catálogos nuevos'),
         (CATALOG_TOTAL, 'Catálogos totales'),
         (CATALOG_UPDATED, 'Catálogos actualizados'),
+        (CATALOG_NOT_UPDATED, 'Catálogos no actualizados'),
         (DATASET_NEW, 'Datasets nuevos'),
         (DATASET_TOTAL, 'Datasets totales'),
         (DATASET_UPDATED, 'Datasets actualizados'),

series_tiempo_ar_api/libs/indexing/report/report_generator.py

@@ -42,10 +42,7 @@ def generate_email(self):
                 'new': Indicator.CATALOG_NEW,
                 'updated': Indicator.CATALOG_UPDATED,
                 'total': Indicator.CATALOG_TOTAL,
-                'not_updated': None,
-                'indexable': None,
-                'not_indexable': None,
-                'error': None,
+                'not_updated': Indicator.CATALOG_NOT_UPDATED,
             })
         msg += self.format_message(
             'Datasets',
@@ -112,24 +109,32 @@ def _get_indicator_value(self, indicator_type):
         return sum([indic.value for indic in indicator_queryset])
 
     def calculate_indicators(self):
-        for node in Node.objects.filter(indexable=True):
-            self.task.indicator_set.create(type=Indicator.CATALOG_TOTAL, value=1, node=node)
 
+        nodes = Node.objects.filter(indexable=True)
+        catalog_ids = nodes.values_list('catalog_id', flat=True)
+        for catalog_model in Catalog.objects.filter(identifier__in=catalog_ids):
+            node = nodes.get(catalog_id=catalog_model.identifier)
+
+            self.calculate_catalog_indicators(node)
             data_json = DataJson(json.loads(node.catalog))
 
-            fields_total = len(data_json.get_fields(only_time_series=True))
-            self.task.indicator_set.create(type=Indicator.FIELD_TOTAL, value=fields_total, node=node)
-            self.calculate_series_indicators(node)
+            self.calculate_series_indicators(node, data_json)
+            self.calculate_distribution_indicators(node, data_json)
+            self.calculate_dataset_indicators(node, data_json)
 
-            distribution_total = len(data_json.get_distributions(only_time_series=True))
-            self.task.indicator_set.create(type=Indicator.DISTRIBUTION_TOTAL, value=distribution_total, node=node)
-            self.calculate_distribution_indicators(node)
+    def calculate_catalog_indicators(self, node):
+        catalog_model = Catalog.objects.get(identifier=node.catalog_id)
+        updated = catalog_model.updated
+        self.task.indicator_set.create(type=Indicator.CATALOG_UPDATED, value=updated, node=node)
+        not_updated = 1 - updated
+        self.task.indicator_set.create(type=Indicator.CATALOG_NOT_UPDATED, value=not_updated,
+                                       node=node)
+        self.task.indicator_set.create(type=Indicator.CATALOG_TOTAL, value=1, node=node)
 
-            dataset_total = len(data_json.get_datasets(only_time_series=True))
-            self.task.indicator_set.create(type=Indicator.DATASET_TOTAL, value=dataset_total, node=node)
-            self.calculate_dataset_indicators(node)
+    def calculate_series_indicators(self, node, data_json):
+        fields_total = len(data_json.get_fields(only_time_series=True))
+        self.task.indicator_set.create(type=Indicator.FIELD_TOTAL, value=fields_total, node=node)
 
-    def calculate_series_indicators(self, node):
         catalog = Catalog.objects.get(identifier=node.catalog_id)
         total = self.task.indicator_set.filter(type=Indicator.FIELD_TOTAL)
         total = total[0].value if total else 0
@@ -145,7 +150,12 @@ def calculate_series_indicators(self, node):
         not_updated = indexable - updated
         self.task.indicator_set.create(type=Indicator.FIELD_NOT_UPDATED, value=not_updated, node=node)
 
-    def calculate_distribution_indicators(self, node):
+    def calculate_distribution_indicators(self, node, data_json):
+        distribution_total = len(data_json.get_distributions(only_time_series=True))
+        self.task.indicator_set.create(type=Indicator.DISTRIBUTION_TOTAL,
+                                       value=distribution_total,
+                                       node=node)
+
         catalog = Catalog.objects.get(identifier=node.catalog_id)
         total = self.task.indicator_set.filter(type=Indicator.DISTRIBUTION_TOTAL)
         total = total[0].value if total else 0
@@ -161,7 +171,10 @@ def calculate_distribution_indicators(self, node):
         not_updated = indexable - updated
         self.task.indicator_set.create(type=Indicator.DISTRIBUTION_NOT_UPDATED, value=not_updated, node=node)
 
-    def calculate_dataset_indicators(self, node):
+    def calculate_dataset_indicators(self, node, data_json):
+        dataset_total = len(data_json.get_datasets(only_time_series=True))
+        self.task.indicator_set.create(type=Indicator.DATASET_TOTAL, value=dataset_total, node=node)
+
         catalog = Catalog.objects.get(identifier=node.catalog_id)
         total = self.task.indicator_set.filter(type=Indicator.DATASET_TOTAL)
         total = total[0].value if total else 0

templates/indexing/report.txt

@@ -1,9 +1,9 @@
 {{ name }}:
-    nuevos: {{ new }},
-    actualizados: {{ updated }},
-    no actualizados: {{ not_updated }},
-    indexables: {{ indexable }},
-    no indexables: {{ not_indexable }},
-    error: {{ error }},
-    totales: {{ total }}
+    nuevos: {{ new }}
+    actualizados: {{ updated }}
+    no actualizados: {{ not_updated }}
+    {% if indexable %}indexables: {{ indexable }}
+    {% endif %}{% if not_indexable %}no indexables: {{ not_indexable }}
+    {% endif %}{% if error %}error: {{ error }}
+    {% endif %}totales: {{ total }}