remove hardwiring of Oxford policy data

policy/README.rst

@@ -6,11 +6,6 @@ In a sustainable portfolio management context, Policy refers to a system of prin
 
 Policies express the plan or course of action by an authority, intended to influence and determine decisions, actions, and other matters. Policy defines limits within which decisions are made.
 
-The equinox
-
-
-
-
 
 Quick Start
 ---------------------

policy/admin.py

@@ -59,7 +59,7 @@ class DataFlowAdmin(admin.ModelAdmin):
     }
     search_fields = ['name', 'short_desc', 'long_desc']
     list_display = (
-        'name', 'identifier', 'short_desc', 'oxford_n', 'dashboard_n', 'live_n')
+        'name', 'identifier', 'short_desc', 'available_n', 'dashboard_n', 'live_n')
     list_filter = ('tracked', 'menu_category')
     save_as = True
     view_on_site = False

policy/description.rst

@@ -1,58 +1,69 @@
 Policy App
 =============================
-The Policy App provides portfolio policy data oriented functionality of equinox.
+The Policy App provides portfolio-wide Policy oriented functionality of equinox.
 
-Policies are mostly discrete choices among options (that can change over time)
-
-
-TODO
------------------
-* settings.py structure for field names is unusable after changes
-* field descriptions and code lists are not used
+Policies are in general discretionary choices among a menu options (that can change over time). They can be numerical or categorical in nature.
 
 Category
 --------
-
 Policies
 
 
 Models
 ------
 
-- Policies: Store generic policy data
+Policy Models Store generic policy data
 
+* Dataflow
+* Dataseries
+* Geoslice
 
 Admin
 -----
 
-- PolicyAdmin:    PrettyJSONWidget  Search, List, Filter
+Standard Model Admin
+
 
 API
 ---
+TODO
 
 Functionality
 -----------------------
 
+* Provide overviews and dashboards for portfolio-wide policy data
+
 Management Commands
 --------------------
-None
+
+* create fixtures for Oxford COVID policy database
+* create fixtures for OECD CAPMF policy database
 
 Architecture
 ------------
 
 Dependencies
 -----------------
 
+* core app Django only
+* management commands:
+
+    * pandas
+    * scipy
+
 Is dependent on by
 --------------------
 
 File structure
 -----------------
+TODO
 
 
 Testing
 ----------------------
 
+* test/test_policy/test_models.py (test creation of core models)
+* policy/tests.py
 
 Backup Procedure
 ---------------------

policy/management/commands/create_capmf_policy_dataflow_catalog.py

@@ -39,7 +39,7 @@
     "GEO": 1
 }
 
-dataflow['CAPMF_N'] = oxford_count
+dataflow['AVAILABLE_N'] = oxford_count
 dataflow['DASHBOARD_N'] = dashboard_count\
 
 """

policy/management/commands/create_covid_policy_dataflow_catalog.py

@@ -39,7 +39,7 @@
     "GEO": 1
 }
 
-dataflow['OXFORD_N'] = oxford_count
+dataflow['AVAILABLE_N'] = oxford_count
 dataflow['DASHBOARD_N'] = dashboard_count\
 
 """
@@ -107,7 +107,7 @@ class Command(BaseCommand):
         for series in series_list:
             if series['DF_NAME'] == key:
                 oxford_count += 1
-        dataflow['OXFORD_N'] = oxford_count
+        dataflow['AVAILABLE_N'] = oxford_count
 
         # Calculate valid (full data) timeseries
         dashboard_count = 0

policy/management/commands/populate_capmf_dataflows.py

@@ -132,7 +132,7 @@ class Command(BaseCommand):
                 node_url="https://localhost/api/",
                 dimensions=dimension_data,
                 dataset_id=dataset_id,
-                oxford_n=df['CAPMF_N'],
+                available_n=df['CAPMF_N'],
                 dashboard_n=df['DASHBOARD_N'],
                 regions_n=df['REGIONS_N'],
                 subregions_n=df['SUBREGIONS_N'],

policy/management/commands/populate_covid_dataflows.py

@@ -132,7 +132,7 @@ class Command(BaseCommand):
                 node_url="https://localhost/api/",
                 dimensions=dimension_data,
                 dataset_id=dataset_id,
-                oxford_n=df['OXFORD_N'],
+                available_n=df['OXFORD_N'],
                 dashboard_n=df['DASHBOARD_N'],
                 regions_n=df['REGIONS_N'],
                 subregions_n=df['SUBREGIONS_N'],

policy/management/commands/process_covid_policy.py

@@ -29,7 +29,7 @@
 
 import policy.settings as settings
 # The field names as they are in the CSV header
-from policy.settings import field_codes, field_description, field_description_long, field_code_list, field_type
+from policy.covid_settings import field_codes, field_description, field_description_long, field_code_list, field_type
 
 """
 Starting with downloaded data

policy/models.py

@@ -170,7 +170,7 @@ class DataFlow(models.Model):
     # dataflow description node url (TODO not used)
     node_url = models.CharField(null=True, blank=True, max_length=400, help_text="Dataflow description node url")
     # number of dataseries
-    oxford_n = models.IntegerField(null=True, blank=True, help_text="Number of available dataseries")
+    available_n = models.IntegerField(null=True, blank=True, help_text="Number of available dataseries")
     # number of regions (level 1)
     regions_n = models.IntegerField(null=True, blank=True, default=0, help_text="Number of regions")
     # number of regions (level 2)

policy/templates/policy/df_breadcrumb.html

@@ -3,11 +3,11 @@
     <span
             class="breadcrumb_button"><a style="color:white;" href="{% url 'policy:DataFlow_categories' %}"
                                          target="_blank">All Data</a></span> <span
-            class="breadcrumb_button">Source:</span><span class="gray_no">Oxford</span> <span
+            class="breadcrumb_button">Source:</span><span class="gray_no">TODO</span> <span
             class="breadcrumb_button">Country:</span> <span class="gray_no"><a
             href="{% url 'policy:DataFlow_country_aggregate' dataflow.identifier %}">  {{ dataflow.name }} </a></span>
         <span
-                class="breadcrumb_button">Total DataSets:</span><span class="gray_no">{{ dataflow.oxford_n }}</span>
+                class="breadcrumb_button">Total DataSets:</span><span class="gray_no">{{ dataflow.available_n }}</span>
         <span
                 class="breadcrumb_button">Tracked DataSets:</span><span
             class="gray_no">{{ dataflow.dashboard_n }}</span> <span

policy/templates/policy/ds_breadcrumb.html

@@ -4,7 +4,7 @@
 <span
         class="breadcrumb_button"><a style="color:white;" href="{% url 'policy:DataFlow_categories' %}"
                                      target="_blank">All Data</a></span> <span class="breadcrumb_button">Source: </span><span
-            class="gray_no">Oxford</span> <span class="breadcrumb_button">Country:</span><span class="gray_no"><a
+            class="gray_no">TODO</span> <span class="breadcrumb_button">Country:</span><span class="gray_no"><a
             href="{% url 'policy:DataFlow_country_aggregate' object.df_name %}">  {{ object.df_name }} </a></span>
         {% if object.region %}
             <span class="breadcrumb_button">Region:</span>

policy/urls.py

@@ -20,7 +20,7 @@
 
 from django.urls import re_path
 
-from . import views
+import policy.views as views
 
 app_name = 'policy'
 

policy/views.py

@@ -51,7 +51,7 @@ def get_context_data(self, **kwargs):
         tracked_datasets = 0
         live_datasets = 0
         for df in object_list:
-            total_datasets += df.oxford_n
+            total_datasets += df.available_n
             if df.tracked:
                 tracked_datasets += df.dashboard_n
                 live_datasets += df.live_n
@@ -64,6 +64,7 @@ def get_context_data(self, **kwargs):
         context.update({'statistics': json.dumps(statistics)})
         return context
 
+
 #
 # 1 DataFlow Views
 #
@@ -294,14 +295,14 @@ def get_context_data(self, **kwargs):
                 f3 = Q(color='yellow')
                 f0 = Q(df_name=df)
                 series_list = DataSeries.objects.filter(f0 & (f1 | f2 | f3))
-                print('all active')
-                print(len(series_list))
+                # print('all active')
+                # print(len(series_list))
             elif color == 'all':
                 # Retrieve all dataseries objects and filter for selected dataflow
                 f0 = Q(df_name=df)
                 series_list = DataSeries.objects.filter(f0)
-                print('all all')
-                print(len(series_list))
+                # print('all all')
+                # print(len(series_list))
         elif len(region) > 0:
             if color == 'active':
                 # Retrieve all active dataseries objects and filter for selected region
@@ -311,17 +312,18 @@ def get_context_data(self, **kwargs):
                 f2 = Q(color='orange')
                 f3 = Q(color='yellow')
                 series_list = DataSeries.objects.filter(f0 & f4 & (f1 | f2 | f3))
-                print('region active')
-                print(len(series_list))
+                # print('region active')
+                # print(len(series_list))
             elif color == 'all':
                 # Retrieve all dataseries objects and filter for selected dataflow
                 f0 = Q(df_name=df)
                 f4 = Q(region=region)
                 series_list = DataSeries.objects.filter(f0 & f4)
-                print('region all')
-                print(len(series_list))
+                # print('region all')
+                # print(len(series_list))
         else:
-            print('NO REGION SPECIFIED')
+            pass
+            # print('NO REGION SPECIFIED')
 
         # Construct content description for help display
         content_data = {}
@@ -334,9 +336,6 @@ def get_context_data(self, **kwargs):
         return context
 
 
-
-
-
 class DataFlowCountryAggregateView(DetailView):
     """
     2g Dataflow Country Aggregate View (Display Only Top Level Aggregate Groupings within Dataflow)
@@ -355,7 +354,7 @@ def get_context_data(self, **kwargs):
         aggregate = []
         for series in series_list:
             if len(series.region) == 0:
-                print(series.title)
+                # print(series.title)
                 aggregate.append(series)
 
         # Construct content description for help display
@@ -523,7 +522,7 @@ def get_context_data(self, **kwargs):
         entries = DataSeries.objects.filter(query)
 
         for full_entry in entries:
-            print(full_entry)
+            # print(full_entry)
             entry = {}
             # the redirect url in case we want to drill down
             entry['url'] = 'plot/' + full_entry.identifier
@@ -657,7 +656,7 @@ def get_context_data(self, **kwargs):
         activity_string = activities[activities_short.index(activity)]
         context.update({'activity': activity_string})
         context.update({'stats_list': stats_list})
-        print(stats_list)
+        # print(stats_list)
         return context
 
 
@@ -687,7 +686,7 @@ def get_context_data(self, **kwargs):
             value = json.loads(series.metrics)[stat]
             values.append(value)
 
-        print(stat)
+        # print(stat)
         context.update({'stat': stat_string})
         context.update({'activity': activity_string})
         context.update({'values': values})
@@ -701,7 +700,7 @@ class StatsCountryCorrelationView(ListView):
     def get_context_data(self, **kwargs):
 
         metadata = DashBoardParams.objects.all()[0].country_metadata
-        print(metadata)
+        # print(metadata)
 
         context = super(ListView, self).get_context_data(**kwargs)
         activity1 = self.kwargs['activity1']