cove_bods: Add support for BODS 0.4 data

openownership · Jan 8, 2025 · 7c8f243 · 7c8f243
1 parent 8b595eb
commit 7c8f243
Show file tree

Hide file tree

Showing 15 changed files with 1,127 additions and 300 deletions.
diff --git a/.github/workflows/branch-deploy.yml b/.github/workflows/branch-deploy.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Setup python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.11
         architecture: x64
     - run: pip install dokkusd
     - uses: oNaiPs/secrets-to-env-action@v1

diff --git a/.github/workflows/branch-destroy.yml b/.github/workflows/branch-destroy.yml
@@ -16,7 +16,7 @@ jobs:
     - name: Setup python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.11
         architecture: x64
     - run: pip install dokkusd
     - run: python -m dokkusd.cli destroy --appname ${{ vars.DOKKU_APP_NAME_PREFIX }}-${{ github.event.ref }}

diff --git a/.github/workflows/live-deploy.yml b/.github/workflows/live-deploy.yml
@@ -20,7 +20,7 @@ jobs:
     - name: Setup python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.11
         architecture: x64
     - run: pip install dokkusd
     - uses: oNaiPs/secrets-to-env-action@v1

diff --git a/README.md b/README.md
@@ -1,4 +1,8 @@
-# openownership-cove-bods-alpha
+# Open Ownership BODS Cove
+
+Checks data complies with the Beneficial Ownership Data Standard (BODS) versions 0.1-0.4,
+and displays any errors. Also converts uploaded data between JSON and XLSX formats.
+Based on: https://github.com/OpenDataServices/cove
 
 ## Dev installation
 

diff --git a/cove_bods/process.py b/cove_bods/process.py
diff --git a/cove_bods/templates/cove_bods/additional_checks_table.html b/cove_bods/templates/cove_bods/additional_checks_table.html
diff --git a/cove_bods/templates/cove_bods/additional_fields_table.html b/cove_bods/templates/cove_bods/additional_fields_table.html
@@ -0,0 +1,84 @@
+{% load i18n %}
+
+<table class="table">
+    <thead>
+    <tr>
+        <th class="">{% trans 'Field Name' %}</th>
+        <th class="">{% trans 'Field Path' %}</th>
+        <th class="">{% trans 'Usage Count' %}</th> 
+        <th class="">{% trans 'First 3 Values' %}</th>
+        <th class="">{% trans 'Child Fields' %}</th>
+    </tr>
+    </thead>
+    <tbody>
+    {% for full_path, info in additional_fields.items %}
+        {% if info.root_additional_field %}
+            <tr>
+                <td>
+                    {{ info.field_name }}
+                </td>
+                <td>
+                    {{ full_path }}
+                </td>
+                <td>
+                    {{ info.count }}
+                </td>
+                <td>
+                    <ul class="list-unstyled">
+                    {% for example in info.examples|slice:":3" %}
+                    <li>
+                        {{ example }}
+                    </li>
+                    {% endfor %}
+                    </ul>
+                </td>
+                <td>
+                    {% if info.additional_field_descendance %}
+                        {{info.additional_field_descendance|length}}
+                        <a data-toggle="modal" data-target="#additional-field-{{full_path|slugify}}">{% trans "(See child fields)" %}</a>
+                    {% endif %}
+                </td>
+            </tr>
+         {% endif %}
+    {% endfor %}
+    </tbody>
+</table>
+
+{% for parent_full_path, parent_info in additional_fields.items %}
+  {% if parent_info.root_additional_field and parent_info.additional_field_descendance %}
+    <div class="modal fade" id="additional-field-{{parent_full_path|slugify}}" tabindex="-1" role="dialog">
+      <div class="modal-dialog modal-lg" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+            <h4>{% blocktrans %}Child fields of {{parent_full_path}}{% endblocktrans %} </h4>
+
+          </div>
+          <div class="modal-body">
+            <table class="table">
+               <thead>
+                 <tr>
+                   <th> {% trans 'Field' %}</th>
+                   <th> {% trans 'Path to Field' %}</th>
+                   <th> {% trans 'Value' %}</th>
+                 </tr>
+               </thead>
+               <tbody>
+               {% for full_path, info in parent_info.additional_field_descendance.items %}
+                 <tr>
+                    <td>{{info.field_name}}</td>
+                    <td>{{info.path}}</td>
+                    <td>
+                        {{ info.examples|first }}
+                    </td>
+                 </tr>
+               {% endfor %}
+               </tbody>
+            </table>
+          </div>
+        </div>
+      </div>
+    </div>
+  {% endif %}
+{% endfor %}
+
diff --git a/cove_bods/templates/cove_bods/base.html b/cove_bods/templates/cove_bods/base.html
@@ -21,7 +21,7 @@
     <div class="page-header">
       <div class="row border-bottom">
         <div class="col-xs-10">
-          <a href="http://standard.openownership.org/">
+          <a href="{% url 'index' %}">
             <h1>Beneficial Ownership Data Standard (BODS)</h1>
           </a>
         </div>

diff --git a/cove_bods/templates/cove_bods/explore.html b/cove_bods/templates/cove_bods/explore.html
@@ -142,7 +142,7 @@ <h4 class="panel-title">
 
     <div class="row">
 
-        {% if additional_fields_count %}
+        {% if additional_fields_count or any_additional_fields_exist %}
 
               <a name="additional-fields" class="anchor"></a>
               <div class="panel panel-danger">
@@ -152,7 +152,7 @@ <h4 class="panel-title">
                   </h4>
                 </div>
                 <div id="additionalFieldsTable" class="collapse in panel-body">
-                  {% include "additional_fields_table.html" %}
+                  {% include "cove_bods/additional_fields_table.html" %}
                 </div>
               </div>
         {% else %}
@@ -259,7 +259,11 @@ <h4>{% trans 'Person Statements' %}</h4>
                     {% endif %}
                 {% endfor %}
 
+                {% if record_schema_used %}
+                <h4>{% trans 'Relationship Statements' %}</h4>
+                {% else %}
                 <h4>{% trans 'Ownership or Control Statements' %}</h4>
+                {% endif %}
 
                 <p>{% trans 'Total Statements' %}: {{ statistics.count_ownership_or_control_statement }}</p>
 

diff --git a/cove_bods/templates/cove_bods/validation_table.html b/cove_bods/templates/cove_bods/validation_table.html
@@ -26,19 +26,23 @@
       {% elif errors.0.validator == "pattern" %}
         <strong><code>{{ errors.0.path_ending }}</code> does not match the regex <code>{{ errors.0.validator_value }}</code></strong>
       {% elif errors.0.validator == "format" and errors.0.validator_value == "date" %}
-        <strong>Date is not in the correct format. The correct format is YYYY-MM-DD.</strong>
+        <strong><code>{{ errors.0.path_ending }}</code> is not a valid date. The correct format is YYYY-MM-DD.</strong>
       {% elif errors.0.validator == "format" and errors.0.validator_value == "date-time" %}
-        <strong>Date is not in the correct format. The correct format is YYYY-MM-DDT00:00:00Z.</strong>
+        <strong><code>{{ errors.0.path_ending }}</code> is not a valid date-time.</strong>
       {% elif errors.0.validator == "format" and errors.0.validator_value == "uri" %}
-        <strong>Invalid uri found</strong>
+        <strong><code>{{ errors.0.path_ending }}</code> is not a valid uri.</strong>
       {% elif errors.0.validator == "minLength" %}
         <strong><code>{{ errors.0.path_ending }}</code> is too short. It should be at least {{ errors.0.validator_value }} characters.</strong>
       {% elif errors.0.validator == "maxLength" %}
         <strong><code>{{ errors.0.path_ending }}</code> is too long. It should not exceed {{ errors.0.validator_value }} characters.</strong>
       {% elif errors.0.validator == "type" and errors.0.validator_value == "number" %}
         <strong><code>{{ errors.0.path_ending }}</code> should be a number. Check that the value is not null, and doesn’t contain any characters other than 0-9 and dot (<code>.</code>). Number values should not be in quotes.</strong>
       {% elif errors.0.validator == "type" and errors.0.validator_value == "array" %}
-        <strong><code>{{ errors.0.path_ending }}</code> should be a JSON array. Check that value(s) appear within square brackets, [...]</strong>
+        {% if errors.0.path_ending == "$" %}
+            <strong>The dataset should be a JSON array. Check that the object(s) appear within square brackets, [...]</strong>
+        {% else %}
+            <strong><code>{{ errors.0.path_ending }}</code> should be a JSON array. Check that value(s) appear within square brackets, [...]</strong>
+        {% endif %}
       {% elif errors.0.validator == "type" and errors.0.validator_value == "string" %}
         <strong><code>{{ errors.0.path_ending }}</code> should be a string. Check that the value is not null, and has quotes at the start and end. Escape any quotes in the value with <code>\</code></strong>
       {% elif errors.0.validator == "type" and errors.0.validator_value == "boolean" %}
@@ -48,7 +52,11 @@
       {% elif errors.0.validator == "dependencies" %}
         <strong>{{ errors.0.message  }}</strong>
       {% elif errors.0.validator == "anyOf" %}
-        <strong>{{ errors.0.message  }}</strong>
+        <strong><code>{{ errors.0.path_ending }}</code> is not a valid value. Check the description of this field in the schema documentation for details.</strong>
+      {% elif errors.0.validator == "oneOf" %}
+        <strong><code>{{ errors.0.path_ending }}</code> is not a valid value. Check the description of this field in the schema documentation for details.</strong>
+      {% elif errors.0.validator == "const" %}
+        <strong><code>{{ errors.0.path_ending }}</code> is expected to be <q>{{ errors.0.validator_value }}</q> in this Statement.</strong>
       {% else %}
         <strong>{{ errors.0.validator  }} = {{ errors.0.message  }}</strong>
       {% endif %}

diff --git a/cove_project/urls.py b/cove_project/urls.py
@@ -1,12 +1,11 @@
-from django.conf.urls import url
 from django.conf.urls.static import static
 from django.conf import settings
 from libcoveweb2.urls import urlpatterns
 import cove_bods.views
 from django.urls import re_path
 
 urlpatterns += [re_path(r"^$", cove_bods.views.NewInput.as_view(), name="index")]
-urlpatterns += [url(r'^data/(.+)$', cove_bods.views.ExploreBODSView.as_view(), name='explore')]
+urlpatterns += [re_path(r'^data/(.+)$', cove_bods.views.ExploreBODSView.as_view(), name='explore')]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 

diff --git a/requirements.in b/requirements.in
@@ -1,9 +1,11 @@
 dealer
 sentry-sdk
-Django>3.2,<3.3
+Django>5.0,<5.1
 jsonschema
-libcovebods>=0.15.0
-libcoveweb2>=0.1.0
+libcovebods>=0.16.0
+libcoveweb2>=0.4.0
 gunicorn
 django-bootstrap3
-flattentool
+flattentool>=0.27.0
+pandas
+urllib3<2 # selenium (requirements_dev.in) does not support v2