Skip to content

Commit

Permalink
Make API more robust
Browse files Browse the repository at this point in the history
  • Loading branch information
@drkane
drkane committed Jan 31, 2024
1 parent aa27af5 commit 6a9cbb3
Show file tree
Hide file tree
Showing 64 changed files with 6,000 additions and 734 deletions.
49 changes: 24 additions & 25 deletions .github/workflows/pythontest.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ name: Django tests
on: [push, pull_request]
jobs:
django-test:

runs-on: ubuntu-latest

services:
postgres_data:
image: postgres:14.5
Expand All @@ -26,32 +25,32 @@ jobs:
- 5433:5432
# needed because the postgres container does not provide a healthcheck
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

env:
DATABASE_URL: postgres://postgres:postgres@localhost:5432/github_actions_data
DATABASE_ADMIN_URL: postgres://postgres:postgres@localhost:5433/github_actions_admin
SECRET_KEY: 1234567890

steps:
- uses: actions/checkout@v3
- name: Parse Python version
run: sed s/python-// runtime.txt | head > .python-version
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
cache: 'pip'
- name: psycopg2 prerequisites
run: sudo apt-get install libxml2-dev libxslt-dev python3-dev libpq-dev
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Run migrations
run: |
python manage.py migrate --database=data --noinput
python manage.py migrate --database=admin --noinput
python manage.py collectstatic
- name: Test with django test
run: |
python manage.py test
- uses: actions/checkout@v3
- name: Parse Python version
run: sed s/python-// runtime.txt | head > .python-version
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: ".python-version"
cache: "pip"
- name: psycopg2 prerequisites
run: sudo apt-get install libxml2-dev libxslt-dev python3-dev libpq-dev
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Run migrations
run: |
python manage.py migrate --database=data --noinput
python manage.py migrate --database=admin --noinput
python manage.py collectstatic
- name: Test with django test
run: |
pytest
99 changes: 53 additions & 46 deletions addtocsv/jinja2/addtocsv.html.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,28 @@
{% block content %}
<form action="" method="post" enctype="multipart/form-data" class="w-100 w-70-l fl-l pr3-l">
<div class="w-100 mb4 entry-content cf">
<p class="measure">Add data to a CSV file by looking it up from a column with a charity number or other organisation identifier.</p>
<p class="measure">Add data to a CSV file by looking it up from a column with a charity number or other organisation
identifier.</p>
<p class="measure">The file should be separated by commas (<code>,</code>) - not semicolons or tabs)
and the first row should contain the column names.</p>
and the first row should contain the column names.</p>
<a class="dn-l link underline blue f5" href="#privacy">See important note on privacy below</a>
</div>
<div class="w-100 b--light-gray ba bw1 br3 mb4" id="addtocsv">

<div class="w-100 pa2 b--light-gray bb bw1 stage" id="stage-select-file">
<h3 class="pa0 ma0 dib">
Step 1:
Step 1:
<span class="normal">Select CSV file</span>
</h3>
<span class="fr contents-top" v-if="stage > 0">
<a href="#" id="reset-select-file" class="stage-reset link f6 blue underline dim" v-on:click.prevent="csv = null">Change file</a>
<a href="#" id="reset-select-file" class="stage-reset link f6 blue underline dim"
v-on:click.prevent="csv = null">Change file</a>
<span class="file-name dib code pa1 bg-light-gray lh-solid" id="csvfilename">[[filename]]</span>
</span>
<div class="contents mv3" v-else-if="stage == 0">
<label class="file-label">
<input class="file-input w-100" type="file" name="csvfile" id="csvfile" accept="text/csv,.csv" v-on:change="selectFile">
<input class="file-input w-100" type="file" name="csvfile" id="csvfile" accept="text/csv,.csv"
v-on:change="selectFile">
<span class="file-cta">
<span class="file-icon">
<i class="fas fa-upload"></i>
Expand All @@ -52,25 +55,27 @@

<div class="w-100 pa2 b--light-gray bb bw1 stage" id="stage-select-identifier-field">
<h3 class="pa0 ma0 dib">
Step 2:
Step 2:
<span class="normal">Select organisation identifier field</span>
</h3>
<span class="fr contents-top" v-if="stage > 1">
<a href="#" id="reset-select-identifier-field" class="stage-reset link f6 blue underline dim" v-on:click.prevent="column_to_use = null">Change field</a>
<a href="#" id="reset-select-identifier-field" class="stage-reset link f6 blue underline dim"
v-on:click.prevent="column_to_use = null">Change field</a>
<span class="dib code pa1 bg-light-gray lh-solid" id="column-name-desc">[[column_to_use]]</span>
<input class="dn" type="text" name="column_name" id="column_name" :value="column_to_use">
</span>
<div class="contents mv4" v-else-if="stage == 1">
<div class="field mb3" id="csvpreview">
<ul class="list pa0 ma0">
<li v-for="f in field_select" class="mb2">
<label class="pointer">
<input type="radio" name="identifier_field" :value=[[f.name]] v-on:click.prevent="column_to_use = f.name" />
<span class="code pa1 bg-light-gray underline-hover">[[f.name]]</span>
<ul class="list pa0 ma0">
<li class="dib mr2 gray mt1 f6 tl truncate mw5" v-for="v in f.example_values">[[v]]</li>
</ul>
</label>
<label class="pointer">
<input type="radio" name="identifier_field" :value=[[f.name]]
v-on:click.prevent="column_to_use = f.name" />
<span class="code pa1 bg-light-gray underline-hover">[[f.name]]</span>
<ul class="list pa0 ma0">
<li class="dib mr2 gray mt1 f6 tl truncate mw5" v-for="v in f.example_values">[[v]]</li>
</ul>
</label>
</li>
</ul>
</div>
Expand All @@ -79,22 +84,23 @@

<div class="w-100 pa2 b--light-gray bb bw1 stage" id="stage-identifier-type-field">
<h3 class="pa0 ma0 dib">
Step 3:
Step 3:
<span class="normal">Select identifier type</span>
</h3>
<span class="fr contents-top" v-if="stage > 2">
<a href="#" id="reset-identifier-type-field" class="stage-reset link f6 blue underline dim" v-on:click.prevent="identifier_type = null">Change field</a>
<a href="#" id="reset-identifier-type-field" class="stage-reset link f6 blue underline dim"
v-on:click.prevent="identifier_type = null">Change field</a>
<span class="dib code pa1 bg-light-gray lh-solid" id="column-name-desc">[[identifier_type]]</span>
<input class="dn" type="text" name="column_name" id="column_name" :value="identifier_type">
</span>
<div class="contents mv4" v-else-if="stage == 2">
<div class="field mb3" id="identifier_select">
<ul class="list pa0 ma0">
<li class="mb2" v-for="label, i in identifier_types">
<label class="pointer">
<input type="radio" name="identifier_type_field" :value="i" v-on:click.prevent="identifier_type = i" />
[[ label[0] ]] (eg <span class="code pa1 bg-light-gray underline-hover">[[ label[1] ]]</span>)
</label>
<label class="pointer">
<input type="radio" name="identifier_type_field" :value="i" v-on:click.prevent="identifier_type = i" />
[[ label[0] ]] (eg <span class="code pa1 bg-light-gray underline-hover">[[ label[1] ]]</span>)
</label>
</li>
</ul>
</div>
Expand All @@ -103,40 +109,41 @@

<div class="w-100 pa2 b--light-gray bb bw1 stage" id="stage-select-fields">
<h3 class="pa0 ma0 dib">
Step 3:
Step 3:
<span class="normal">Select data to add</span>
</h3>
<span class="fr contents-top" v-if="stage > 3">
<span class="dib code pa1 bg-light-gray" id="fields_to_add"></span>
</span>
<div class="contents mv4" v-else-if="stage == 3">
<ul class="list ma0 pa0" style="column-count: 3;">
<li>
<label>
<input type="checkbox" name="select_all_fields" v-on:click.prevent="selectAllFields">
<strong>Select all</strong>
</label>
</li>
<li v-for="field in fields" :key="field.id">
<label>
<input type="checkbox" name="fields" :value="field.id" v-model="fields_to_add">
[[field.name]]
</label>
</li>
<li>
<label>
<input type="checkbox" name="select_all_fields" v-on:click.prevent="selectAllFields">
<strong>Select all</strong>
</label>
</li>
<li v-for="field in fields" :key="field.id">
<label>
<input type="checkbox" name="fields" :value="field.id" v-model="fields_to_add">
[[field.name]]
</label>
</li>
</ul>

<div class="contents mv4">
<input class="button-reset bn pv3 ph4 b tc bg-animate bg-yellow dim near-black pointer br2-ns ml4-l" type="submit"
value="Add data to CSV" id='fetch_identifiers' v-on:click.prevent="getResults" />
<div id="result" v-if="progress !== null">
<p class="pa0 mv3 mh2" id="result-text">Creating file…</p>
<div id="progress-bar" class="bg-light-blue h2 mt4 mh2">
<div class="progress-bar-inner bg-blue h2 ph3 pv1 f6 tr white" v-bind:style="{ width: progress + '%' }" id="progress-bar-inner">[[progress]]%</div>
</div>
<input class="button-reset bn pv3 ph4 b tc bg-animate bg-yellow dim near-black pointer br2-ns ml4-l"
type="submit" value="Add data to CSV" id='fetch_identifiers' v-on:click.prevent="getResults" />
<div id="result" v-if="progress !== null">
<p class="pa0 mv3 mh2" id="result-text">Creating file…</p>
<div id="progress-bar" class="bg-light-blue h2 mt4 mh2">
<div class="progress-bar-inner bg-blue h2 ph3 pv1 f6 tr white" v-bind:style="{ width: progress + '%' }"
id="progress-bar-inner">[[progress]]%</div>
</div>
</div>
</div>
</div>

</div>

</div>
Expand All @@ -145,18 +152,18 @@
<div class="content w-100 w-30-l fl-l pa3 bg-light-gray f5" id="privacy">
<h3 class="pa0 ma0 header-font">Privacy</h3>
<p>Your file will not leave your computer, but the organsiation identifiers from
the column you select will be sent to the findthatcharity server in order to
the column you select will be sent to the findthatcharity server in order to
retrieve the information. No other data is sent to the server.</p>
<p>It could therefore be possible to reconstruct the organisations contained
in your file. <strong>You should think carefully before using this tool
for any personal or sensitive information.</strong></p>
for any personal or sensitive information.</strong></p>
</div>
{% endblock %}

{% block bodyscripts %}
<script type="text/javascript">
const PROPERTIES_URL = {{ request.build_absolute_uri(url('propose_properties'))|tojson }};
const EXTEND_URL = {{ request.build_absolute_uri(url('reconcile'))|tojson }};
const PROPERTIES_URL = {{ (request.build_absolute_uri(url('api-1.0:propose_properties')) + '?type=Organization') | tojson }};
const EXTEND_URL = {{ request.build_absolute_uri(url('api-1.0:reconcile_entities')) | tojson }};
</script>
<script src='{{ static("js/csv.js") }}' type="text/javascript"></script>
{% endblock %}
13 changes: 13 additions & 0 deletions addtocsv/static/js/csv.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ function openSaveFileDialog(data, filename, mimetype) {

}

function extractProperties(orgRecord) {
// properties are stored as a list of dicts, with a single key/value pair
// // We don't neeed to extract the key, so we just return the value
return Object.fromEntries(Object.entries(orgRecord).map(
([k, v]) => [k, Object.values(v).map((v) => Object.values(v)).join("; ")]
));
}

var app = new Vue({
el: '#addtocsv',
data: {
Expand Down Expand Up @@ -166,6 +174,11 @@ var app = new Vue({
})
}))
.then(data => Object.assign({}, ...data))
.then(data => Object.fromEntries(Object.entries(data).map(
([orgid, properties]) => (
[orgid, extractProperties(properties)]
)
)))
.then(new_data => openSaveFileDialog(
Papa.unparse({
fields: component.csv_results.meta.fields.concat(component.fields_to_add),
Expand Down
45 changes: 34 additions & 11 deletions charity/api/charities.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from datetime import date
from datetime import date as Date
from datetime import timedelta
from typing import Optional

from django.http import Http404
from django.shortcuts import get_list_or_404, get_object_or_404
Expand All @@ -14,16 +16,16 @@

class CharityResult(Schema):
success: bool = True
error: str = None
error: Optional[str] = None
params: dict = {}
result: CharityOut = None
result: Optional[CharityOut] = None


class CharityFinancialResult(Schema):
success: bool = True
error: str = None
error: Optional[str] = None
params: dict = {}
result: CharityFinancialOut = None
result: Optional[CharityFinancialOut] = None


api = Router(tags=["Charities"])
Expand Down Expand Up @@ -78,23 +80,44 @@ def get_charity_finance_latest(request, charity_id: str):
"/{charity_id}/financial/{date}",
response={200: CharityFinancialResult, 404: ResultError},
)
def get_charity_finance_by_date(request, charity_id: str, date: date):
def get_charity_finance_by_date(request, charity_id: str, date: Date):
charity_id = regno_to_orgid(charity_id)
try:
charity = get_object_or_404(Charity, id=charity_id)
financial_years = get_list_or_404(
CharityFinancial,
charity=charity,

# try using financial year start date
financial_year = charity.financial.filter(
fyend__gte=date,
fystart__lte=date,
)
).first()

# financial year end exact match
if not financial_year:
financial_year = charity.financial.filter(
fyend=date,
).first()

# financial year end is after or equal to the date
if not financial_year:
next_year = date + timedelta(days=365)
financial_year = (
charity.financial.filter(
fyend__gte=date,
fyend__lt=next_year,
)
.order_by("fyend")
.first()
)

if not financial_year:
raise Http404("No CharityFinancial matches the given query.")
return {
"error": None,
"params": {
"charity_id": charity_id,
"date": date,
},
"result": financial_years[0],
"result": financial_year,
}
except Http404 as e:
return 404, {
Expand Down
Loading

0 comments on commit 6a9cbb3

Please sign in to comment.