Update to current Django/python/YAML.

As part of this, also includes: * Adding GitHub Actions * Fixing flake8 errors * Switching to pylibmc
mysociety · Oct 10, 2024 · 85a222f · 85a222f
1 parent 4402f81
commit 85a222f
Show file tree

Hide file tree

Showing 17 changed files with 138 additions and 130 deletions.
diff --git a/.github/workflows/default.yml b/.github/workflows/default.yml
@@ -0,0 +1,62 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    name: Test MapIt
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgis/postgis:13-3.2
+        env:
+          POSTGRES_PASSWORD: mapit
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      PGHOST: localhost
+      PGUSER: postgres
+      PGPASSWORD: mapit
+      CFLAGS: "-O0"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        thing-to-test: [flake8, py310]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        set -xe
+        python -VV
+        python -m site
+        pip install tox tox-gh-actions
+
+    - name: Install GDAL
+      run: sudo apt-get install gdal-bin
+
+    - name: Set up config
+      run: |
+        createdb mapit
+        psql -d mapit -c 'create extension postgis;'
+        sed -r -e "s,(MAPIT_DB_USER:) 'mapit',\\1 'postgres'," conf/general.yml-example > conf/general.yml
+
+    - name: Run tests
+      run: tox
+      env:
+        THING_TO_TEST: ${{ matrix.thing-to-test }}
diff --git a/.travis.yml b/.travis.yml
diff --git a/bin/boundaries.py b/bin/boundaries.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-
-
 import errno
 from mock import Mock, patch # noqa
 import requests
@@ -23,10 +21,11 @@
 
 with open(os.path.join(
         os.path.dirname(__file__), '..', 'conf', 'general.yml')) as f:
-    config = yaml.load(f)
+    config = yaml.load(f, Loader=yaml.SafeLoader)
 
 CACHE_VISITED = set()
 
+
 # Suggested by http://stackoverflow.com/q/600268/223092
 def mkdir_p(path):
     """Create a directory (and parents if necessary) like mkdir -p
@@ -123,7 +122,7 @@ def get_remote(query_xml, filename):
     url = config['OVERPASS_SERVER']
     r = requests.get(url, params={'data': query_xml})
     r.raise_for_status()
-    data = r.content
+    data = r.text
     with open(filename, "w") as fp:
         fp.write(data)
     return data
@@ -381,13 +380,10 @@ def xml_wrapping():
         >>> print(
         ...     etree.tostring(OSMElement.xml_wrapping(), encoding='unicode',
         ...     pretty_print=True), end='') # doctest: +NORMALIZE_WHITESPACE
-        <osm generator="mySociety Boundary Extractor" version="0.6">
-          <note>The data included in this document is from www.openstreetmap.org.
-          It has there been collected by a large group of contributors.
-          For individual attribution of each item please refer to
-          https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
+        <osm version="0.6" generator="mySociety Boundary Extractor">
+          <note>The data included in this document is from www.openstreetmap.org. It has there been collected by a large group of contributors. For individual attribution of each item please refer to https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
         </osm>
-        """
+        """  # noqa: E501
 
         osm = etree.Element("osm", attrib={"version": "0.6",
                                            "generator": "mySociety Boundary Extractor"})
@@ -499,14 +495,11 @@ def to_xml(self, parent_element=None, include_node_dependencies=False):
         ...    full_result,
         ...    encoding='unicode',
         ...    pretty_print=True), end='') # doctest: +NORMALIZE_WHITESPACE
-        <osm generator="mySociety Boundary Extractor" version="0.6">
-          <note>The data included in this document is from www.openstreetmap.org.
-          It has there been collected by a large group of contributors.
-          For individual attribution of each item please refer to
-          https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
+        <osm version="0.6" generator="mySociety Boundary Extractor">
+          <note>The data included in this document is from www.openstreetmap.org. It has there been collected by a large group of contributors. For individual attribution of each item please refer to https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
           <node id="1234" lat="51.2" lon="-0.2"/>
         </osm>
-        """
+        """  # noqa: E501
 
         if parent_element is None:
             parent_element = OSMElement.xml_wrapping()
@@ -886,11 +879,8 @@ def to_xml(self, parent_element=None, include_node_dependencies=False):
         ...     result,
         ...     encoding='unicode',
         ...     pretty_print=True), end='') # doctest: +NORMALIZE_WHITESPACE
-        <osm generator="mySociety Boundary Extractor" version="0.6">
-          <note>The data included in this document is from www.openstreetmap.org.
-          It has there been collected by a large group of contributors.
-          For individual attribution of each item please refer to
-          https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
+        <osm version="0.6" generator="mySociety Boundary Extractor">
+          <note>The data included in this document is from www.openstreetmap.org. It has there been collected by a large group of contributors. For individual attribution of each item please refer to https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
           <way id="76543">
             <nd ref="12"/>
             <nd ref="13"/>
@@ -900,7 +890,7 @@ def to_xml(self, parent_element=None, include_node_dependencies=False):
             <tag k="boundary" v="administrative"/>
           </way>
         </osm>
-        """
+        """  # noqa: E501
 
         if parent_element is None:
             parent_element = OSMElement.xml_wrapping()
@@ -1179,13 +1169,13 @@ def to_xml(self, parent_element=None, include_node_dependencies=False):
         >>> print(etree.tostring(xe, encoding='unicode', pretty_print=True), end='')
         <example>
           <relation id="98765">
-            <member ref="76542" role="" type="node"/>
-            <member ref="76543" role="" type="way"/>
-            <member ref="98764" role="" type="relation"/>
-            <member ref="98764" role="" type="way"/>
-            <member ref="76545" role="inner" type="way"/>
-            <member ref="87654" role="inner" type="relation"/>
-            <member ref="76546" role="" type="way"/>
+            <member type="node" ref="76542" role=""/>
+            <member type="way" ref="76543" role=""/>
+            <member type="relation" ref="98764" role=""/>
+            <member type="way" ref="98764" role=""/>
+            <member type="way" ref="76545" role="inner"/>
+            <member type="relation" ref="87654" role="inner"/>
+            <member type="way" ref="76546" role=""/>
             <tag k="admin_level" v="2"/>
             <tag k="boundary" v="administrative"/>
           </relation>
@@ -1200,13 +1190,13 @@ def to_xml(self, parent_element=None, include_node_dependencies=False):
         <example-with-nodes>
           <node id="76542" lat="52" lon="0.3"/>
           <relation id="98765">
-            <member ref="76542" role="" type="node"/>
-            <member ref="76543" role="" type="way"/>
-            <member ref="98764" role="" type="relation"/>
-            <member ref="98764" role="" type="way"/>
-            <member ref="76545" role="inner" type="way"/>
-            <member ref="87654" role="inner" type="relation"/>
-            <member ref="76546" role="" type="way"/>
+            <member type="node" ref="76542" role=""/>
+            <member type="way" ref="76543" role=""/>
+            <member type="relation" ref="98764" role=""/>
+            <member type="way" ref="98764" role=""/>
+            <member type="way" ref="76545" role="inner"/>
+            <member type="relation" ref="87654" role="inner"/>
+            <member type="way" ref="76546" role=""/>
             <tag k="admin_level" v="2"/>
             <tag k="boundary" v="administrative"/>
           </relation>
@@ -1220,19 +1210,16 @@ def to_xml(self, parent_element=None, include_node_dependencies=False):
         ...     result,
         ...     encoding='unicode',
         ...     pretty_print=True), end='') # doctest: +NORMALIZE_WHITESPACE
-        <osm generator="mySociety Boundary Extractor" version="0.6">
-          <note>The data included in this document is from www.openstreetmap.org.
-          It has there been collected by a large group of contributors.
-          For individual attribution of each item please refer to
-          https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
+        <osm version="0.6" generator="mySociety Boundary Extractor">
+          <note>The data included in this document is from www.openstreetmap.org. It has there been collected by a large group of contributors. For individual attribution of each item please refer to https://api.openstreetmap.org/api/0.6/[node|way|relation]/#id/history</note>
           <relation id="98765">
-            <member ref="76542" role="" type="node"/>
-            <member ref="76543" role="" type="way"/>
-            <member ref="98764" role="" type="relation"/>
-            <member ref="98764" role="" type="way"/>
-            <member ref="76545" role="inner" type="way"/>
-            <member ref="87654" role="inner" type="relation"/>
-            <member ref="76546" role="" type="way"/>
+            <member type="node" ref="76542" role=""/>
+            <member type="way" ref="76543" role=""/>
+            <member type="relation" ref="98764" role=""/>
+            <member type="way" ref="98764" role=""/>
+            <member type="way" ref="76545" role="inner"/>
+            <member type="relation" ref="87654" role="inner"/>
+            <member type="way" ref="76546" role=""/>
             <tag k="admin_level" v="2"/>
             <tag k="boundary" v="administrative"/>
           </relation>
@@ -1248,7 +1235,7 @@ def to_xml(self, parent_element=None, include_node_dependencies=False):
         Traceback (most recent call last):
           ...
         Exception: Trying out output a missing node %s as XML
-        """
+        """  # noqa: E501
 
         if parent_element is None:
             parent_element = OSMElement.xml_wrapping()
@@ -2235,8 +2222,8 @@ def fake_requests_get(url, params):
         'tests',
         'overpass-responses',
         '{type}-{id}.xml'.format(type=osm_type, id=osm_id))
-    with open(filename, 'rb') as f:
-        return Mock(content=f.read())
+    with open(filename, 'r') as f:
+        return Mock(text=f.read())
 
 
 if __name__ == "__main__":

diff --git a/bin/check-capitals-in-global.py b/bin/check-capitals-in-global.py
@@ -11,8 +11,6 @@
 #   pip install requests
 #   pip install SPARQLWrapper
 
-
-
 from collections import defaultdict
 import json
 import re

diff --git a/bin/generate_kml.py b/bin/generate_kml.py
@@ -1,8 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-
-
 import sys
 from lxml import etree
 from shapely.geometry import Polygon

diff --git a/bin/get-boundaries-by-admin-level.py b/bin/get-boundaries-by-admin-level.py
@@ -4,8 +4,6 @@
 # This script fetches all administrative and political boundaries from
 # OpenStreetMap and writes them out as KML.
 
-
-
 import os
 import re
 import sys

diff --git a/bin/scrape_admin_boundaries_tables.py b/bin/scrape_admin_boundaries_tables.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-
-
 import re
 import sys
 
@@ -32,7 +30,7 @@ def get_country_name(s):
 
 
 def make_missing_none(s):
-    if re.search('(?uis)^\s*N/A\s*$', s):
+    if re.search(r'(?uis)^\s*N/A\s*$', s):
         return None
     else:
         return s

diff --git a/bin/subdivide-new-cache.py b/bin/subdivide-new-cache.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-
-
 import os
 import re
 import sys

diff --git a/mapit_global/management/commands/mapit_global_import.py b/mapit_global/management/commands/mapit_global_import.py
@@ -27,7 +27,7 @@
 # from django.contrib.gis.utils import LayerMapping
 from django.contrib.gis.gdal import DataSource
 from django.contrib.gis.db.models import Collect
-from django.utils.encoding import smart_str, smart_text
+from django.utils.encoding import smart_str
 
 import requests
 import shapely.wkb
@@ -39,7 +39,7 @@
 
 def make_missing_none(s):
     """If s is empty (considering Unicode spaces) return None, else s"""
-    if re.search('(?uis)^\s*$', s):
+    if re.search(r'(?uis)^\s*$', s):
         return None
     else:
         return s
@@ -125,7 +125,7 @@ def handle_label(self, directory_name, **options):
 
         os.chdir(directory_name)
 
-        mapit_type_glob = smart_text("[A-Z0-9][A-Z0-9][A-Z0-9]")
+        mapit_type_glob = smart_str("[A-Z0-9][A-Z0-9][A-Z0-9]")
 
         if not glob(mapit_type_glob):
             raise Exception(

diff --git a/mapit_global/mapit_settings.py b/mapit_global/mapit_settings.py
@@ -48,10 +48,10 @@
     CACHE_MIDDLEWARE_SECONDS = 0
 else:
     try:
-        import memcache  # noqa
+        import pylibmc  # noqa
         CACHES = {
             'default': {
-                'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+                'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
                 'LOCATION': '127.0.0.1:11211',
                 'TIMEOUT': 86400,
             }
@@ -120,10 +120,6 @@
 # to load the internationalization machinery.
 USE_I18N = True
 
-# If you set this to False, Django will not format dates, numbers and
-# calendars according to the current locale.
-USE_L10N = True
-
 # If you set this to False, Django will not use timezone-aware datetimes.
 USE_TZ = False
 

diff --git a/mapit_global/settings.py b/mapit_global/settings.py
@@ -4,6 +4,8 @@
 
 # Update a couple of things to suit our changes
 
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
 # Insert our project app before mapit so that the templates take precedence
 INSTALLED_APPS.insert(INSTALLED_APPS.index('mapit'), 'mapit_global')
 ROOT_URLCONF = 'mapit_global.urls'