Skip to content

Commit

Permalink
Fake place creation + geofabrik testing functionality
Browse files Browse the repository at this point in the history
nominatim-docker-test.yml: added environment variable for the geofabrik query url with GitHub actions secrets. we will see if this works in this commit.

docker-compose.yml: commented out healthcheck because it isn't currently doing anything useful to delay the start. Should remove completely once docker commit is working.

TestNominatim.py: added functionality for geofabrik calls with GH actions secrets.

Added creation of a fake place to use for testing of get_filtered_place.

Added test comparing geofabrik and nominatim hard-coded calls (these may not be possible from within the container so might be removed)

Added test for get_filtered_place in clean_and_resample, which passes in the fake place created above.

Modified test_get_json_reverse to only check the display name for now.
  • Loading branch information
nataliejschultz committed Sep 14, 2023
1 parent bdc70ae commit 86d51e4
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 32 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/nominatim-docker-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,7 @@ jobs:
- name: Workflow test
run: echo Smoke test

- name: Test nominatim
- name: Test nominatim
env:
GEOFABRIK_QUERY_URL: https://geocoding.geofabrik.de/${{secrets.GEOFABRIK_API}}
run: docker-compose -f emission/integrationTests/docker-compose.yml up --exit-code-from web-server
78 changes: 53 additions & 25 deletions emission/individual_tests/TestNominatim.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,51 @@
from builtins import *
import unittest
import unittest.mock as mock
# import overpy
import os
from emission.core.wrapper.trip_old import Coordinate
import requests
import emission.core.wrapper.entry as ecwe
import emission.core.wrapper.wrapperbase as ecww
import emission.net.ext_service.geocoder.nominatim as eco
import emission.analysis.intake.cleaning.clean_and_resample as clean


#temporarily sets NOMINATIM_QUERY_URL to the environment variable for testing.
NOMINATIM_QUERY_URL_env = os.environ.get("NOMINATIM_QUERY_URL", "")
NOMINATIM_QUERY_URL = NOMINATIM_QUERY_URL_env if NOMINATIM_QUERY_URL_env != "" else eco.NOMINATIM_QUERY_URL
print("query URL in TestNominatim:", NOMINATIM_QUERY_URL)
GEOFABRIK_QUERY_URL = os.environ.get("GEOFABRIK_QUERY_URL", "")

#Creates a fake place in Rhode Island to use for testing.
fake_id = "rhodeislander"
key = "segmentation/raw_place"
write_ts = 1694344333
data = {'source': 'FakeTripGenerator','location': {'type': 'Point', 'coordinates': [-71.4128343, 41.8239891]}}
fake_place = ecwe.Entry.create_fake_entry(fake_id, key, data, write_ts)

class NominatimTest(unittest.TestCase):
maxDiff = None

#basic query to check that both nominatim and geofabrik are acting as drop-ins for eachother.
def test_geofabrik_and_nominatim(self):
nominatim_result = requests.get(NOMINATIM_QUERY_URL + "/reverse?lat=41.8239891&lon=-71.4128343&format=json")
geofabrik_result = requests.get(GEOFABRIK_QUERY_URL + "/reverse?lat=41.8239891&lon=-71.4128343&format=json")
self.assertEqual(nominatim_result, geofabrik_result)

#checks the display name generated by get_filtered place in clean_and_resample.py, which creates a cleaned place from the fake place
# and reverse geocodes with the coordinates.
def test_get_filtered_place(self):
raw_result = ecww.WrapperBase.__getattr__(clean.get_filtered_place(fake_place), "data")
actual_result = ecww.WrapperBase.__getattr__(raw_result, "display_name")
expected_result = "Fulton Street, Providence"
self.assertEqual(expected_result, actual_result)

def test_make_url_geo(self):
expected_result = NOMINATIM_QUERY_URL + "/search?q=Providence%2C+Rhode+Island&format=json"
actual_result = eco.Geocoder.make_url_geo("Providence, Rhode Island")
self.assertEqual(expected_result, actual_result)

#we ignore the place_id because it is an internal Nominatim identifier
#we ignore the place_id because it is an internal Nominatim identifier
def test_get_json_geo(self):
expected_result = [{'place_id': mock.ANY, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 121496393, 'boundingbox': ['41.8237547', '41.8243153', '-71.4132816', '-71.4125278'], 'lat': '41.824034499999996', 'lon': '-71.41290469687814', 'display_name': 'Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States', 'class': 'amenity', 'type': 'townhall', 'importance': 1.25001}]
actual_result = eco.Geocoder.get_json_geo("Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States")
Expand All @@ -44,34 +72,34 @@ def test_make_url_reverse(self):
actual_result = (eco.Geocoder.make_url_reverse(41.8239891, -71.4128343))
self.assertEqual(expected_result, actual_result)

#we ignore the place_id because it is an internal Nominatim identifier
#tested result was modified to only look at the name returned with the coordinates, rather than the entire dictionary.
def test_get_json_reverse(self):
expected_result = {'place_id': mock.ANY, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 121496393, 'lat': '41.824034499999996', 'lon': '-71.41290469687814', 'display_name': 'Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States', 'address': {'amenity': 'Providence City Hall', 'road': 'Fulton Street', 'neighbourhood': 'Downtown', 'city': 'Providence', 'county': 'Providence County', 'postcode': '02903', 'country': 'United States', 'country_code': 'us'}, 'boundingbox': ['41.8237547', '41.8243153', '-71.4132816', '-71.4125278']}
actual_docker = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)
# actual_nominatim = "httsp://nominatim.openstreetmap.org/"
self.assertEqual(expected_result, actual_docker)
# self.assertEqual(actual_docker, actual_nominatim)
expected_result = "Providence City Hall"
actual_result = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)["display_name"].split(",")[0]
self.assertEqual(expected_result, actual_result)

def test_reverse_geocode(self):
expected_result = "Portugal Parkway, Fox Point, Providence, Providence County, 02906, United States"
actual_result = eco.Geocoder.reverse_geocode(41.8174476, -71.3903767)
self.assertEqual(expected_result, actual_result)

#this test was written with the intention of using a ground truth file. Once a fake trip is generated in Rhode island, this section will be modified.
# def test_display_name(self):
# nominatim_reverse_query = NOMINATIM_QUERY_URL + "/reverse?"
# params = {
# "lat" : 41.831174,
# "lon" : -71.414907,
# "format" : "json"
# }
# encoded_params = urllib.parse.urlencode(params)
# url = nominatim_reverse_query + encoded_params
# request = urllib.request.Request(url)
# response = urllib.request.urlopen(request)
# parsed_response = json.loads(response.read())
# actual_result = str(parsed_response.get("display_name"))
# expected_result = "Rhode Island State Capitol Building, 82, Smith Street, Downtown, Providence, Providence County, 02903, United States"
# self.assertEqual(expected_result, actual_result)

#a hard-coded nominatim call to compare with our container.
def test_nominatim_api(self):
nominatim_url = "http://nominatim.openstreetmap.org/reverse?lat=41.832942092439694&lon=-71.41558148857203&format=json"
nominatim_result_raw = requests.get(nominatim_url)
nominatim_result = nominatim_result_raw.json()['display_name']
# NOMINATIM_QUERY_URL = eco.NOMINATIM_QUERY_URL
docker_result = eco.Geocoder.reverse_geocode(41.832942092439694, -71.41558148857203)
print("HEYYY")
print(docker_result)
print("hey2")
print(nominatim_result)
self.assertEqual(nominatim_result, docker_result)

# def test_overpass_api(self):
# api = overpy.Overpass()
# result = api.query("""way["name"="Gielgenstraße"](50.7,7.1,50.8,7.25);out;""")


if __name__ == '__main__':
unittest.main()
13 changes: 7 additions & 6 deletions emission/integrationTests/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,15 @@ services:
environment:
- PBF_URL=https://download.geofabrik.de/north-america/us/rhode-island-latest.osm.pbf
- REPLICATION_URL=https://download.geofabrik.de/north-america/us/rhode-island-updates/
- IMPORT_WIKIPEDIA=false
# - IMPORT_WIKIPEDIA=false
ports:
- "8080:8080"
healthcheck:
test: ["CMD", "curl", "-v", "http://localhost:8080"]
interval: 1m
timeout: 15s
retries: 3
#healthcheck currently commented out, since it isn't necessary and keeps going the entire time the container is running.
# healthcheck:
# test: ["CMD", "curl", "-v", "http://localhost:8080"]
# interval: 1m
# timeout: 15s
# retries: 3
deploy:
replicas: 1
restart_policy:
Expand Down

0 comments on commit 86d51e4

Please sign in to comment.