From 86d51e489586d7984fb1a3d99c84396eb770ccb2 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Thu, 14 Sep 2023 00:15:17 -0600 Subject: [PATCH] Fake place creation + geofabrik testing functionality nominatim-docker-test.yml: added environment variable for the geofabrik query url with GitHub actions secrets. we will see if this works in this commit. docker-compose.yml: commented out healthcheck because it isn't currently doing anything useful to delay the start. Should remove completely once docker commit is working. TestNominatim.py: added functionality for geofabrik calls with GH actions secrets. Added creation of a fake place to use for testing of get_filtered_place. Added test comparing geofabrik and nominatim hard-coded calls (these may not be possible from within the container so might be removed) Added test for get_filtered_place in clean_and_resample, which passes in the fake place created above. Modified test_get_json_reverse to only check the display name for now. --- .github/workflows/nominatim-docker-test.yml | 4 +- emission/individual_tests/TestNominatim.py | 78 +++++++++++++------- emission/integrationTests/docker-compose.yml | 13 ++-- 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/.github/workflows/nominatim-docker-test.yml b/.github/workflows/nominatim-docker-test.yml index 882d5fd86..a373ebd4b 100644 --- a/.github/workflows/nominatim-docker-test.yml +++ b/.github/workflows/nominatim-docker-test.yml @@ -26,5 +26,7 @@ jobs: - name: Workflow test run: echo Smoke test - - name: Test nominatim + - name: Test nominatim + env: + GEOFABRIK_QUERY_URL: https://geocoding.geofabrik.de/${{secrets.GEOFABRIK_API}} run: docker-compose -f emission/integrationTests/docker-compose.yml up --exit-code-from web-server diff --git a/emission/individual_tests/TestNominatim.py b/emission/individual_tests/TestNominatim.py index a7e638178..e1792a886 100644 --- a/emission/individual_tests/TestNominatim.py +++ b/emission/individual_tests/TestNominatim.py @@ -7,23 +7,51 @@ from builtins import * import unittest import unittest.mock as mock +# import overpy import os from emission.core.wrapper.trip_old import Coordinate +import requests +import emission.core.wrapper.entry as ecwe +import emission.core.wrapper.wrapperbase as ecww import emission.net.ext_service.geocoder.nominatim as eco +import emission.analysis.intake.cleaning.clean_and_resample as clean + #temporarily sets NOMINATIM_QUERY_URL to the environment variable for testing. NOMINATIM_QUERY_URL_env = os.environ.get("NOMINATIM_QUERY_URL", "") NOMINATIM_QUERY_URL = NOMINATIM_QUERY_URL_env if NOMINATIM_QUERY_URL_env != "" else eco.NOMINATIM_QUERY_URL -print("query URL in TestNominatim:", NOMINATIM_QUERY_URL) +GEOFABRIK_QUERY_URL = os.environ.get("GEOFABRIK_QUERY_URL", "") + +#Creates a fake place in Rhode Island to use for testing. +fake_id = "rhodeislander" +key = "segmentation/raw_place" +write_ts = 1694344333 +data = {'source': 'FakeTripGenerator','location': {'type': 'Point', 'coordinates': [-71.4128343, 41.8239891]}} +fake_place = ecwe.Entry.create_fake_entry(fake_id, key, data, write_ts) class NominatimTest(unittest.TestCase): maxDiff = None + + #basic query to check that both nominatim and geofabrik are acting as drop-ins for eachother. + def test_geofabrik_and_nominatim(self): + nominatim_result = requests.get(NOMINATIM_QUERY_URL + "/reverse?lat=41.8239891&lon=-71.4128343&format=json") + geofabrik_result = requests.get(GEOFABRIK_QUERY_URL + "/reverse?lat=41.8239891&lon=-71.4128343&format=json") + self.assertEqual(nominatim_result, geofabrik_result) + +#checks the display name generated by get_filtered place in clean_and_resample.py, which creates a cleaned place from the fake place +# and reverse geocodes with the coordinates. + def test_get_filtered_place(self): + raw_result = ecww.WrapperBase.__getattr__(clean.get_filtered_place(fake_place), "data") + actual_result = ecww.WrapperBase.__getattr__(raw_result, "display_name") + expected_result = "Fulton Street, Providence" + self.assertEqual(expected_result, actual_result) + def test_make_url_geo(self): expected_result = NOMINATIM_QUERY_URL + "/search?q=Providence%2C+Rhode+Island&format=json" actual_result = eco.Geocoder.make_url_geo("Providence, Rhode Island") self.assertEqual(expected_result, actual_result) -#we ignore the place_id because it is an internal Nominatim identifier + #we ignore the place_id because it is an internal Nominatim identifier def test_get_json_geo(self): expected_result = [{'place_id': mock.ANY, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 121496393, 'boundingbox': ['41.8237547', '41.8243153', '-71.4132816', '-71.4125278'], 'lat': '41.824034499999996', 'lon': '-71.41290469687814', 'display_name': 'Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States', 'class': 'amenity', 'type': 'townhall', 'importance': 1.25001}] actual_result = eco.Geocoder.get_json_geo("Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States") @@ -44,34 +72,34 @@ def test_make_url_reverse(self): actual_result = (eco.Geocoder.make_url_reverse(41.8239891, -71.4128343)) self.assertEqual(expected_result, actual_result) -#we ignore the place_id because it is an internal Nominatim identifier + #tested result was modified to only look at the name returned with the coordinates, rather than the entire dictionary. def test_get_json_reverse(self): - expected_result = {'place_id': mock.ANY, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'way', 'osm_id': 121496393, 'lat': '41.824034499999996', 'lon': '-71.41290469687814', 'display_name': 'Providence City Hall, Fulton Street, Downtown, Providence, Providence County, 02903, United States', 'address': {'amenity': 'Providence City Hall', 'road': 'Fulton Street', 'neighbourhood': 'Downtown', 'city': 'Providence', 'county': 'Providence County', 'postcode': '02903', 'country': 'United States', 'country_code': 'us'}, 'boundingbox': ['41.8237547', '41.8243153', '-71.4132816', '-71.4125278']} - actual_docker = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343) - # actual_nominatim = "httsp://nominatim.openstreetmap.org/" - self.assertEqual(expected_result, actual_docker) - # self.assertEqual(actual_docker, actual_nominatim) + expected_result = "Providence City Hall" + actual_result = eco.Geocoder.get_json_reverse(41.8239891, -71.4128343)["display_name"].split(",")[0] + self.assertEqual(expected_result, actual_result) def test_reverse_geocode(self): expected_result = "Portugal Parkway, Fox Point, Providence, Providence County, 02906, United States" actual_result = eco.Geocoder.reverse_geocode(41.8174476, -71.3903767) self.assertEqual(expected_result, actual_result) - -#this test was written with the intention of using a ground truth file. Once a fake trip is generated in Rhode island, this section will be modified. - # def test_display_name(self): - # nominatim_reverse_query = NOMINATIM_QUERY_URL + "/reverse?" - # params = { - # "lat" : 41.831174, - # "lon" : -71.414907, - # "format" : "json" - # } - # encoded_params = urllib.parse.urlencode(params) - # url = nominatim_reverse_query + encoded_params - # request = urllib.request.Request(url) - # response = urllib.request.urlopen(request) - # parsed_response = json.loads(response.read()) - # actual_result = str(parsed_response.get("display_name")) - # expected_result = "Rhode Island State Capitol Building, 82, Smith Street, Downtown, Providence, Providence County, 02903, United States" - # self.assertEqual(expected_result, actual_result) + + #a hard-coded nominatim call to compare with our container. + def test_nominatim_api(self): + nominatim_url = "http://nominatim.openstreetmap.org/reverse?lat=41.832942092439694&lon=-71.41558148857203&format=json" + nominatim_result_raw = requests.get(nominatim_url) + nominatim_result = nominatim_result_raw.json()['display_name'] + # NOMINATIM_QUERY_URL = eco.NOMINATIM_QUERY_URL + docker_result = eco.Geocoder.reverse_geocode(41.832942092439694, -71.41558148857203) + print("HEYYY") + print(docker_result) + print("hey2") + print(nominatim_result) + self.assertEqual(nominatim_result, docker_result) + + # def test_overpass_api(self): + # api = overpy.Overpass() + # result = api.query("""way["name"="Gielgenstraße"](50.7,7.1,50.8,7.25);out;""") + + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/emission/integrationTests/docker-compose.yml b/emission/integrationTests/docker-compose.yml index d4c28f981..4ea9ddde1 100644 --- a/emission/integrationTests/docker-compose.yml +++ b/emission/integrationTests/docker-compose.yml @@ -44,14 +44,15 @@ services: environment: - PBF_URL=https://download.geofabrik.de/north-america/us/rhode-island-latest.osm.pbf - REPLICATION_URL=https://download.geofabrik.de/north-america/us/rhode-island-updates/ - - IMPORT_WIKIPEDIA=false + # - IMPORT_WIKIPEDIA=false ports: - "8080:8080" - healthcheck: - test: ["CMD", "curl", "-v", "http://localhost:8080"] - interval: 1m - timeout: 15s - retries: 3 + #healthcheck currently commented out, since it isn't necessary and keeps going the entire time the container is running. + # healthcheck: + # test: ["CMD", "curl", "-v", "http://localhost:8080"] + # interval: 1m + # timeout: 15s + # retries: 3 deploy: replicas: 1 restart_policy: