From 0b30b4bd8b7aef567e9a0b69199f713a18102998 Mon Sep 17 00:00:00 2001
From: User <8444617+pnoll1@users.noreply.github.com>
Date: Sat, 23 Mar 2024 22:48:05 +0000
Subject: [PATCH] clean data with z geometry

---
 osmand_osm/osm/oa.py    |  4 ++++
 osmand_osm/osm/tests.py | 47 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/osmand_osm/osm/oa.py b/osmand_osm/osm/oa.py
index bfa4818..9217a14 100644
--- a/osmand_osm/osm/oa.py
+++ b/osmand_osm/osm/oa.py
@@ -250,6 +250,10 @@ def merge_oa(self, db_name):
                 logging.info(source.table + ' Insert ' + str(cur.rowcount))
             except psycopg.errors.InvalidParameterValue as error:
                 logging.error(f'{source.table} errored on merge_oa insert: {error}')
+                conn.rollback()
+                # get rid of z geometry
+                cur.execute(f'alter table {source.table_temp} alter column wkb_geometry type geometry(point, 4326) using ST_Force2D(wkb_geometry);')
+                cur.execute(f'insert into {source.table} select * from {source.table_temp} on conflict do nothing')
             # get rid of temp table
             cur.execute(f'drop table {source.table_temp}')
             conn.commit()
diff --git a/osmand_osm/osm/tests.py b/osmand_osm/osm/tests.py
index 50a18e4..32efaac 100644
--- a/osmand_osm/osm/tests.py
+++ b/osmand_osm/osm/tests.py
@@ -168,7 +168,6 @@ def test_filter_data(self):
         data = self.cur.fetchall()
         self.assertEqual(len(data), 1)
 
-
     def test_merge_oa(self):
         '''       
         test additional data inserted from temp table when table already exists with data
@@ -252,6 +251,52 @@ def test_merge_oa_first_run(self):
         self.assertIn((2, None, '115', 'NW 41st ST', None, None, None, None, '98107', '87d28792bee6b164', '0101000020E6100000DD7C23BAE7965EC0FC3ACB87FBD34740'), data)
         self.assertIn((1, None, '119', 'NW 41st ST', None, None, None, None, '98107', 'e8605a496593386e', '0101000020E61000003BEFB556EA965EC03EFC4685FBD34740'), data)
 
+    def test_merge_oa_zdata(self):
+        '''
+        test that addresses with z data cleaned and inserted into db
+        '''
+        # cleanup postgres table
+        self.cur.execute("drop table if exists aa_merge_oa_addresses_city;")
+        self.cur.execute("drop table if exists aa_merge_oa_addresses_city_temp;")
+        self.conn.commit()
+        # load data into postgres
+        self.cur.execute("create table aa_merge_oa_addresses_city_temp (ogc_fid integer NOT NULL, \
+                id character varying, number character varying, street character varying, unit character varying, \
+                city character varying, district character varying, region character varying, \
+                postcode character varying, hash character varying, wkb_geometry public.geometry(PointZ, 4326));")
+        self.cur.execute("insert into aa_merge_oa_addresses_city_temp (ogc_fid, number, street, postcode, hash, wkb_geometry) \
+                values (%s, %s, %s, %s, %s, ST_GEOMFromText(%s, 4326))" \
+                , (1, '119', 'NW 41st ST', '98107', 'e8605a496593386e', 'POINT(-122.3580529 47.6561133 0.0)'))
+        self.cur.execute("insert into aa_merge_oa_addresses_city_temp (ogc_fid, number, street, postcode, hash, wkb_geometry) \
+                values (%s, %s, %s, %s, %s, ST_GEOMFromText(%s, 4326))" \
+                , (2, '115', 'NW 41st ST', '98107', '87d28792bee6b164', 'POINT(-122.3578935 47.6561136 0.0)'))
+        self.cur.execute("insert into aa_merge_oa_addresses_city_temp (ogc_fid, number, street, postcode, hash, wkb_geometry) \
+                values (%s, %s, %s, %s, %s, ST_GEOMFromText(%s, 4326))" \
+                , (3, '34', 'BARR ROAD', '98643', '2a7175361ea6f935', 'POINT(-123.6361676 46.32554 0.0)'))
+        # create table
+        self.cur.execute("create table aa_merge_oa_addresses_city (ogc_fid integer NOT NULL, \
+                  id character varying, number character varying, street character varying, \
+                  unit character varying, city character varying, district character varying, \
+                  region character varying, postcode character varying, hash character varying, \
+                  wkb_geometry public.geometry(Point, 4326));")
+        self.cur.execute('alter table aa_merge_oa_addresses_city add constraint aa_merge_oa_addresses_city_unique \
+                unique nulls not distinct (number, street, unit, wkb_geometry)')
+        self.conn.commit()
+        working_area = oa.WorkingArea('aa')
+        working_area.master_list = [oa.Source(Path('aa/merge-oa-addresses-city.geojson'))]
+        working_area.merge_oa(DB_NAME)
+        self.cur.execute("select * from aa_merge_oa_addresses_city;")
+        data = self.cur.fetchall()
+        # check that good data exists
+        self.assertIn((2, None, '115', 'NW 41st ST', None, None, None, None, '98107', \
+                '87d28792bee6b164', '0101000020E6100000DD7C23BAE7965EC0FC3ACB87FBD34740'), data)
+        self.assertIn((1, None, '119', 'NW 41st ST', None, None, None, None, '98107', \
+                'e8605a496593386e', '0101000020E61000003BEFB556EA965EC03EFC4685FBD34740'), data)
+        # check if z still in geom data
+        self.cur.execute("select ST_Z(wkb_geometry) from aa_merge_oa_addresses_city where number='34';")
+        data = self.cur.fetchall()
+        self.assertEqual(data[0][0], None)
+
     def test_output_osm_ids(self):
         '''
         make sure address ids in output files don't overlap