feat(postgres): add materialized views (#210)

odd_collector/adapters/postgresql/mappers/tables.py

@@ -39,13 +39,13 @@ def map_tables(
     for table in tables:
         logger.debug(f"Map table {table.table_name} {table.table_type}")
 
-        if table.table_type == "BASE TABLE":
+        if table.table_type == "r":
             entity = map_table(generator, table)
-        elif table.table_type == "VIEW":
+        elif table.table_type in ("v", "m"):
             entity = map_view(generator, table)
         else:
             logger.warning(
-                f"Parsing only [BASE_TABLE, VIEW]. Got unknown {table.table_type=} {table.oid=}"
+                f"Parsing only [BASE_TABLE, VIEW, MATERIALIZED_VIEW]. Got unknown {table.table_type=} {table.oid=}"
             )
             continue
 

odd_collector/adapters/postgresql/models.py

@@ -84,7 +84,7 @@ class Column:
     is_primary: bool = False
 
     @property
-    def odd_metadata(self) -> dict:
+    def odd_metadata(self):
         return omit(
             self.__dict__,
             {

odd_collector/adapters/postgresql/repository.py

@@ -18,7 +18,7 @@
 @dataclass(frozen=True)
 class ConnectionParams:
     host: str
-    port: str
+    port: int
     dbname: str
     user: str
     password: str
@@ -93,7 +93,7 @@ def pks_query(self):
                 join pg_catalog.pg_class c on c.oid = a.attrelid
                 join pg_catalog.pg_namespace n on n.oid = c.relnamespace
             where i.indisprimary
-            and c.relkind in ('r', 'v')
+            and c.relkind in ('r', 'v', 'm')
             and a.attnum > 0
             and n.nspname not like 'pg_temp_%'
             and n.nspname not in ('pg_toast', 'pg_internal', 'catalog_history', 'pg_catalog', 'information_schema')
@@ -102,11 +102,12 @@ def pks_query(self):
     @property
     def tables_query(self):
         return """
-            select c.oid
+            select
+                c.oid
                 , it.table_catalog
-                , it.table_schema
-                , it.table_name
-                , it.table_type
+                , n.nspname as table_schema
+                , c.relname as table_name
+                , c.relkind as table_type
                 , it.self_referencing_column_name
                 , it.reference_generation
                 , it.user_defined_type_catalog
@@ -115,7 +116,7 @@ def tables_query(self):
                 , it.is_insertable_into
                 , it.is_typed
                 , it.commit_action
-                , iw.view_definition
+                , pg_get_viewdef(c.oid, true)            as view_definition
                 , iw.check_option                        as view_check_option
                 , iw.is_updatable                        as view_is_updatable
                 , iw.is_insertable_into                  as view_is_insertable_into
@@ -127,10 +128,9 @@ def tables_query(self):
                 , pg_catalog.obj_description(c.oid)      as description
             from pg_catalog.pg_class c
                     join pg_catalog.pg_namespace n on n.oid = c.relnamespace
-                    join information_schema.tables it on it.table_schema = n.nspname and it.table_name = c.relname
+                    left join information_schema.tables it on it.table_schema = n.nspname and it.table_name = c.relname
                     left join information_schema.views iw on iw.table_schema = n.nspname and iw.table_name = c.relname
-            where c.relkind in ('r', 'v')
-            and it.table_type in ('BASE TABLE', 'VIEW')
+            where c.relkind in ('r', 'v', 'm')
             and n.nspname not like 'pg_temp_%'
             and n.nspname not in ('pg_toast', 'pg_internal', 'catalog_history', 'pg_catalog', 'information_schema')
             order by n.nspname, c.relname
@@ -140,15 +140,15 @@ def tables_query(self):
     def columns_query(self):
         return """
             select
-                a.attrelid  
+                a.attrelid
                 , ic.table_catalog
-                , ic.table_schema
-                , ic.table_name
-                , ic.column_name
+                , nspname as table_schema
+                , c.relname as table_name
+                , a.attname as column_name
                 , ic.ordinal_position
                 , ic.column_default
                 , ic.is_nullable
-                , ic.data_type
+                , t.typname as data_type
                 , ic.character_maximum_length
                 , ic.character_octet_length
                 , ic.numeric_precision
@@ -190,9 +190,11 @@ def columns_query(self):
             from pg_catalog.pg_attribute a
                     join pg_catalog.pg_class c on c.oid = a.attrelid
                     join pg_catalog.pg_namespace n on n.oid = c.relnamespace
-                    join information_schema.columns ic on ic.table_schema = n.nspname and ic.table_name = c.relname and
-                                                        ic.ordinal_position = a.attnum
-            where c.relkind in ('r', 'v')
+                    join pg_catalog.pg_type t on t.oid = a.atttypid
+                    left join information_schema.columns ic on ic.table_schema = n.nspname
+                        and ic.table_name = c.relname
+                        and ic.ordinal_position = a.attnum
+            where c.relkind in ('r', 'v', 'm')
             and a.attnum > 0
             and n.nspname not like 'pg_temp_%'
             and n.nspname not in ('pg_toast', 'pg_internal', 'catalog_history', 'pg_catalog', 'information_schema')

tests/integration/test_postgres.py

@@ -47,6 +47,12 @@
 FROM VIEW_ONE v1, other_schema.VIEW_THREE v3
 """
 
+create_materialized_view = """
+CREATE MATERIALIZED VIEW materialized_view AS
+SELECT *
+FROM TABLE_ONE
+"""
+
 from odd_collector.adapters.postgresql.adapter import Adapter
 from odd_collector.domain.plugin import PostgreSQLPlugin
 
@@ -66,6 +72,7 @@ def test_postgres():
             connection.exec_driver_sql(create_schema)
             connection.exec_driver_sql(create_view_three)
             connection.exec_driver_sql(create_view_four)
+            connection.exec_driver_sql(create_materialized_view)
 
         config = PostgreSQLPlugin(
             type="postgresql",
@@ -83,15 +90,15 @@ def test_postgres():
         )
         assert len(database_services) == 1
         database_service = database_services[0]
-        assert len(database_service.data_entity_group.entities_list) == 5
+        assert len(database_service.data_entity_group.entities_list) == 6
 
         tables = find_by_type(data_entities, DataEntityType.TABLE)
         assert len(tables) == 1
         table = tables[0]
         assert len(table.dataset.field_list) == 7
 
         views = find_by_type(data_entities, DataEntityType.VIEW)
-        assert len(views) == 4
+        assert len(views) == 5
         view_one = first(filter(lambda x: x.name == "view_one", views))
         assert len(view_one.dataset.field_list) == 7
 
@@ -111,3 +118,10 @@ def test_postgres():
         depends = view_four.data_transformer.inputs
         assert view_one.oddrn in depends
         assert view_three.oddrn in depends
+
+        mat_view = first(filter(lambda x: x.name == "materialized_view", views))
+        depends = mat_view.data_transformer.inputs
+        assert len(mat_view.dataset.field_list) == 7
+        assert table.oddrn in depends
+
+        assert data_entities.json()