Refactor kcidb.orm module for improved readability and maintainability

Extract parts of kcidb.orm module into separate modules for improved maintainability and readability. Specifically, we refactored the module by creating two new modules - kcidb.orm.data for SCHEMA and dependencies, and kcidb.orm.query for Pattern and dependencies. Updated all relevant references to the new modules. No changes in behavior. Signed-off-by: Abhishek Kumar <[email protected]>
kernelci · Mar 29, 2023 · a217ae7 · a217ae7
1 parent 3caa42f
commit a217ae7
Show file tree

Hide file tree

Showing 19 changed files with 1,447 additions and 1,412 deletions.
diff --git a/kcidb/__init__.py b/kcidb/__init__.py
@@ -314,7 +314,7 @@ def notify_main():
     oo_client = oo.Client(db.Client(args.database))
     pattern_set = set()
     for pattern_string in args.pattern_strings:
-        pattern_set |= orm.Pattern.parse(pattern_string)
+        pattern_set |= orm.query.Pattern.parse(pattern_string)
     for notification in monitor.match(oo_client.query(pattern_set)):
         sys.stdout.write(
             notification.render().
@@ -349,9 +349,9 @@ def ingest_main():
         data = io.SCHEMA.upgrade(data, copy=False)
         # Record patterns matching the loaded objects and all their parents
         pattern_set = set()
-        for pattern in orm.Pattern.from_io(data):
+        for pattern in orm.query.Pattern.from_io(data):
             # TODO Avoid formatting and parsing
-            pattern_set |= orm.Pattern.parse(repr(pattern) + "<*#")
+            pattern_set |= orm.query.Pattern.parse(repr(pattern) + "<*#")
         LOGGER.debug("Notification patterns: %r", pattern_set)
         # Reset the OO cache
         oo_client.reset_cache()

diff --git a/kcidb/db/__init__.py b/kcidb/db/__init__.py
@@ -338,15 +338,16 @@ def oo_query(self, pattern_set):
         Query raw object-oriented data from the database.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
             objects of the corresponding type.
         """
         assert LIGHT_ASSERTS or self.is_initialized()
         assert isinstance(pattern_set, set)
-        assert all(isinstance(r, kcidb.orm.Pattern) for r in pattern_set)
+        assert all(isinstance(r, kcidb.orm.query.Pattern)
+                   for r in pattern_set)
         LOGGER.debug("OO Query: %r", pattern_set)
         return self.driver.oo_query(pattern_set)
 

diff --git a/kcidb/db/abstract.py b/kcidb/db/abstract.py
@@ -191,14 +191,15 @@ def oo_query(self, pattern_set):
         The database must be initialized.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
             objects of the corresponding type.
         """
         assert isinstance(pattern_set, set)
-        assert all(isinstance(r, orm.Pattern) for r in pattern_set)
+        assert all(isinstance(r, orm.query.Pattern)
+                   for r in pattern_set)
         assert self.is_initialized()
 
     @abstractmethod

diff --git a/kcidb/db/bigquery/v04_00.py b/kcidb/db/bigquery/v04_00.py
@@ -834,13 +834,13 @@ def _oo_query_render(cls, pattern):
         Render a pattern matching raw OO data into a query.
 
         Args:
-            pattern:    The pattern (instance of kcidb.orm.Pattern) to
+            pattern:    The pattern (instance of kcidb.orm.query.Pattern) to
                         render.
 
         Returns:
             The SQL query string and the query parameters.
         """
-        assert isinstance(pattern, orm.Pattern)
+        assert isinstance(pattern, orm.query.Pattern)
         obj_type = pattern.obj_type
         type_query_string = cls.OO_QUERIES[obj_type.name]
         if pattern.obj_id_set:
@@ -905,18 +905,19 @@ def oo_query(self, pattern_set):
         Query raw object-oriented data from the database.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
             objects of the corresponding type.
         """
         assert isinstance(pattern_set, set)
-        assert all(isinstance(r, orm.Pattern) for r in pattern_set)
+        assert all(isinstance(r, orm.query.Pattern)
+                   for r in pattern_set)
 
         # Render all queries for each type
         obj_type_queries = {}
-        for obj_type in orm.SCHEMA.types.values():
+        for obj_type in orm.data.SCHEMA.types.values():
             for pattern in pattern_set:
                 # TODO: Avoid adding the same patterns multiple times
                 if pattern.obj_type == obj_type:
@@ -949,7 +950,7 @@ def oo_query(self, pattern_set):
                 for row in job.result()
             ]
 
-        assert LIGHT_ASSERTS or orm.SCHEMA.is_valid(objs)
+        assert LIGHT_ASSERTS or orm.data.SCHEMA.is_valid(objs)
         return objs
 
     @classmethod

diff --git a/kcidb/db/postgresql/v04_00.py b/kcidb/db/postgresql/v04_00.py
@@ -677,13 +677,13 @@ def _oo_query_render(cls, pattern):
         Render a pattern for raw OO data into a query.
 
         Args:
-            pattern:    The pattern (instance of kcidb.orm.Pattern) to
+            pattern:    The pattern (instance of kcidb.orm.query.Pattern) to
                         render.
 
         Returns:
             The SQL query string and the query parameters.
         """
-        assert isinstance(pattern, orm.Pattern)
+        assert isinstance(pattern, orm.query.Pattern)
         obj_type = pattern.obj_type
         type_query_string = cls.OO_QUERIES[obj_type.name]["statement"]
         if pattern.obj_id_set:
@@ -760,18 +760,18 @@ def oo_query(self, pattern_set):
         Query raw object-oriented data from the database.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
             objects of the corresponding type.
         """
         assert isinstance(pattern_set, set)
-        assert all(isinstance(r, orm.Pattern) for r in pattern_set)
+        assert all(isinstance(r, orm.query.Pattern) for r in pattern_set)
 
         # Render all queries for each type
         obj_type_queries = {}
-        for obj_type in orm.SCHEMA.types.values():
+        for obj_type in orm.data.SCHEMA.types.values():
             for pattern in pattern_set:
                 if pattern.obj_type == obj_type:
                     if obj_type not in obj_type_queries:
@@ -791,7 +791,7 @@ def oo_query(self, pattern_set):
                     oo_query["schema"].unpack_iter(cursor, drop_null=False)
                 )
 
-        assert LIGHT_ASSERTS or orm.SCHEMA.is_valid(objs)
+        assert LIGHT_ASSERTS or orm.data.SCHEMA.is_valid(objs)
         return objs
 
     def load(self, data):

diff --git a/kcidb/db/schematic.py b/kcidb/db/schematic.py
@@ -296,14 +296,15 @@ def oo_query(self, pattern_set):
         The database must be initialized.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
             objects of the corresponding type.
         """
         assert isinstance(pattern_set, set)
-        assert all(isinstance(r, orm.Pattern) for r in pattern_set)
+        assert all(isinstance(r, orm.query.Pattern)
+                   for r in pattern_set)
 
     @abstractmethod
     def load(self, data):
@@ -583,14 +584,15 @@ def oo_query(self, pattern_set):
         The database must be initialized.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
             objects of the corresponding type.
         """
         assert isinstance(pattern_set, set)
-        assert all(isinstance(r, orm.Pattern) for r in pattern_set)
+        assert all(isinstance(r, orm.query.Pattern)
+                   for r in pattern_set)
         assert self.is_initialized()
         return self.schema.oo_query(pattern_set)
 

diff --git a/kcidb/db/sqlite/v04_00.py b/kcidb/db/sqlite/v04_00.py
@@ -646,13 +646,13 @@ def _oo_query_render(cls, pattern):
         Render a pattern for raw OO data into a query.
 
         Args:
-            pattern:    The pattern (instance of kcidb.orm.Pattern) to
+            pattern:    The pattern (instance of kcidb.orm.query.Pattern) to
                         render.
 
         Returns:
             The SQL query string and the query parameters.
         """
-        assert isinstance(pattern, orm.Pattern)
+        assert isinstance(pattern, orm.query.Pattern)
         obj_type = pattern.obj_type
         type_query_string = cls.OO_QUERIES[obj_type.name]["statement"]
         if pattern.obj_id_set:
@@ -713,18 +713,18 @@ def oo_query(self, pattern_set):
         Query raw object-oriented data from the database.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
             objects of the corresponding type.
         """
         assert isinstance(pattern_set, set)
-        assert all(isinstance(r, orm.Pattern) for r in pattern_set)
+        assert all(isinstance(r, orm.query.Pattern) for r in pattern_set)
 
         # Render all queries for each type
         obj_type_queries = {}
-        for obj_type in orm.SCHEMA.types.values():
+        for obj_type in orm.data.SCHEMA.types.values():
             for pattern in pattern_set:
                 # TODO: Avoid adding the same patterns multiple times
                 if pattern.obj_type == obj_type:
@@ -765,7 +765,7 @@ def oo_query(self, pattern_set):
             finally:
                 cursor.close()
 
-        assert LIGHT_ASSERTS or orm.SCHEMA.is_valid(objs)
+        assert LIGHT_ASSERTS or orm.data.SCHEMA.is_valid(objs)
         return objs
 
     def load(self, data):

diff --git a/kcidb/monitor/test_output.py b/kcidb/monitor/test_output.py
@@ -42,7 +42,7 @@ def test_min():
             },
         ],
     })
-    oo_data = oo_client.query(orm.Pattern.parse(">*#"))
+    oo_data = oo_client.query(orm.query.Pattern.parse(">*#"))
 
     notification_message = NotificationMessage(
         to=["[email protected]", "[email protected]"],
@@ -119,7 +119,9 @@ def test_subject_and_body_length():
             },
         ]
     })
-    revision = oo_client.query(orm.Pattern.parse(">revision#"))["revision"][0]
+    revision = oo_client.query(
+        orm.query.Pattern.parse(">revision#")
+    )["revision"][0]
 
     # test under-limit length subject/body are left intact in the email
     notification_message = NotificationMessage(
@@ -235,7 +237,9 @@ def test_subject_invalid_character():
             },
         ]
     })
-    revision = oo_client.query(orm.Pattern.parse(">revision#"))["revision"][0]
+    revision = oo_client.query(
+        orm.query.Pattern.parse(">revision#")
+    )["revision"][0]
 
     # test invalid character in subject are replaced
     # by the uncertainty sign character in the email

diff --git a/kcidb/mq/__init__.py b/kcidb/mq/__init__.py
@@ -503,7 +503,7 @@ class ORMPatternPublisher(Publisher):
 
     def encode_data(self, data):
         """
-        Encode a set of kcidb.orm.Pattern objects, into message data.
+        Encode a set of kcidb.orm.query.Pattern objects, into message data.
 
         Args:
             data:   The set to encode.
@@ -515,7 +515,7 @@ def encode_data(self, data):
             An exception in case data encoding failed.
         """
         assert isinstance(data, set)
-        assert all(isinstance(pattern, kcidb.orm.Pattern)
+        assert all(isinstance(pattern, kcidb.orm.query.Pattern)
                    for pattern in data)
         return "".join(
             repr(pattern) + "\n" for pattern in data
@@ -527,22 +527,22 @@ class ORMPatternSubscriber(Subscriber):
 
     def decode_data(self, message_data):
         """
-        Decode message data to extract kcidb.orm.Pattern objects.
+        Decode message data to extract kcidb.orm.query.Pattern objects.
 
         Args:
             message_data:   The message data from the message queue
                             ("data" field of pubsub.types.PubsubMessage) to be
                             decoded.
 
         Returns
-            The decoded set of kcidb.orm.Pattern objects.
+            The decoded set of kcidb.orm.query.Pattern objects.
 
         Raises:
             An exception in case data decoding failed.
         """
         pattern_set = set()
         for line in message_data.decode().splitlines():
-            pattern_set |= kcidb.orm.Pattern.parse(line)
+            pattern_set |= kcidb.orm.query.Pattern.parse(line)
         return pattern_set
 
 
@@ -818,7 +818,7 @@ def pattern_publisher_main():
     parser = PublisherArgumentParser("ORM patterns", description=description)
     parser.subparsers["publish"].add_argument(
         '--pattern-help',
-        action=kcidb.orm.PatternHelpAction,
+        action=kcidb.orm.query.PatternHelpAction,
         help='Print pattern string documentation and exit.'
     )
     args = parser.parse_args()
@@ -831,7 +831,7 @@ def pattern_publisher_main():
         pattern_set = set()
         for line_idx, line in enumerate(sys.stdin):
             try:
-                pattern_set |= kcidb.orm.Pattern.parse(line)
+                pattern_set |= kcidb.orm.query.Pattern.parse(line)
             except Exception as exc:
                 raise Exception(
                     f"Failed parsing ORM pattern on line {line_idx + 1}: "

diff --git a/kcidb/oo/__init__.py b/kcidb/oo/__init__.py
@@ -8,7 +8,9 @@
 from cached_property import cached_property
 import kcidb.db
 from kcidb.misc import LIGHT_ASSERTS
-from kcidb.orm import Type, SCHEMA, Pattern, Source
+from kcidb.orm import Source
+from kcidb.orm.query import Pattern
+from kcidb.orm.data import Type, SCHEMA
 
 
 class Object:
@@ -23,7 +25,7 @@ def __init__(self, client, type, data):
             client:     The object-oriented database client to query for
                         references.
             type:       The type of represented object.
-                        Instance of kcidb.orm.Type.
+                        Instance of kcidb.orm.data.Type.
             data:       The raw data of the object to represent.
         """
         assert isinstance(client, Client)
@@ -38,7 +40,7 @@ def get_type(self):
         Retrieve the object's type.
 
         Returns:
-            The object's type, an instance of kcidb.orm.Type.
+            The object's type, an instance of kcidb.orm.data.Type.
         """
         return self._type
 
@@ -743,7 +745,7 @@ def query(self, pattern_set):
         Retrieve objects specified via a pattern list.
 
         Args:
-            pattern_set:    A set of patterns ("kcidb.orm.Pattern"
+            pattern_set:    A set of patterns ("kcidb.orm.query.Pattern"
                             instances) matching objects to fetch.
         Returns:
             A dictionary of object type names and lists containing retrieved
@@ -831,7 +833,7 @@ def query_main():
     db_client = kcidb.db.Client(args.database)
     pattern_set = set()
     for pattern_string in args.pattern_strings:
-        pattern_set |= kcidb.orm.Pattern.parse(pattern_string)
+        pattern_set |= Pattern.parse(pattern_string)
     kcidb.misc.json_dump(
         db_client.oo_query(pattern_set),
         sys.stdout, indent=args.indent, seq=args.seq