OntologyManager: create_ontology_world() allow loading from an SQL ba…

…ckend file

.github/workflows/new-pycram-ci.yml

@@ -55,10 +55,6 @@ jobs:
           source /opt/ros/overlay_ws/devel/setup.bash 
           roslaunch pycram ik_and_description.launch &
 
-      - name: Verify Java installation
-        run: |
-          java -version
-
       - name: Install python dependencies
         run: |
           pip3 install --upgrade pip --root-user-action=ignore

src/pycram/ontology/ontology.py

@@ -3,6 +3,7 @@
 import inspect
 import itertools
 import logging
+import os.path
 
 from pathlib import Path
 from typing import Callable, Dict, List, Optional, Type, Tuple, Union
@@ -20,7 +21,8 @@
 from ..helper import Singleton
 from ..designator import DesignatorDescription, ObjectDesignatorDescription
 
-from ..ontology.ontology_common import OntologyConceptHolderStore, OntologyConceptHolder
+from ..ontology.ontology_common import (OntologyConceptHolderStore, OntologyConceptHolder,
+                                        ONTOLOGY_SQL_BACKEND_FILE_EXTENSION)
 
 SOMA_HOME_ONTOLOGY_IRI = "http://www.ease-crc.org/ont/SOMA-HOME.owl"
 SOMA_ONTOLOGY_IRI = "http://www.ease-crc.org/ont/SOMA.owl"
@@ -65,17 +67,23 @@ def __init__(self, main_ontology_iri: Optional[str] = None, ontology_search_path
         # Ref: https://owlready2.readthedocs.io/en/latest/world.html
         self.main_ontology_world: Optional[OntologyWorld] = None
 
-        # Ontology IRI (Internationalized Resource Identifier), either a URL to a remote OWL file or the full
-        # name path of a local one
+        #: Ontology IRI (Internationalized Resource Identifier), either a URL to a remote OWL file or the full name path of a local one
+        # Ref: https://owlready2.readthedocs.io/en/latest/onto.html
         self.main_ontology_iri: str = main_ontology_iri if main_ontology_iri else SOMA_HOME_ONTOLOGY_IRI
 
         #: Namespace of the main ontology
         self.main_ontology_namespace: Optional[Namespace] = None
 
-        # Create the main ontology world with parallelized file parsing enabled
-        self.main_ontology_world = self.create_ontology_world(use_global_default_world=use_global_default_world)
+        # Create the main ontology world holding triples, of which a sqlite3 file path, of same name with `main_ontology` &
+        # at the same folder with `main_ontology_iri` (if it is a local abosulte path), is automatically registered as cache of the world
+        self.main_ontology_world = self.create_ontology_world(
+            sql_backend_filename=os.path.join(self.get_main_ontology_dir(),
+                                              f"{Path(self.main_ontology_iri).stem}{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}"),
+            use_global_default_world=use_global_default_world)
 
         # Load ontologies from `main_ontology_iri` to `main_ontology_world`
+        # If `main_ontology_iri` is a remote URL, Owlready2 first searches for a local copy of the OWL file (from `onto_path`),
+        # if not found, tries to download it from the Internet.
         self.main_ontology, self.main_ontology_namespace = self.load_ontology(self.main_ontology_iri)
         if self.main_ontology.loaded:
             self.soma = self.ontologies.get(SOMA_ONTOLOGY_NAMESPACE)
@@ -128,18 +136,55 @@ def print_ontology_property(ontology_property: Property):
         rospy.loginfo("-------------------")
 
     @staticmethod
-    def create_ontology_world(use_global_default_world: bool = False) -> OntologyWorld:
+    def get_default_ontology_search_path() -> Optional[str]:
+        """
+        Get the first ontology search path from owlready2.onto_path
+
+        :return: the path to the ontology search path if existing, otherwise None
+        """
+        if onto_path:
+            return onto_path[0]
+        else:
+            rospy.logerr("No ontology search path has been configured!")
+            return None
+
+    def get_main_ontology_dir(self) -> Optional[str]:
+        """
+        Get path to the directory of :attr:`main_ontology_iri` if it is a local absolute path,
+        otherwise path to the default ontology search directory
+
+        :return: the path to the directory of the main ontology IRI
+        """
+        return os.path.dirname(self.main_ontology_iri) if os.path.isabs(
+            self.main_ontology_iri) else self.get_default_ontology_search_path()
+
+    @staticmethod
+    def create_ontology_world(use_global_default_world: bool = False,
+                              sql_backend_filename: Optional[str] = None) -> OntologyWorld:
         """
         Either reuse the owlready2-provided global default ontology world or create a new one
 
         :param use_global_default_world: whether or not using the owlready2-provided global default persistent world
+        :param sql_backend_filename: a full file path (no need to already exist) being used as SQL backend for the ontology world. If None, memory is used instead
         :return: owlready2-provided global default ontology world or a newly created ontology world
         """
         world = default_world
+        sql_backend_path_valid = sql_backend_filename and os.path.isabs(sql_backend_filename)
+        sql_backend_name = sql_backend_filename if sql_backend_path_valid else "memory"
         if use_global_default_world:
-            world.set_backend(exclusive=False, enable_thread_parallelism=True)
+            # Reuse default world
+            if sql_backend_path_valid:
+                world.set_backend(filename=sql_backend_filename, exclusive=False, enable_thread_parallelism=True)
+            else:
+                world.set_backend(exclusive=False, enable_thread_parallelism=True)
+            rospy.loginfo(f"Using global default ontology world with SQL backend: {sql_backend_name}")
         else:
-            world = OntologyWorld(exclusive=False, enable_thread_parallelism=True)
+            # Create a new world with parallelized file parsing enabled
+            if sql_backend_path_valid:
+                world = OntologyWorld(filename=sql_backend_filename, exclusive=False, enable_thread_parallelism=True)
+            else:
+                world = OntologyWorld(exclusive=False, enable_thread_parallelism=True)
+            rospy.loginfo(f"Created a new ontology world with SQL backend: {sql_backend_name}")
         return world
 
     def load_ontology(self, ontology_iri: str) -> tuple[Optional[Ontology], Optional[Namespace]]:
@@ -221,7 +266,7 @@ def browse_ontologies(ontology: Ontology,
                     func(sub_onto, **kwargs)
                     break
 
-    def save(self, target_filename: str = "", overwrite: bool = False) -> bool:
+    def save(self, target_filename: Optional[str] = None, overwrite: bool = False) -> bool:
         """
         Save :attr:`main_ontology` to a file on disk, also caching :attr:`main_ontology_world` to a sqlite3 file
 
@@ -231,9 +276,9 @@ def save(self, target_filename: str = "", overwrite: bool = False) -> bool:
         """
 
         # Save ontologies to OWL
-        is_current_ontology_local = Path(self.main_ontology_iri).exists()
+        is_current_ontology_local = os.path.isfile(self.main_ontology_iri)
         current_ontology_filename = self.main_ontology_iri if is_current_ontology_local \
-            else f"{Path(self.main_ontology_world.filename).parent.absolute()}/{Path(self.main_ontology_iri).stem}.owl"
+            else f"{self.get_main_ontology_dir()}/{Path(self.main_ontology_iri).name}"
         save_to_same_file = is_current_ontology_local and (target_filename == current_ontology_filename)
         if save_to_same_file and not overwrite:
             rospy.logerr(
@@ -249,13 +294,12 @@ def save(self, target_filename: str = "", overwrite: bool = False) -> bool:
 
             # Commit the whole graph data of the current ontology world, saving it into SQLite3, to be reused the next time
             # the ontologies are loaded
-            main_ontology_sql_filename = f"{onto_path[0]}/{Path(save_filename).stem}.sqlite3"
-            self.main_ontology_world.save(file=main_ontology_sql_filename)
-            if Path(main_ontology_sql_filename).is_file():
+            main_ontology_sql_filename = self.main_ontology_world.filename
+            self.main_ontology_world.save()
+            if os.path.isfile(main_ontology_sql_filename):
                 rospy.loginfo(
                     f"Main ontology world for {self.main_ontology.name} has been cached and saved to SQL: {main_ontology_sql_filename}")
-            else:
-                rospy.logwarn(f"Failed caching main ontology world for {self.main_ontology.name} to SQL")
+            #else: it could be using memory cache as SQL backend
             return True
 
     def create_ontology_concept_class(self, class_name: str,
@@ -707,9 +751,10 @@ def create_rule_transitivity(self, ontology_concept_class_name: str,
 
     def reason(self, world: OntologyWorld = None, use_pellet_reasoner: bool = True) -> bool:
         """
-        Run the reasoning with Pellet or HermiT reasoner on :attr:`main_ontology`, the two currently supported by owlready2
+        Run the reasoning on a given ontology world or :attr:`main_ontology_world` with Pellet or HermiT reasoner,
+        the two currently supported by owlready2
         - By default, the reasoning works on `owlready2.default_world`
-        - The reasoning also automatically save ontologies to a temporary sqlite3 file
+        - The reasoning also automatically save ontologies (to either in-memory cache or a temporary sqlite3 file)
         Ref:
         - https://owlready2.readthedocs.io/en/latest/reasoning.html
         - https://owlready2.readthedocs.io/en/latest/rule.html

src/pycram/ontology/ontology_common.py

@@ -10,6 +10,9 @@
 
 from owlready2 import issubclass, Thing
 
+ONTOLOGY_SQL_BACKEND_FILE_EXTENSION = ".sqlite3"
+ONTOLOGY_OWL_FILE_EXTENSION = ".owl"
+
 
 class OntologyConceptHolderStore(object, metaclass=Singleton):
     """

test/test_ontology.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os.path
 import unittest
 import logging
 from pathlib import Path
@@ -15,8 +16,9 @@
     owlready2 = None
     rospy.logwarn("Could not import owlready2, Ontology unit-tests could not run!")
 
-from pycram.ontology.ontology import OntologyManager, SOMA_ONTOLOGY_IRI
-from pycram.ontology.ontology_common import OntologyConceptHolderStore, OntologyConceptHolder
+from pycram.ontology.ontology import OntologyManager, SOMA_HOME_ONTOLOGY_IRI, SOMA_ONTOLOGY_IRI
+from pycram.ontology.ontology_common import (OntologyConceptHolderStore, OntologyConceptHolder,
+                                             ONTOLOGY_SQL_BACKEND_FILE_EXTENSION, ONTOLOGY_OWL_FILE_EXTENSION)
 
 
 class TestOntologyManager(unittest.TestCase):
@@ -39,9 +41,9 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
-        save_dir = Path(f"{Path.home()}/ontologies")
-        owl_filepath = f"{save_dir}/{Path(cls.ontology_manager.main_ontology_iri).stem}.owl"
-        sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}.sqlite3"
+        save_dir = cls.ontology_manager.get_main_ontology_dir()
+        owl_filepath = f"{save_dir}/{Path(cls.ontology_manager.main_ontology_iri).stem}{ONTOLOGY_OWL_FILE_EXTENSION}"
+        sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}"
         os.remove(owl_filepath)
         cls.remove_sql_file(sql_filepath)
 
@@ -59,15 +61,34 @@ def test_ontology_manager(self):
             self.assertTrue(self.ontology_manager.initialized())
 
     def test_ontology_world(self):
-        if owlready2:
-            self.assertIsNotNone(self.ontology_manager.main_ontology_world)
-            extra_world = self.ontology_manager.create_ontology_world()
-            self.assertIsNotNone(extra_world)
-            extra_world_sql_filepath = f"{onto_path[0]}/extra_world.sqlite3"
-            extra_world.save(file=extra_world_sql_filepath)
-            world_saved_to_sql = Path(extra_world_sql_filepath).is_file()
-            self.remove_sql_file(extra_world_sql_filepath)
-            self.assertTrue(world_saved_to_sql)
+        if not owlready2:
+            return
+        # Main ontology world as the global default world
+        main_world = self.ontology_manager.main_ontology_world
+        self.assertIsNotNone(main_world)
+        self.assertTrue(main_world is owlready2.default_world)
+
+        # Extra world with memory backend
+        extra_memory_world = self.ontology_manager.create_ontology_world(use_global_default_world=False)
+        self.assertIsNotNone(extra_memory_world)
+        self.assertTrue(extra_memory_world != owlready2.default_world)
+
+        # Extra world with SQL backend from a non-existing SQL file
+        extra_world_sql_filename = f"{self.ontology_manager.get_main_ontology_dir()}/extra_world{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}"
+        extra_sql_world = self.ontology_manager.create_ontology_world(use_global_default_world=False,
+                                                                      sql_backend_filename=extra_world_sql_filename)
+        self.assertIsNotNone(extra_sql_world)
+        # Save it at [extra_world_sql_filename]
+        extra_sql_world.save()
+        self.assertTrue(os.path.isfile(extra_world_sql_filename))
+
+        # Extra world with SQL backend from an existing SQL file
+        extra_sql_world_2 = self.ontology_manager.create_ontology_world(use_global_default_world=False,
+                                                                        sql_backend_filename=extra_world_sql_filename)
+        self.assertIsNotNone(extra_sql_world_2)
+
+        # Remove SQL file finally
+        self.remove_sql_file(extra_world_sql_filename)
 
     def test_ontology_concept_holder(self):
         if not owlready2:
@@ -83,7 +104,8 @@ def test_loaded_ontologies(self):
             return
         self.assertIsNotNone(self.main_ontology)
         self.assertTrue(self.main_ontology.loaded)
-        if self.ontology_manager.main_ontology_iri is SOMA_ONTOLOGY_IRI:
+        if self.ontology_manager.main_ontology_iri is SOMA_ONTOLOGY_IRI or \
+                self.ontology_manager.main_ontology_iri is SOMA_HOME_ONTOLOGY_IRI:
             self.assertIsNotNone(self.soma)
             self.assertTrue(self.soma.loaded)
             self.assertIsNotNone(self.dul)
@@ -124,17 +146,18 @@ def test_ontology_triple_classes_dynamic_creation(self):
         PLACEABLE_ON_PREDICATE_NAME = "placeable_on"
         HOLD_OBJ_PREDICATE_NAME = "hold_obj"
         self.assertTrue(
-            self.ontology_manager.create_ontology_triple_classes(ontology_subject_parent_class=self.soma.Container if self.soma else None,
-                                                                 subject_class_name="OntologyPlaceHolderObject",
-                                                                 ontology_object_parent_class=self.dul.PhysicalObject
-                                                                 if self.dul else None,
-                                                                 object_class_name="OntologyHandheldObject",
-                                                                 predicate_class_name=PLACEABLE_ON_PREDICATE_NAME,
-                                                                 inverse_predicate_class_name=HOLD_OBJ_PREDICATE_NAME,
-                                                                 ontology_property_parent_class=self.soma.affordsBearer
-                                                                 if self.soma else None,
-                                                                 ontology_inverse_property_parent_class=self.soma.isBearerAffordedBy
-                                                                 if self.soma else None))
+            self.ontology_manager.create_ontology_triple_classes(
+                ontology_subject_parent_class=self.soma.Container if self.soma else None,
+                subject_class_name="OntologyPlaceHolderObject",
+                ontology_object_parent_class=self.dul.PhysicalObject
+                if self.dul else None,
+                object_class_name="OntologyHandheldObject",
+                predicate_class_name=PLACEABLE_ON_PREDICATE_NAME,
+                inverse_predicate_class_name=HOLD_OBJ_PREDICATE_NAME,
+                ontology_property_parent_class=self.soma.affordsBearer
+                if self.soma else None,
+                ontology_inverse_property_parent_class=self.soma.isBearerAffordedBy
+                if self.soma else None))
 
         def create_ontology_handheld_object_designator(object_name: str, ontology_parent_class: Type[owlready2.Thing]):
             return self.ontology_manager.create_ontology_linked_designator(object_name=object_name,
@@ -179,17 +202,18 @@ def test_ontology_class_destruction(self):
         self.assertIsNone(self.ontology_manager.get_ontology_class(concept_class_name))
         self.assertFalse(OntologyConceptHolderStore().get_ontology_concepts_by_class(dynamic_ontology_concept_class))
 
+    #@unittest.skip("owlready2 reasoning requires Java runtime, which is available only for CI running on master/dev")
     def test_ontology_reasoning(self):
         if not owlready2:
             return
 
-        REASONING_TEST_ONTOLOGY_IRI = "reasoning_test.owl"
+        REASONING_TEST_ONTOLOGY_IRI = f"reasoning_test{ONTOLOGY_OWL_FILE_EXTENSION}"
         ENTITY_CONCEPT_NAME = "Entity"
         CAN_TRANSPORT_PREDICATE_NAME = "can_transport"
         TRANSPORTABLE_BY_PREDICATE_NAME = "transportable_by"
         CORESIDE_PREDICATE_NAME = "coreside"
 
-        # Create a test world for reasoning
+        # Create a test world (with memory SQL backend) for reasoning
         reasoning_world = self.ontology_manager.create_ontology_world()
         reasoning_ontology = reasoning_world.get_ontology(REASONING_TEST_ONTOLOGY_IRI)
 
@@ -249,7 +273,6 @@ def coresidents(a: reasoning_ontology.Entity, b: reasoning_ontology.Entity) -> b
 
             # Reason on [reasoning_world]
             self.ontology_manager.reason(world=reasoning_world)
-            self.remove_sql_file(sql_filepath=f"{onto_path[0]}/{Path(reasoning_ontology.name).stem}.sqlite3")
 
             # Test reflexivity
             for entity in entities:
@@ -270,11 +293,10 @@ def test_ontology_save(self):
         if not owlready2:
             return
 
-        save_dir = Path(f"{Path.home()}/ontologies")
-        save_dir.mkdir(parents=True, exist_ok=True)
-        owl_filepath = f"{save_dir}/{Path(self.ontology_manager.main_ontology_iri).stem}.owl"
-        sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}.sqlite3"
-        self.ontology_manager.save(owl_filepath)
+        save_dir = self.ontology_manager.get_main_ontology_dir()
+        owl_filepath = f"{save_dir}/{Path(self.ontology_manager.main_ontology_iri).stem}{ONTOLOGY_OWL_FILE_EXTENSION}"
+        sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}"
+        self.assertTrue(self.ontology_manager.save(owl_filepath))
         self.assertTrue(Path(owl_filepath).is_file())
         self.assertTrue(Path(sql_filepath).is_file())