From 61b699193f8118f80869c99e560bfe2763b9dd14 Mon Sep 17 00:00:00 2001 From: duc than Date: Tue, 11 Jun 2024 14:19:22 +0200 Subject: [PATCH] OntologyManager: create_ontology_world() allow loading from an SQL backend file --- .github/workflows/new-pycram-ci.yml | 4 -- src/pycram/ontology/ontology.py | 81 +++++++++++++++++------ src/pycram/ontology/ontology_common.py | 3 + test/test_ontology.py | 90 ++++++++++++++++---------- 4 files changed, 122 insertions(+), 56 deletions(-) diff --git a/.github/workflows/new-pycram-ci.yml b/.github/workflows/new-pycram-ci.yml index 827b49193..7ad8b1486 100644 --- a/.github/workflows/new-pycram-ci.yml +++ b/.github/workflows/new-pycram-ci.yml @@ -55,10 +55,6 @@ jobs: source /opt/ros/overlay_ws/devel/setup.bash roslaunch pycram ik_and_description.launch & - - name: Verify Java installation - run: | - java -version - - name: Install python dependencies run: | pip3 install --upgrade pip --root-user-action=ignore diff --git a/src/pycram/ontology/ontology.py b/src/pycram/ontology/ontology.py index de23c7da2..7acfefbb7 100644 --- a/src/pycram/ontology/ontology.py +++ b/src/pycram/ontology/ontology.py @@ -3,6 +3,7 @@ import inspect import itertools import logging +import os.path from pathlib import Path from typing import Callable, Dict, List, Optional, Type, Tuple, Union @@ -20,7 +21,8 @@ from ..helper import Singleton from ..designator import DesignatorDescription, ObjectDesignatorDescription -from ..ontology.ontology_common import OntologyConceptHolderStore, OntologyConceptHolder +from ..ontology.ontology_common import (OntologyConceptHolderStore, OntologyConceptHolder, + ONTOLOGY_SQL_BACKEND_FILE_EXTENSION) SOMA_HOME_ONTOLOGY_IRI = "http://www.ease-crc.org/ont/SOMA-HOME.owl" SOMA_ONTOLOGY_IRI = "http://www.ease-crc.org/ont/SOMA.owl" @@ -65,17 +67,23 @@ def __init__(self, main_ontology_iri: Optional[str] = None, ontology_search_path # Ref: https://owlready2.readthedocs.io/en/latest/world.html self.main_ontology_world: Optional[OntologyWorld] = None - # Ontology IRI (Internationalized Resource Identifier), either a URL to a remote OWL file or the full - # name path of a local one + #: Ontology IRI (Internationalized Resource Identifier), either a URL to a remote OWL file or the full name path of a local one + # Ref: https://owlready2.readthedocs.io/en/latest/onto.html self.main_ontology_iri: str = main_ontology_iri if main_ontology_iri else SOMA_HOME_ONTOLOGY_IRI #: Namespace of the main ontology self.main_ontology_namespace: Optional[Namespace] = None - # Create the main ontology world with parallelized file parsing enabled - self.main_ontology_world = self.create_ontology_world(use_global_default_world=use_global_default_world) + # Create the main ontology world holding triples, of which a sqlite3 file path, of same name with `main_ontology` & + # at the same folder with `main_ontology_iri` (if it is a local abosulte path), is automatically registered as cache of the world + self.main_ontology_world = self.create_ontology_world( + sql_backend_filename=os.path.join(self.get_main_ontology_dir(), + f"{Path(self.main_ontology_iri).stem}{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}"), + use_global_default_world=use_global_default_world) # Load ontologies from `main_ontology_iri` to `main_ontology_world` + # If `main_ontology_iri` is a remote URL, Owlready2 first searches for a local copy of the OWL file (from `onto_path`), + # if not found, tries to download it from the Internet. self.main_ontology, self.main_ontology_namespace = self.load_ontology(self.main_ontology_iri) if self.main_ontology.loaded: self.soma = self.ontologies.get(SOMA_ONTOLOGY_NAMESPACE) @@ -128,18 +136,55 @@ def print_ontology_property(ontology_property: Property): rospy.loginfo("-------------------") @staticmethod - def create_ontology_world(use_global_default_world: bool = False) -> OntologyWorld: + def get_default_ontology_search_path() -> Optional[str]: + """ + Get the first ontology search path from owlready2.onto_path + + :return: the path to the ontology search path if existing, otherwise None + """ + if onto_path: + return onto_path[0] + else: + rospy.logerr("No ontology search path has been configured!") + return None + + def get_main_ontology_dir(self) -> Optional[str]: + """ + Get path to the directory of :attr:`main_ontology_iri` if it is a local absolute path, + otherwise path to the default ontology search directory + + :return: the path to the directory of the main ontology IRI + """ + return os.path.dirname(self.main_ontology_iri) if os.path.isabs( + self.main_ontology_iri) else self.get_default_ontology_search_path() + + @staticmethod + def create_ontology_world(use_global_default_world: bool = False, + sql_backend_filename: Optional[str] = None) -> OntologyWorld: """ Either reuse the owlready2-provided global default ontology world or create a new one :param use_global_default_world: whether or not using the owlready2-provided global default persistent world + :param sql_backend_filename: a full file path (no need to already exist) being used as SQL backend for the ontology world. If None, memory is used instead :return: owlready2-provided global default ontology world or a newly created ontology world """ world = default_world + sql_backend_path_valid = sql_backend_filename and os.path.isabs(sql_backend_filename) + sql_backend_name = sql_backend_filename if sql_backend_path_valid else "memory" if use_global_default_world: - world.set_backend(exclusive=False, enable_thread_parallelism=True) + # Reuse default world + if sql_backend_path_valid: + world.set_backend(filename=sql_backend_filename, exclusive=False, enable_thread_parallelism=True) + else: + world.set_backend(exclusive=False, enable_thread_parallelism=True) + rospy.loginfo(f"Using global default ontology world with SQL backend: {sql_backend_name}") else: - world = OntologyWorld(exclusive=False, enable_thread_parallelism=True) + # Create a new world with parallelized file parsing enabled + if sql_backend_path_valid: + world = OntologyWorld(filename=sql_backend_filename, exclusive=False, enable_thread_parallelism=True) + else: + world = OntologyWorld(exclusive=False, enable_thread_parallelism=True) + rospy.loginfo(f"Created a new ontology world with SQL backend: {sql_backend_name}") return world def load_ontology(self, ontology_iri: str) -> tuple[Optional[Ontology], Optional[Namespace]]: @@ -221,7 +266,7 @@ def browse_ontologies(ontology: Ontology, func(sub_onto, **kwargs) break - def save(self, target_filename: str = "", overwrite: bool = False) -> bool: + def save(self, target_filename: Optional[str] = None, overwrite: bool = False) -> bool: """ Save :attr:`main_ontology` to a file on disk, also caching :attr:`main_ontology_world` to a sqlite3 file @@ -231,9 +276,9 @@ def save(self, target_filename: str = "", overwrite: bool = False) -> bool: """ # Save ontologies to OWL - is_current_ontology_local = Path(self.main_ontology_iri).exists() + is_current_ontology_local = os.path.isfile(self.main_ontology_iri) current_ontology_filename = self.main_ontology_iri if is_current_ontology_local \ - else f"{Path(self.main_ontology_world.filename).parent.absolute()}/{Path(self.main_ontology_iri).stem}.owl" + else f"{self.get_main_ontology_dir()}/{Path(self.main_ontology_iri).name}" save_to_same_file = is_current_ontology_local and (target_filename == current_ontology_filename) if save_to_same_file and not overwrite: rospy.logerr( @@ -249,13 +294,12 @@ def save(self, target_filename: str = "", overwrite: bool = False) -> bool: # Commit the whole graph data of the current ontology world, saving it into SQLite3, to be reused the next time # the ontologies are loaded - main_ontology_sql_filename = f"{onto_path[0]}/{Path(save_filename).stem}.sqlite3" - self.main_ontology_world.save(file=main_ontology_sql_filename) - if Path(main_ontology_sql_filename).is_file(): + main_ontology_sql_filename = self.main_ontology_world.filename + self.main_ontology_world.save() + if os.path.isfile(main_ontology_sql_filename): rospy.loginfo( f"Main ontology world for {self.main_ontology.name} has been cached and saved to SQL: {main_ontology_sql_filename}") - else: - rospy.logwarn(f"Failed caching main ontology world for {self.main_ontology.name} to SQL") + #else: it could be using memory cache as SQL backend return True def create_ontology_concept_class(self, class_name: str, @@ -707,9 +751,10 @@ def create_rule_transitivity(self, ontology_concept_class_name: str, def reason(self, world: OntologyWorld = None, use_pellet_reasoner: bool = True) -> bool: """ - Run the reasoning with Pellet or HermiT reasoner on :attr:`main_ontology`, the two currently supported by owlready2 + Run the reasoning on a given ontology world or :attr:`main_ontology_world` with Pellet or HermiT reasoner, + the two currently supported by owlready2 - By default, the reasoning works on `owlready2.default_world` - - The reasoning also automatically save ontologies to a temporary sqlite3 file + - The reasoning also automatically save ontologies (to either in-memory cache or a temporary sqlite3 file) Ref: - https://owlready2.readthedocs.io/en/latest/reasoning.html - https://owlready2.readthedocs.io/en/latest/rule.html diff --git a/src/pycram/ontology/ontology_common.py b/src/pycram/ontology/ontology_common.py index 931145ed1..080078a08 100644 --- a/src/pycram/ontology/ontology_common.py +++ b/src/pycram/ontology/ontology_common.py @@ -10,6 +10,9 @@ from owlready2 import issubclass, Thing +ONTOLOGY_SQL_BACKEND_FILE_EXTENSION = ".sqlite3" +ONTOLOGY_OWL_FILE_EXTENSION = ".owl" + class OntologyConceptHolderStore(object, metaclass=Singleton): """ diff --git a/test/test_ontology.py b/test/test_ontology.py index c33a55681..23d90978b 100644 --- a/test/test_ontology.py +++ b/test/test_ontology.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os.path import unittest import logging from pathlib import Path @@ -15,8 +16,9 @@ owlready2 = None rospy.logwarn("Could not import owlready2, Ontology unit-tests could not run!") -from pycram.ontology.ontology import OntologyManager, SOMA_ONTOLOGY_IRI -from pycram.ontology.ontology_common import OntologyConceptHolderStore, OntologyConceptHolder +from pycram.ontology.ontology import OntologyManager, SOMA_HOME_ONTOLOGY_IRI, SOMA_ONTOLOGY_IRI +from pycram.ontology.ontology_common import (OntologyConceptHolderStore, OntologyConceptHolder, + ONTOLOGY_SQL_BACKEND_FILE_EXTENSION, ONTOLOGY_OWL_FILE_EXTENSION) class TestOntologyManager(unittest.TestCase): @@ -39,9 +41,9 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - save_dir = Path(f"{Path.home()}/ontologies") - owl_filepath = f"{save_dir}/{Path(cls.ontology_manager.main_ontology_iri).stem}.owl" - sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}.sqlite3" + save_dir = cls.ontology_manager.get_main_ontology_dir() + owl_filepath = f"{save_dir}/{Path(cls.ontology_manager.main_ontology_iri).stem}{ONTOLOGY_OWL_FILE_EXTENSION}" + sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}" os.remove(owl_filepath) cls.remove_sql_file(sql_filepath) @@ -59,15 +61,34 @@ def test_ontology_manager(self): self.assertTrue(self.ontology_manager.initialized()) def test_ontology_world(self): - if owlready2: - self.assertIsNotNone(self.ontology_manager.main_ontology_world) - extra_world = self.ontology_manager.create_ontology_world() - self.assertIsNotNone(extra_world) - extra_world_sql_filepath = f"{onto_path[0]}/extra_world.sqlite3" - extra_world.save(file=extra_world_sql_filepath) - world_saved_to_sql = Path(extra_world_sql_filepath).is_file() - self.remove_sql_file(extra_world_sql_filepath) - self.assertTrue(world_saved_to_sql) + if not owlready2: + return + # Main ontology world as the global default world + main_world = self.ontology_manager.main_ontology_world + self.assertIsNotNone(main_world) + self.assertTrue(main_world is owlready2.default_world) + + # Extra world with memory backend + extra_memory_world = self.ontology_manager.create_ontology_world(use_global_default_world=False) + self.assertIsNotNone(extra_memory_world) + self.assertTrue(extra_memory_world != owlready2.default_world) + + # Extra world with SQL backend from a non-existing SQL file + extra_world_sql_filename = f"{self.ontology_manager.get_main_ontology_dir()}/extra_world{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}" + extra_sql_world = self.ontology_manager.create_ontology_world(use_global_default_world=False, + sql_backend_filename=extra_world_sql_filename) + self.assertIsNotNone(extra_sql_world) + # Save it at [extra_world_sql_filename] + extra_sql_world.save() + self.assertTrue(os.path.isfile(extra_world_sql_filename)) + + # Extra world with SQL backend from an existing SQL file + extra_sql_world_2 = self.ontology_manager.create_ontology_world(use_global_default_world=False, + sql_backend_filename=extra_world_sql_filename) + self.assertIsNotNone(extra_sql_world_2) + + # Remove SQL file finally + self.remove_sql_file(extra_world_sql_filename) def test_ontology_concept_holder(self): if not owlready2: @@ -83,7 +104,8 @@ def test_loaded_ontologies(self): return self.assertIsNotNone(self.main_ontology) self.assertTrue(self.main_ontology.loaded) - if self.ontology_manager.main_ontology_iri is SOMA_ONTOLOGY_IRI: + if self.ontology_manager.main_ontology_iri is SOMA_ONTOLOGY_IRI or \ + self.ontology_manager.main_ontology_iri is SOMA_HOME_ONTOLOGY_IRI: self.assertIsNotNone(self.soma) self.assertTrue(self.soma.loaded) self.assertIsNotNone(self.dul) @@ -124,17 +146,18 @@ def test_ontology_triple_classes_dynamic_creation(self): PLACEABLE_ON_PREDICATE_NAME = "placeable_on" HOLD_OBJ_PREDICATE_NAME = "hold_obj" self.assertTrue( - self.ontology_manager.create_ontology_triple_classes(ontology_subject_parent_class=self.soma.Container if self.soma else None, - subject_class_name="OntologyPlaceHolderObject", - ontology_object_parent_class=self.dul.PhysicalObject - if self.dul else None, - object_class_name="OntologyHandheldObject", - predicate_class_name=PLACEABLE_ON_PREDICATE_NAME, - inverse_predicate_class_name=HOLD_OBJ_PREDICATE_NAME, - ontology_property_parent_class=self.soma.affordsBearer - if self.soma else None, - ontology_inverse_property_parent_class=self.soma.isBearerAffordedBy - if self.soma else None)) + self.ontology_manager.create_ontology_triple_classes( + ontology_subject_parent_class=self.soma.Container if self.soma else None, + subject_class_name="OntologyPlaceHolderObject", + ontology_object_parent_class=self.dul.PhysicalObject + if self.dul else None, + object_class_name="OntologyHandheldObject", + predicate_class_name=PLACEABLE_ON_PREDICATE_NAME, + inverse_predicate_class_name=HOLD_OBJ_PREDICATE_NAME, + ontology_property_parent_class=self.soma.affordsBearer + if self.soma else None, + ontology_inverse_property_parent_class=self.soma.isBearerAffordedBy + if self.soma else None)) def create_ontology_handheld_object_designator(object_name: str, ontology_parent_class: Type[owlready2.Thing]): return self.ontology_manager.create_ontology_linked_designator(object_name=object_name, @@ -179,17 +202,18 @@ def test_ontology_class_destruction(self): self.assertIsNone(self.ontology_manager.get_ontology_class(concept_class_name)) self.assertFalse(OntologyConceptHolderStore().get_ontology_concepts_by_class(dynamic_ontology_concept_class)) + #@unittest.skip("owlready2 reasoning requires Java runtime, which is available only for CI running on master/dev") def test_ontology_reasoning(self): if not owlready2: return - REASONING_TEST_ONTOLOGY_IRI = "reasoning_test.owl" + REASONING_TEST_ONTOLOGY_IRI = f"reasoning_test{ONTOLOGY_OWL_FILE_EXTENSION}" ENTITY_CONCEPT_NAME = "Entity" CAN_TRANSPORT_PREDICATE_NAME = "can_transport" TRANSPORTABLE_BY_PREDICATE_NAME = "transportable_by" CORESIDE_PREDICATE_NAME = "coreside" - # Create a test world for reasoning + # Create a test world (with memory SQL backend) for reasoning reasoning_world = self.ontology_manager.create_ontology_world() reasoning_ontology = reasoning_world.get_ontology(REASONING_TEST_ONTOLOGY_IRI) @@ -249,7 +273,6 @@ def coresidents(a: reasoning_ontology.Entity, b: reasoning_ontology.Entity) -> b # Reason on [reasoning_world] self.ontology_manager.reason(world=reasoning_world) - self.remove_sql_file(sql_filepath=f"{onto_path[0]}/{Path(reasoning_ontology.name).stem}.sqlite3") # Test reflexivity for entity in entities: @@ -270,11 +293,10 @@ def test_ontology_save(self): if not owlready2: return - save_dir = Path(f"{Path.home()}/ontologies") - save_dir.mkdir(parents=True, exist_ok=True) - owl_filepath = f"{save_dir}/{Path(self.ontology_manager.main_ontology_iri).stem}.owl" - sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}.sqlite3" - self.ontology_manager.save(owl_filepath) + save_dir = self.ontology_manager.get_main_ontology_dir() + owl_filepath = f"{save_dir}/{Path(self.ontology_manager.main_ontology_iri).stem}{ONTOLOGY_OWL_FILE_EXTENSION}" + sql_filepath = f"{save_dir}/{Path(owl_filepath).stem}{ONTOLOGY_SQL_BACKEND_FILE_EXTENSION}" + self.assertTrue(self.ontology_manager.save(owl_filepath)) self.assertTrue(Path(owl_filepath).is_file()) self.assertTrue(Path(sql_filepath).is_file())