Merge pull request #43 from linkml/linkml-issue-157-more-tests

Do not convert lists using as_json_obj Fixes #42

linkml_runtime/utils/formatutils.py

@@ -133,8 +133,9 @@ def remove_empty_items(obj: Any, hide_protected_keys: bool = False, inside: bool
     :return: copy of obj with empty items removed or None if obj itself is "empty"
     """
     if is_list(obj):
+        # for discussion of logic, see: https://github.com/linkml/linkml-runtime/issues/42
         obj_list = [e for e in [remove_empty_items(l, hide_protected_keys=hide_protected_keys, inside=True)
-                                for l in as_json_obj(obj)] if not is_empty(e)]
+                                for l in obj if l != '_root'] if not is_empty(e)]
         return obj_list if not inside or not is_empty(obj_list) else None
     elif is_dict(obj):
         obj_dict = {k: v for k, v in [(k2, remove_empty_items(v2, hide_protected_keys=hide_protected_keys, inside=True))
@@ -146,7 +147,7 @@ def remove_empty_items(obj: Any, hide_protected_keys: bool = False, inside: bool
             enum_text = list(obj_dict.values())[0].get('text', None)
             if enum_text is not None:
                 return enum_text
-        if hide_protected_keys and len(obj_dict) == 1 and list(obj_dict.keys())[0].startswith('_'):
+        if hide_protected_keys and len(obj_dict) == 1 and str(list(obj_dict.keys())[0]).startswith('_'):
             inner_element = list(obj_dict.values())[0]
             if isinstance(inner_element, dict):
                 obj_dict = inner_element

tests/test_loaders_dumpers/models/books_normalized.py

@@ -1,5 +1,5 @@
 # Auto generated from books_normalized.yaml by pythongen.py version: 0.9.0
-# Generation date: 2021-07-27 12:45
+# Generation date: 2021-09-10 16:59
 # Schema: example
 #
 # id: https://w3id.org/example
@@ -65,7 +65,7 @@ class CreativeWork(YAMLRoot):
 
     id: Union[str, CreativeWorkId] = None
     name: Optional[str] = None
-    genres: Optional[Union[str, List[str]]] = empty_list()
+    genres: Optional[Union[Union[str, "GenreEnum"], List[Union[str, "GenreEnum"]]]] = empty_list()
     creator: Optional[Union[dict, "Author"]] = None
     summary: Optional[str] = None
     reviews: Optional[Union[Union[dict, "Review"], List[Union[dict, "Review"]]]] = empty_list()
@@ -81,14 +81,14 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]):
 
         if not isinstance(self.genres, list):
             self.genres = [self.genres] if self.genres is not None else []
-        self.genres = [v if isinstance(v, str) else str(v) for v in self.genres]
+        self.genres = [v if isinstance(v, GenreEnum) else GenreEnum(v) for v in self.genres]
 
         if self.creator is not None and not isinstance(self.creator, Author):
             self.creator = Author(**as_dict(self.creator))
 
         if not isinstance(self.genres, list):
             self.genres = [self.genres] if self.genres is not None else []
-        self.genres = [v if isinstance(v, str) else str(v) for v in self.genres]
+        self.genres = [v if isinstance(v, GenreEnum) else GenreEnum(v) for v in self.genres]
 
         if self.summary is not None and not isinstance(self.summary, str):
             self.summary = str(self.summary)
@@ -139,7 +139,7 @@ class BookSeries(CreativeWork):
 
     id: Union[str, BookSeriesId] = None
     books: Optional[Union[Dict[Union[str, BookId], Union[dict, Book]], List[Union[dict, Book]]]] = empty_dict()
-    genres: Optional[Union[str, List[str]]] = empty_list()
+    genres: Optional[Union[Union[str, "GenreEnum"], List[Union[str, "GenreEnum"]]]] = empty_list()
     price: Optional[float] = None
 
     def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]):
@@ -152,7 +152,7 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]):
 
         if not isinstance(self.genres, list):
             self.genres = [self.genres] if self.genres is not None else []
-        self.genres = [v if isinstance(v, str) else str(v) for v in self.genres]
+        self.genres = [v if isinstance(v, GenreEnum) else GenreEnum(v) for v in self.genres]
 
         if self.price is not None and not isinstance(self.price, float):
             self.price = float(self.price)
@@ -170,7 +170,7 @@ class Author(YAMLRoot):
     class_model_uri: ClassVar[URIRef] = EXAMPLE.Author
 
     name: Optional[str] = None
-    genres: Optional[Union[str, List[str]]] = empty_list()
+    genres: Optional[Union[Union[str, "GenreEnum"], List[Union[str, "GenreEnum"]]]] = empty_list()
     from_country: Optional[Union[str, CountryName]] = None
 
     def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]):
@@ -179,7 +179,7 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]):
 
         if not isinstance(self.genres, list):
             self.genres = [self.genres] if self.genres is not None else []
-        self.genres = [v if isinstance(v, str) else str(v) for v in self.genres]
+        self.genres = [v if isinstance(v, GenreEnum) else GenreEnum(v) for v in self.genres]
 
         if self.from_country is not None and not isinstance(self.from_country, CountryName):
             self.from_country = CountryName(self.from_country)
@@ -286,7 +286,7 @@ class slots:
                    model_uri=EXAMPLE.creator, domain=None, range=Optional[Union[dict, Author]])
 
 slots.genres = Slot(uri=EXAMPLE.genres, name="genres", curie=EXAMPLE.curie('genres'),
-                   model_uri=EXAMPLE.genres, domain=None, range=Optional[Union[str, List[str]]])
+                   model_uri=EXAMPLE.genres, domain=None, range=Optional[Union[Union[str, "GenreEnum"], List[Union[str, "GenreEnum"]]]])
 
 slots.from_country = Slot(uri=EXAMPLE.from_country, name="from_country", curie=EXAMPLE.curie('from_country'),
                    model_uri=EXAMPLE.from_country, domain=None, range=Optional[Union[str, CountryName]])

tests/test_loaders_dumpers/models/books_normalized.yaml

@@ -78,8 +78,7 @@ slots:
     examples:
     - value: Ian M Banks
   genres:
-    #range: genre_enum
-    range: string
+    range: genre_enum
     multivalued: true
   from_country:
     range: country

tests/test_loaders_dumpers/test_csv_loader_dumper.py

@@ -1,13 +1,18 @@
 import os
 import unittest
 import json
+import logging
 
-from linkml_runtime.dumpers import json_dumper
+from jsonasobj2 import as_json_obj, JsonObj
+
+from linkml_runtime.dumpers import json_dumper, yaml_dumper
 from linkml_runtime.loaders import yaml_loader
+from linkml_runtime.utils.formatutils import remove_empty_items, is_empty
 from linkml_runtime.utils.schemaview import SchemaView
 from linkml_runtime.dumpers import csv_dumper
 from linkml_runtime.loaders import csv_loader
-from tests.test_loaders_dumpers.models.books_normalized import Shop, Book
+from linkml_runtime.utils.yamlutils import as_json_object
+from tests.test_loaders_dumpers.models.books_normalized import Shop, Book, GenreEnum, BookSeries
 
 
 ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -27,24 +32,57 @@ def _json(obj) -> str:
 
 class CSVGenTestCase(unittest.TestCase):
 
+    def test_object_model(self):
+        book = Book(id='B1', genres=['fantasy'], creator={})
+        print(book.genres)
+        print(type(book.genres[0]))
+        logging.debug(as_json_obj(book.genres[0]))
+        assert str(book.genres[0]) == 'fantasy'
+        assert book.genres[0].code.text == 'fantasy'
+        processed = remove_empty_items(book.genres)
+        print(f'PR={processed}')
+        assert processed[0] == 'fantasy'
+        series = BookSeries(id='S1')
+        series.books.append(book)
+        schemaview = SchemaView(SCHEMA)
+        shop = Shop()
+        shop.all_book_series.append(book)
+        #csvstr = csv_dumper.dumps(shop, index_slot='all_book_series', schemaview=schemaview)
+        #logging.debug(csvstr)
+
     def test_csvgen_roundtrip(self):
         schemaview = SchemaView(SCHEMA)
         data = yaml_loader.load(DATA, target_class=Shop)
         csv_dumper.dump(data, to_file=OUTPUT, index_slot='all_book_series', schemaview=schemaview)
         roundtrip = csv_loader.load(OUTPUT, target_class=Shop, index_slot='all_book_series', schemaview=schemaview)
-        print(json_dumper.dumps(roundtrip))
+        logging.debug(json_dumper.dumps(roundtrip))
+        logging.debug(f'COMPARE 1: {roundtrip}')
+        logging.debug(f'COMPARE 2: {data}')
         assert roundtrip == data
 
     def test_csvgen_unroundtrippable(self):
         schemaview = SchemaView(SCHEMA)
         #schema = YAMLGenerator(SCHEMA).schema
         data = yaml_loader.load(DATA2, target_class=Shop)
-        #print(json_dumper.dumps(data))
-        #print(csv_dumper.dumps(data, index_slot='all_book_series', schema=schema))
+        logging.debug(data.all_book_series[0])
+        logging.debug(data.all_book_series[0].genres[0])
+        assert str(data.all_book_series[0].genres[0]) == 'fantasy'
+        logging.debug(yaml_dumper.dumps(data))
+        logging.debug(json_dumper.dumps(data))
+        processed = remove_empty_items(data)
+        logging.debug(f'PROC {processed["all_book_series"]}')
+        asj = as_json_object(processed, None)
+        logging.debug(f'ASJ {asj["all_book_series"]}')
+        reconstituted_json = json.loads(json_dumper.dumps(data))
+        s0 = reconstituted_json['all_book_series'][0]
+        logging.debug(s0)
+        logging.debug(json_dumper.dumps(data))
+        #logging.debug(csv_dumper.dumps(data, index_slot='all_book_series', schema=schema))
         csv_dumper.dump(data, to_file=OUTPUT2, index_slot='all_book_series', schemaview=schemaview)
+        #assert False
         roundtrip = csv_loader.load(OUTPUT2, target_class=Shop, index_slot='all_book_series', schemaview=schemaview)
-        print(json_dumper.dumps(roundtrip))
-        #assert roundtrip == data
+        logging.debug(json_dumper.dumps(roundtrip))
+        assert roundtrip == data
 
 
 

tests/test_utils/test_formatutils.py

@@ -127,6 +127,8 @@ def test_empty_functions(self):
             self.assertTrue(is_empty(thing), msg=f"{thing} should clock in as empty")
         for thing in non_empty_things:
             self.assertFalse(is_empty(thing))
+        obj = JsonObj([])
+        assert is_empty(obj)
 
     def test_remove_empty_items(self):
         """ Test the various remove empty items paths """