Do not serialize non-default fields by default (#1452)

Added a configuration entry (enabled by default) that serializes only the modified fields in an SDFG. This leads to a reduction in size. Merging this PR is contingent on updating the SDFG renderer to use the defaults/metadata for properties. --------- Co-authored-by: Philipp Schaad <[email protected]>
spcl · Dec 4, 2023 · 6374843 · 6374843
1 parent b0cd25b
commit 6374843
Show file tree

Hide file tree

Showing 11 changed files with 68 additions and 25 deletions.
diff --git a/dace/codegen/codegen.py b/dace/codegen/codegen.py
@@ -165,6 +165,7 @@ def generate_code(sdfg, validate=True) -> List[CodeObject]:
 
     if Config.get_bool('testing', 'serialization'):
         from dace.sdfg import SDFG
+        import difflib
         import filecmp
         import shutil
         import tempfile
@@ -174,9 +175,16 @@ def generate_code(sdfg, validate=True) -> List[CodeObject]:
             sdfg2.save(f'{tmp_dir}/test2.sdfg', hash=False)
             print('Testing SDFG serialization...')
             if not filecmp.cmp(f'{tmp_dir}/test.sdfg', f'{tmp_dir}/test2.sdfg'):
-                shutil.move(f"{tmp_dir}/test.sdfg", "test.sdfg")
-                shutil.move(f"{tmp_dir}/test2.sdfg", "test2.sdfg")
-                raise RuntimeError('SDFG serialization failed - files do not match')
+                with open(f'{tmp_dir}/test.sdfg', 'r') as f1:
+                    with open(f'{tmp_dir}/test2.sdfg', 'r') as f2:
+                        diff = difflib.unified_diff(f1.readlines(),
+                                                    f2.readlines(),
+                                                    fromfile='test.sdfg  (first save)',
+                                                    tofile='test2.sdfg (after roundtrip)')
+                diff = ''.join(diff)
+                shutil.move(f'{tmp_dir}/test.sdfg', 'test.sdfg')
+                shutil.move(f'{tmp_dir}/test2.sdfg', 'test2.sdfg')
+                raise RuntimeError(f'SDFG serialization failed - files do not match:\n{diff}')
 
     # Convert any loop constructs with hierarchical loop regions into simple 1-level state machine loops.
     # TODO (later): Adapt codegen to deal with hierarchical CFGs instead.

diff --git a/dace/codegen/targets/cpu.py b/dace/codegen/targets/cpu.py
@@ -1914,7 +1914,7 @@ def _generate_ConsumeEntry(
                                               'size_t')
 
         # Take quiescence condition into account
-        if node.consume.condition.code is not None:
+        if node.consume.condition is not None:
             condition_string = "[&]() { return %s; }, " % cppunparse.cppunparse(node.consume.condition.code, False)
         else:
             condition_string = ""
@@ -1933,7 +1933,7 @@ def _generate_ConsumeEntry(
             "{num_pes}, {condition}"
             "[&](int {pe_index}, {element_or_chunk}) {{".format(
                 chunksz=node.consume.chunksize,
-                cond="" if node.consume.condition.code is None else "_cond",
+                cond="" if node.consume.condition is None else "_cond",
                 condition=condition_string,
                 stream_in=input_stream.data,  # TODO: stream arrays
                 element_or_chunk=chunk,

diff --git a/dace/config_schema.yml b/dace/config_schema.yml
@@ -943,6 +943,14 @@ required:
                     When an exception is raised in a deserialization process (e.g., due to missing library node),
                     by default a warning is issued. If this setting is True, the exception will be raised as-is.
 
+            serialize_all_fields:
+                type: bool
+                default: false
+                title: Serialize all unmodified fields in SDFG files
+                description: >
+                    If False (default), saving an SDFG keeps only the modified non-default properties. If True,
+                    saves all fields.
+
     #############################################
     # DaCe library settings
 

diff --git a/dace/properties.py b/dace/properties.py
@@ -1023,6 +1023,14 @@ def as_string(self, code):
         else:
             self.code = code
 
+    def __eq__(self, other):
+        if isinstance(other, str) or other is None:
+            return self.as_string == other
+        elif isinstance(other, CodeBlock):
+            return self.as_string == other.as_string and self.language == other.language
+        else:
+            return super().__eq__(other)
+
     def to_json(self):
         # Two roundtrips to avoid issues in AST parsing/unparsing of negative
         # numbers, i.e., "(-1)" becomes "(- 1)"
@@ -1382,6 +1390,8 @@ def to_json(self, obj):
     def from_json(obj, context=None):
         if obj is None:
             return None
+        elif isinstance(obj, typeclass):
+            return obj
         elif isinstance(obj, str):
             return TypeClassProperty.from_string(obj)
         elif isinstance(obj, dict):

diff --git a/dace/sdfg/nodes.py b/dace/sdfg/nodes.py
@@ -1005,8 +1005,10 @@ def __str__(self):
     @property
     def free_symbols(self) -> Set[str]:
         dyn_inputs = set(c for c in self.in_connectors if not c.startswith('IN_'))
-        return ((set(self._consume.num_pes.free_symbols)
-                 | set(self._consume.condition.get_free_symbols())) - dyn_inputs)
+        result = set(self._consume.num_pes.free_symbols)
+        if self._consume.condition is not None:
+            result |= set(self._consume.condition.get_free_symbols())
+        return result - dyn_inputs
 
     def new_symbols(self, sdfg, state, symbols) -> Dict[str, dtypes.typeclass]:
         from dace.codegen.tools.type_inference import infer_expr_type
@@ -1094,7 +1096,7 @@ class Consume(object):
     label = Property(dtype=str, desc="Name of the consume node")
     pe_index = Property(dtype=str, desc="Processing element identifier")
     num_pes = SymbolicProperty(desc="Number of processing elements", default=1)
-    condition = CodeProperty(desc="Quiescence condition", allow_none=True)
+    condition = CodeProperty(desc="Quiescence condition", allow_none=True, default=None)
     schedule = EnumProperty(dtype=dtypes.ScheduleType, desc="Consume schedule", default=dtypes.ScheduleType.Default)
     chunksize = Property(dtype=int, desc="Maximal size of elements to consume at a time", default=1)
     debuginfo = DebugInfoProperty()

diff --git a/dace/sdfg/sdfg.py b/dace/sdfg/sdfg.py
@@ -579,7 +579,8 @@ def to_json(self, hash=False):
         tmp = super().to_json()
 
         # Ensure properties are serialized correctly
-        tmp['attributes']['constants_prop'] = json.loads(dace.serialize.dumps(tmp['attributes']['constants_prop']))
+        if 'constants_prop' in tmp['attributes']:
+            tmp['attributes']['constants_prop'] = json.loads(dace.serialize.dumps(tmp['attributes']['constants_prop']))
 
         tmp['sdfg_list_id'] = int(self.sdfg_id)
         tmp['start_state'] = self._start_block
@@ -604,8 +605,13 @@ def from_json(cls, json_obj, context_info=None):
         nodes = json_obj['nodes']
         edges = json_obj['edges']
 
+        if 'constants_prop' in attrs:
+            constants_prop = dace.serialize.loads(dace.serialize.dumps(attrs['constants_prop']))
+        else:
+            constants_prop = None
+
         ret = SDFG(name=attrs['name'],
-                   constants=dace.serialize.loads(dace.serialize.dumps(attrs['constants_prop'])),
+                   constants=constants_prop,
                    parent=context_info['sdfg'])
 
         dace.serialize.set_properties_from_json(ret,

diff --git a/dace/sdfg/state.py b/dace/sdfg/state.py
@@ -1644,7 +1644,9 @@ def add_consume(self,
         pe_tuple = (elements[0], SymbolicProperty.from_string(elements[1]))
 
         debuginfo = _getdebuginfo(debuginfo or self._default_lineinfo)
-        consume = nd.Consume(name, pe_tuple, CodeBlock(condition, language), schedule, chunksize, debuginfo=debuginfo)
+        if condition is not None:
+            condition = CodeBlock(condition, language)
+        consume = nd.Consume(name, pe_tuple, condition, schedule, chunksize, debuginfo=debuginfo)
         entry = nd.ConsumeEntry(consume)
         exit = nd.ConsumeExit(consume)
 

diff --git a/dace/serialize.py b/dace/serialize.py
@@ -175,8 +175,11 @@ def dump(*args, **kwargs):
 
 
 def all_properties_to_json(object_with_properties):
+    save_all_fields = config.Config.get_bool('testing', 'serialize_all_fields')
     retdict = {}
     for x, v in object_with_properties.properties():
+        if not save_all_fields and v == x.default:  # Skip default fields
+            continue
         if x.optional and not x.optional_condition(object_with_properties):
             continue
         retdict[x.attr_name] = x.to_json(v)

diff --git a/dace/transformation/transformation.py b/dace/transformation/transformation.py
@@ -391,12 +391,13 @@ def from_json(json_obj: Dict[str, Any], context: Dict[str, Any] = None) -> 'Patt
                      if ext.__name__ == json_obj['transformation'])
 
         # Recreate subgraph
-        expr = xform.expressions()[json_obj['expr_index']]
-        subgraph = {expr.node(int(k)): int(v) for k, v in json_obj['_subgraph'].items()}
+        expr = xform.expressions()[json_obj.get('expr_index', 0)]
+        subgraph = {expr.node(int(k)): int(v) for k, v in json_obj.get('_subgraph', {}).items()}
 
         # Reconstruct transformation
         ret = xform()
-        ret.setup_match(None, json_obj['sdfg_id'], json_obj['state_id'], subgraph, json_obj['expr_index'])
+        ret.setup_match(None, json_obj.get('sdfg_id', 0), json_obj.get('state_id', 0), subgraph,
+                        json_obj.get('expr_index', 0))
         context = context or {}
         context['transformation'] = ret
         serialize.set_properties_from_json(ret, json_obj, context=context, ignore_properties={'transformation', 'type'})
@@ -652,12 +653,13 @@ def from_json(json_obj: Dict[str, Any], context: Dict[str, Any] = None) -> 'Expa
         xform = pydoc.locate(json_obj['classpath'])
 
         # Recreate subgraph
-        expr = xform.expressions()[json_obj['expr_index']]
-        subgraph = {expr.node(int(k)): int(v) for k, v in json_obj['_subgraph'].items()}
+        expr = xform.expressions()[json_obj.get('expr_index', 0)]
+        subgraph = {expr.node(int(k)): int(v) for k, v in json_obj.get('_subgraph', {}).items()}
 
         # Reconstruct transformation
         ret = xform()
-        ret.setup_match(None, json_obj['sdfg_id'], json_obj['state_id'], subgraph, json_obj['expr_index'])
+        ret.setup_match(None, json_obj.get('sdfg_id', 0), json_obj.get('state_id', 0), subgraph,
+                        json_obj.get('expr_index', 0))
         context = context or {}
         context['transformation'] = ret
         serialize.set_properties_from_json(ret,
@@ -864,7 +866,7 @@ def from_json(json_obj: Dict[str, Any], context: Dict[str, Any] = None) -> 'Subg
 
         # Reconstruct transformation
         ret = xform()
-        ret.setup_match(json_obj['subgraph'], json_obj['sdfg_id'], json_obj['state_id'])
+        ret.setup_match(json_obj.get('subgraph', {}), json_obj.get('sdfg_id', 0), json_obj.get('state_id', 0))
         context = context or {}
         context['transformation'] = ret
         serialize.set_properties_from_json(ret, json_obj, context=context, ignore_properties={'transformation', 'type'})

diff --git a/tests/openmp_test.py b/tests/openmp_test.py
@@ -54,10 +54,7 @@ def test_omp_props():
             break
 
     mapnode.schedule = dtypes.ScheduleType.CPU_Multicore
-    json = sdfg.to_json()
-    assert (key_exists(json, 'omp_num_threads'))
-    assert (key_exists(json, 'omp_schedule'))
-    assert (key_exists(json, 'omp_chunk_size'))
+
     code = sdfg.generate_code()[0].clean_code
     assert ("#pragma omp parallel for" in code)
 
@@ -73,6 +70,11 @@ def test_omp_props():
     code = sdfg.generate_code()[0].clean_code
     assert ("#pragma omp parallel for schedule(guided, 5) num_threads(10)" in code)
 
+    json = sdfg.to_json()
+    assert (key_exists(json, 'omp_num_threads'))
+    assert (key_exists(json, 'omp_schedule'))
+    assert (key_exists(json, 'omp_chunk_size'))
+
 
 def test_omp_parallel():
 

diff --git a/tests/transformations/local_storage_test.py b/tests/transformations/local_storage_test.py
@@ -124,7 +124,7 @@ def test_in_local_storage_implicit():
 
     # Check array was set correctly
     serialized = sdfg.transformation_hist[0].to_json()
-    assert serialized["array"] == None
+    assert "array" not in serialized or serialized["array"] is None
 
 
 def test_out_local_storage_explicit():
@@ -217,7 +217,7 @@ def test_out_local_storage_implicit():
 
     # Check array was set correctly
     serialized = sdfg.transformation_hist[0].to_json()
-    assert serialized["array"] == None
+    assert "array" not in serialized or serialized["array"] is None
 
 
 @dace.program
@@ -250,8 +250,8 @@ def test_uneven(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
     test_in_local_storage_explicit()
     test_in_local_storage_implicit()
     test_out_local_storage_explicit()
     test_out_local_storage_implicit()
+    unittest.main()