From 1f2268aa06b052551bb7122327d5dc039d9be743 Mon Sep 17 00:00:00 2001 From: James Johnson Date: Tue, 22 Oct 2019 17:30:07 -0700 Subject: [PATCH 1/2] Added test to reproduce metadata problem --- tests/test_basic_parse.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_basic_parse.py b/tests/test_basic_parse.py index 7f72a30..1846926 100644 --- a/tests/test_basic_parse.py +++ b/tests/test_basic_parse.py @@ -102,6 +102,11 @@ def test_metadata(self): local int a ; """, optimize=True) + def test_metadata_with_quoted_strings(self): + res = parse_string(""" + local int a ">; + """, optimize=True) + def test_typedef(self): res = parse_string(""" typedef unsigned int UINT2; From b61d2c9cdcd2f0bb167fa05bf93b9757bc28ae21 Mon Sep 17 00:00:00 2001 From: James Johnson Date: Wed, 23 Oct 2019 06:12:29 -0700 Subject: [PATCH 2/2] Needs more testing, almost done --- py010parser/__init__.py | 5 +++-- py010parser/c_ast.py | 20 ++++---------------- py010parser/c_lexer.py | 40 ++++++++++++++++++++++++++------------- py010parser/c_parser.py | 36 +++++++++++++++++++++++++++++------ tests/test_basic_parse.py | 9 +++++++-- 5 files changed, 71 insertions(+), 39 deletions(-) diff --git a/py010parser/__init__.py b/py010parser/__init__.py index 0df6995..8dda949 100644 --- a/py010parser/__init__.py +++ b/py010parser/__init__.py @@ -112,7 +112,7 @@ def parse_file(filename, use_cpp=True, cpp_path='cpp', cpp_args='', ) def parse_string(text, parser=None, filename="", optimize=True, predefine_types=True, - use_cpp=True, cpp_path='cpp', cpp_args='', keep_scopes=False): + use_cpp=True, cpp_path='cpp', cpp_args='', keep_scopes=False, debuglevel=0): if use_cpp: tempfile_path = '' @@ -134,5 +134,6 @@ def parse_string(text, parser=None, filename="", optimize=True, predefin text, filename, predefine_types = predefine_types, - keep_scopes = keep_scopes + keep_scopes = keep_scopes, + debuglevel = debuglevel, ) diff --git a/py010parser/c_ast.py b/py010parser/c_ast.py index e90d935..876a8c8 100644 --- a/py010parser/c_ast.py +++ b/py010parser/c_ast.py @@ -528,26 +528,14 @@ def children(self): attr_names = ('name',) class Metadata010(Node): - metadata010 = r'<((\w+)=(.*?))(,\s*(\w+)=(.*))*>' - - def __init__(self, value, coord=None): - self.value = value + def __init__(self, values, coord=None): + self.values = values self.coord = coord - - match = re.match(self.metadata010, value) - kvs = {} - - # split into groups of three - for full,k,v in zip(*(iter(match.groups()),) * 3): - if full is not None: - kvs[k] = v - - self.keyvals = kvs def children(self): - return tuple([]) + return self.values - attr_names = ('keyvals') + attr_names = ('values') class IdentifierType(Node): def __init__(self, names, coord=None): diff --git a/py010parser/c_lexer.py b/py010parser/c_lexer.py index 52d9e37..aa6ac4d 100644 --- a/py010parser/c_lexer.py +++ b/py010parser/c_lexer.py @@ -219,7 +219,7 @@ def _make_tok_location(self, token): 'PPHASH', # '#' # 010-metadata - 'METADATA010', + #'METADATA010', ) ## @@ -236,6 +236,20 @@ def _make_tok_location(self, token): # typedef ushort FIXEDPT ; # etc. #metadata010 = r'<((\w+)=([^\s]+))(,\s*((\w+)=([^\s]+)))*>' + + # Official reference: + # https://www.sweetscape.com/010editor/manual/TemplateVariables.htm + # + # < format=hex|decimal|octal|binary, + # fgcolor=, + # bgcolor=, + # comment=""|, + # name=""|, + # open=true|false|suppress, + # hidden=true|false, + # read=, + # write= + # size=| > metadata010 = r'<((\w+)=(.*?))(,(\w+)\s*=(.*))*>' hex_prefix = '0[xX]' @@ -530,19 +544,19 @@ def t_ID(self, t): t.type = "TYPEID" return t - @TOKEN(metadata010) - def t_METADATA010(self, t): - match = re.match(self.metadata010, t.value) - kvs = {} - - # split into groups of three - #for full,k,v in zip(*(iter(match.groups()),) * 3): - #if full is not None: - #kvs[k] = v +# @TOKEN(metadata010) +# def t_METADATA010(self, t): +# match = re.match(self.metadata010, t.value) +# kvs = {} # - #t.keyvals = kvs - - return t +# # split into groups of three +# #for full,k,v in zip(*(iter(match.groups()),) * 3): +# #if full is not None: +# #kvs[k] = v +## +# #t.keyvals = kvs +# +# return t def t_error(self, t): msg = 'Illegal character %s' % repr(t.value[0]) diff --git a/py010parser/c_parser.py b/py010parser/c_parser.py index 26ce498..ea2dc2d 100644 --- a/py010parser/c_parser.py +++ b/py010parser/c_parser.py @@ -19,11 +19,11 @@ class CParser(PLYParser): def __init__( self, - lex_optimize=True, + lex_optimize=False, lextab='py010parser.lextab', - yacc_optimize=True, + yacc_optimize=False, yacctab='py010parser.yacctab', - yacc_debug=False): + yacc_debug=True): """ Create a new CParser. Some arguments for controlling the debug/optimization @@ -91,6 +91,7 @@ def __init__( 'type_qualifier_list', 'struct_declarator_list', 'metadata010', + 'metadata_assignment_list', 'enum_type' ] @@ -1586,12 +1587,35 @@ def p_block_item_list(self, p): p[0] = p[1] if (len(p) == 2 or p[2] == [None]) else p[1] + p[2] def p_metadata010(self, p): - """ metadata010 : METADATA010 + """ metadata010 : LT metadata_assignment_list GT """ - meta = c_ast.Metadata010(p[1]) - + meta = c_ast.Metadata010(p[2]) p[0] = meta + def p_metadata_assignment_list_1(self, p): + """ metadata_assignment_list : metadata_assignment + | metadata_assignment_list COMMA + | metadata_assignment_list COMMA metadata_assignment + """ + if isinstance(p[1], list) and isinstance(p[1][0], list) and len(p[1]) == 1: + res = p[1] + vals = p[3:] + else: + res = [] + vals = p[1:] + + while len(vals) > 0: + res.append(vals[0]) + vals = vals[2:] + + p[0] = res + + def p_metadata_assignment(self, p): + """ metadata_assignment : ID EQUALS constant + | ID EQUALS STRING_LITERAL + """ + p[0] = [p[1], p[3]] + def p_compound_statement_1(self, p): """ compound_statement : brace_open block_item_list_opt brace_close """ p[0] = c_ast.Compound( diff --git a/tests/test_basic_parse.py b/tests/test_basic_parse.py index 1846926..cc3a4fd 100644 --- a/tests/test_basic_parse.py +++ b/tests/test_basic_parse.py @@ -104,8 +104,13 @@ def test_metadata(self): def test_metadata_with_quoted_strings(self): res = parse_string(""" - local int a ">; - """, optimize=True) + local int asdfsdf there">; + """, optimize=True, predefine_types=False) + + node = res.children()[0][1] + self.assertEqual(len(node.metadata.values), 1) + self.assertEqual(node.metadata.values[0][0], "comment") + self.assertEqual(node.metadata.values[0][1], '"Hello > there"') def test_typedef(self): res = parse_string("""