From 7a2ff8aa77ff52fd710f53283dad9540c963db3e Mon Sep 17 00:00:00 2001 From: Daniel Podolsky Date: Sat, 2 Mar 2019 12:51:21 +0300 Subject: [PATCH 1/4] improvement: make generated code safer --- internal/parser/gen/golang/actiontable.go | 17 +++++++++++++---- internal/token/gen/golang/token.go | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/internal/parser/gen/golang/actiontable.go b/internal/parser/gen/golang/actiontable.go index 66192427..2d96723b 100644 --- a/internal/parser/gen/golang/actiontable.go +++ b/internal/parser/gen/golang/actiontable.go @@ -115,16 +115,16 @@ func getActionRowData(prods ast.SyntaxProdList, set *items.ItemSet, tokMap *toke switch act1 := act.(type) { case action.Accept: pad := max + 1 - len("accept(true),") - data.Actions[i] = fmt.Sprintf("accept(true),%*c/* %s */", pad, ' ', sym) + data.Actions[i] = fmt.Sprintf("accept(true),%*c/* %s */", pad, ' ', safeSym(sym)) case action.Error: pad := max + 1 - len("nil,") - data.Actions[i] = fmt.Sprintf("nil,%*c/* %s */", pad, ' ', sym) + data.Actions[i] = fmt.Sprintf("nil,%*c/* %s */", pad, ' ', safeSym(sym)) case action.Reduce: pad := max + 1 - (len("reduce(") + nbytes(int(act1)) + len("),")) - data.Actions[i] = fmt.Sprintf("reduce(%d),%*c/* %s, reduce: %s */", int(act1), pad, ' ', sym, prods[int(act1)].Id) + data.Actions[i] = fmt.Sprintf("reduce(%d),%*c/* %s, reduce: %s */", int(act1), pad, ' ', safeSym(sym), prods[int(act1)].Id) case action.Shift: pad := max + 1 - (len("shift(") + nbytes(int(act1)) + len("),")) - data.Actions[i] = fmt.Sprintf("shift(%d),%*c/* %s */", int(act1), pad, ' ', sym) + data.Actions[i] = fmt.Sprintf("shift(%d),%*c/* %s */", int(act1), pad, ' ', safeSym(sym)) default: panic(fmt.Sprintf("Unknown action type: %T", act1)) } @@ -132,6 +132,15 @@ func getActionRowData(prods ast.SyntaxProdList, set *items.ItemSet, tokMap *toke return } +func safeSym(sym string) string { + switch sym { + case `*/`: + return "special case: closing multiline comment sequence" + } + + return sym +} + const actionTableSrc = ` // Code generated by gocc; DO NOT EDIT. diff --git a/internal/token/gen/golang/token.go b/internal/token/gen/golang/token.go index 86fd2981..6beed79f 100644 --- a/internal/token/gen/golang/token.go +++ b/internal/token/gen/golang/token.go @@ -107,7 +107,7 @@ func (m TokenMap) Type(tok string) Type { func (m TokenMap) TokenString(tok *Token) string { //TODO: refactor to print pos & token string properly - return fmt.Sprintf("%s(%d,%s)", m.Id(tok.Type), tok.Type, tok.Lit) + return fmt.Sprintf("%s(%d,%q)", m.Id(tok.Type), tok.Type, tok.Lit) } func (m TokenMap) StringType(typ Type) string { From 1f04dba411f8f445f259ba6bbabe09d1653a476e Mon Sep 17 00:00:00 2001 From: Daniel Podolsky Date: Fri, 8 Mar 2019 09:46:38 +0300 Subject: [PATCH 2/4] improvement: some explanation comments --- internal/parser/gen/golang/actiontable.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/parser/gen/golang/actiontable.go b/internal/parser/gen/golang/actiontable.go index 2d96723b..b949b2d1 100644 --- a/internal/parser/gen/golang/actiontable.go +++ b/internal/parser/gen/golang/actiontable.go @@ -134,6 +134,7 @@ func getActionRowData(prods ast.SyntaxProdList, set *items.ItemSet, tokMap *toke func safeSym(sym string) string { switch sym { + // avoid `/* */ */` comment causing compilation error case `*/`: return "special case: closing multiline comment sequence" } From 5fbf58b41a6141ab814a58a425d7de5a16cca120 Mon Sep 17 00:00:00 2001 From: Daniel Podolsky Date: Sat, 2 Mar 2019 22:25:02 +0300 Subject: [PATCH 3/4] development: current entity name can be provided to DST function --- internal/frontend/token/token.go | 33 ++++--------------- .../parser/gen/golang/productionstable.go | 8 ++++- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/internal/frontend/token/token.go b/internal/frontend/token/token.go index f30eb7a0..a200e565 100644 --- a/internal/frontend/token/token.go +++ b/internal/frontend/token/token.go @@ -91,33 +91,14 @@ func (T *Token) UintValue() (uint64, error) { return strconv.ParseUint(string(T.Lit), 10, 64) } +var ( + sdtPlaceholders = regexp.MustCompile(`\$(\d+)`) + sdtReplacement = []byte("X[$1]") +) + func (T *Token) SDTVal() string { - sdt := string(T.Lit) - rex, err := regexp.Compile("\\$[0-9]+") - if err != nil { - panic(err) - } - idx := rex.FindAllStringIndex(sdt, -1) - res := "" - if len(idx) <= 0 { - res = sdt - } else { - for i, loc := range idx { - if loc[0] > 0 { - if i > 0 { - res += sdt[idx[i-1][1]:loc[0]] - } else { - res += sdt[0:loc[0]] - } - } - res += "X[" - res += sdt[loc[0]+1 : loc[1]] - res += "]" - } - if idx[len(idx)-1][1] < len(sdt) { - res += sdt[idx[len(idx)-1][1]:] - } - } + res := string(sdtPlaceholders.ReplaceAll(T.Lit, sdtReplacement)) + return strings.TrimSpace(res[2 : len(res)-2]) } diff --git a/internal/parser/gen/golang/productionstable.go b/internal/parser/gen/golang/productionstable.go index cbcce7a4..d384e66c 100644 --- a/internal/parser/gen/golang/productionstable.go +++ b/internal/parser/gen/golang/productionstable.go @@ -18,6 +18,7 @@ import ( "bytes" "fmt" "path" + "regexp" "text/template" "github.com/goccmack/gocc/internal/ast" @@ -42,6 +43,8 @@ func GenProductionsTable(pkg, outDir, header string, prods ast.SyntaxProdList, s io.WriteFile(fname, wr.Bytes()) } +var sdtTokenName = regexp.MustCompile(`\$ยง`) + func getProdsTab(header string, prods ast.SyntaxProdList, symbols *symbols.Symbols, itemsets *items.ItemSets, tokMap *token.TokenMap) *prodsTabData { @@ -61,7 +64,10 @@ func getProdsTab(header string, prods ast.SyntaxProdList, symbols *symbols.Symbo } switch { case len(prod.Body.SDT) > 0: - data.ProdTab[i].ReduceFunc = fmt.Sprintf("return %s", prod.Body.SDT) + data.ProdTab[i].ReduceFunc = fmt.Sprintf( + "return %s", + sdtTokenName.ReplaceAllString(prod.Body.SDT, fmt.Sprintf("%q", prod.Id)), + ) case isEmpty: // Empty production with no semantic action. data.ProdTab[i].ReduceFunc = "return nil, nil" From 5ae033d1b27901b65e52fe0ca68fbc8a732b8ec5 Mon Sep 17 00:00:00 2001 From: Daniel Podolsky Date: Sat, 9 Mar 2019 11:13:40 +0300 Subject: [PATCH 4/4] development: pass all the tokens to SDT function --- internal/frontend/token/token.go | 16 +++++++++++++--- internal/parser/gen/golang/errors.go | 2 +- internal/parser/gen/golang/parser.go | 2 +- internal/parser/gen/golang/productionstable.go | 8 ++++++++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/internal/frontend/token/token.go b/internal/frontend/token/token.go index a200e565..6876c939 100644 --- a/internal/frontend/token/token.go +++ b/internal/frontend/token/token.go @@ -92,12 +92,22 @@ func (T *Token) UintValue() (uint64, error) { } var ( - sdtPlaceholders = regexp.MustCompile(`\$(\d+)`) - sdtReplacement = []byte("X[$1]") + sdtPlaceholdersNum = regexp.MustCompile(`\$(\d+)`) + sdtReplacementNum = []byte(`X[$1]`) + sdtPlaceholdersAll = regexp.MustCompile(`\$\*`) + sdtReplacementAll = []byte(`attribsSliceToEmpyInterfaceSlice(X)...`) ) func (T *Token) SDTVal() string { - res := string(sdtPlaceholders.ReplaceAll(T.Lit, sdtReplacement)) + res := string( + sdtPlaceholdersNum.ReplaceAll( + sdtPlaceholdersAll.ReplaceAll( + T.Lit, + sdtReplacementAll, + ), + sdtReplacementNum, + ), + ) return strings.TrimSpace(res[2 : len(res)-2]) } diff --git a/internal/parser/gen/golang/errors.go b/internal/parser/gen/golang/errors.go index a4345fe4..965ba7cc 100644 --- a/internal/parser/gen/golang/errors.go +++ b/internal/parser/gen/golang/errors.go @@ -39,8 +39,8 @@ const errorsSrc = ` package errors import ( - "bytes" "fmt" + "strings" "{{.}}" ) diff --git a/internal/parser/gen/golang/parser.go b/internal/parser/gen/golang/parser.go index 81fe3f19..f417cd44 100644 --- a/internal/parser/gen/golang/parser.go +++ b/internal/parser/gen/golang/parser.go @@ -64,8 +64,8 @@ const parserSrc = ` package parser import ( - "bytes" "fmt" + "strings" parseError "{{.ErrorImport}}" "{{.TokenImport}}" diff --git a/internal/parser/gen/golang/productionstable.go b/internal/parser/gen/golang/productionstable.go index d384e66c..30e5d1a7 100644 --- a/internal/parser/gen/golang/productionstable.go +++ b/internal/parser/gen/golang/productionstable.go @@ -128,4 +128,12 @@ var productionsTable = ProdTab{ }, {{- end }} } + +func attribsSliceToEmpyInterfaceSlice(X []Attrib) []interface{} { + res := make([]interface{}, 0, len(X)) + for _, attrib := range X { + res = append(res, attrib) + } + return res +} `