Skip to content

Commit

Permalink
Support percent notation in grammar. (VirusTotal#43)
Browse files Browse the repository at this point in the history
Add support for "x% of them" notation to gyp.

While I was here I noticed that the protobuf generated code was being put into
the github.com/VirusTotal/gyp/pb directory, which was causing various problems
when trying to build things as they all expected to find the protobuf related
code in the pb/ directory. To deal with this I modified the Makefile so that the
protobuf generated code is going in the pb/ directory yet the go_package remains
as github.com/VirusTotal/gyp/pb. I'm not sure if this is going to cause problems
with builds of other things using bazel that depend upon this so it may be good
to get @Zohiartze to review this so we can avoid a repeat of VirusTotal#42.

* Rework the percentage handling.

Address comments from Victor, which I understand conceptually but may have
implemented in a weird way. ;)
  • Loading branch information
wxsBSD authored Dec 10, 2021
1 parent 5952a9a commit 51d2ad1
Show file tree
Hide file tree
Showing 9 changed files with 828 additions and 619 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ hexgrammar:
flexgo -G -v -o hex/hex_lexer.go hex/hex_lexer.l && goyacc -p hex -o hex/hex_parser.go hex/hex_grammar.y

proto:
protoc --go_out=. pb/yara.proto
protoc --go_out=. --go_opt=paths=source_relative pb/yara.proto

j2y:
go build github.com/VirusTotal/gyp/cmd/j2y
Expand Down
45 changes: 39 additions & 6 deletions ast/ast.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,12 @@ type Quantifier struct {
Expression
}

// Percentage is an Expression used in evaluating string sets. Example:
// <expression>% of <string set>
type Percentage struct {
Expression
}

// ForIn is an Expression representing a "for in" loop. Example:
// for <quantifier> <variables> in <iterator> : ( <condition> )
type ForIn struct {
Expand Down Expand Up @@ -499,6 +505,15 @@ func (o *Of) WriteSource(w io.Writer) error {
return err
}

// WriteSource writes the node's source into the writer w.
func (p *Percentage) WriteSource(w io.Writer) error {
err := p.Expression.WriteSource(w)
if err == nil {
_, err = io.WriteString(w, "%")
}
return err
}

// WriteSource writes the operation into the writer w.
func (o *Operation) WriteSource(w io.Writer) error {
if len(o.Operands) < 2 {
Expand Down Expand Up @@ -892,26 +907,44 @@ func (s *Subscripting) AsProto() *pb.Expression {
}
}

// AsProto returns the Expression serialized as a pb.Expression.
func (p *Percentage) AsProto() *pb.Expression {
return &pb.Expression{
Expression: &pb.Expression_PercentageExpression{
PercentageExpression: &pb.Percentage{
Expression: p.Expression.AsProto(),
},
},
}
}

// AsProto returns the Expression serialized as a pb.Expression.
func (q *Quantifier) AsProto() *pb.ForExpression {
var expr *pb.ForExpression
if kw, isKeyword := q.Expression.(Keyword); isKeyword {
switch v := q.Expression.(type) {
case *Percentage:
expr = &pb.ForExpression{
For: &pb.ForExpression_Expression{
Expression: v.AsProto(),
},
}
case Keyword:
var pbkw pb.ForKeyword
if kw == KeywordAll {
if v == KeywordAll {
pbkw = pb.ForKeyword_ALL
} else if kw == KeywordAny {
} else if v == KeywordAny {
pbkw = pb.ForKeyword_ANY
} else if kw == KeywordNone {
} else if v == KeywordNone {
pbkw = pb.ForKeyword_NONE
} else {
panic(fmt.Sprintf("unexpected keyword in for: %s", kw))
panic(fmt.Sprintf("unexpected keyword in for: %s", v))
}
expr = &pb.ForExpression{
For: &pb.ForExpression_Keyword{
Keyword: pbkw,
},
}
} else {
default:
expr = &pb.ForExpression{
For: &pb.ForExpression_Expression{
Expression: q.Expression.AsProto(),
Expand Down
4 changes: 4 additions & 0 deletions ast/serialization.go
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,10 @@ func expressionFromProto(e *pb.Expression) Expression {
default:
panic(fmt.Sprintf(`unknown keyword "%T"`, keyword))
}
case *pb.Expression_PercentageExpression:
return &Percentage{
Expression: expressionFromProto(v.PercentageExpression.Expression),
}
default:
panic(fmt.Sprintf(`unexpected node "%T"`, v))
}
Expand Down
17 changes: 17 additions & 0 deletions parser/grammar.y
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ type stringModifiers struct {
%type <exprs> arguments_list
%type <exprs> arguments
%type <quantifier> for_expression
%type <percentage> percent_expression
%type <exprs> integer_enumeration
%type <node> iterator
%type <node> integer_set
Expand Down Expand Up @@ -190,6 +191,7 @@ type stringModifiers struct {
si *ast.StringIdentifier
sis []*ast.StringIdentifier
quantifier *ast.Quantifier
percentage *ast.Percentage

// lineno is not a symbol type, it's the line number where the symbol
// appears in the source file. This is a little hack used for passing
Expand Down Expand Up @@ -824,6 +826,13 @@ expression
Strings: $3,
}
}
| percent_expression _OF_ string_set
{
$$ = &ast.Of{
Quantifier: &ast.Quantifier{$1},
Strings: $3,
}
}
| _NOT_ boolean_expression
{
$$ = &ast.Not{$2}
Expand Down Expand Up @@ -964,6 +973,14 @@ string_enumeration_item
;


percent_expression
: primary_expression '%'
{
$$ = &ast.Percentage{$1}
}
;


for_expression
: primary_expression
{
Expand Down
Loading

0 comments on commit 51d2ad1

Please sign in to comment.