Initial commit

.gitignore

@@ -0,0 +1,7 @@
+y2j
+!y2j/
+y2j-linux
+y2j-mac
+y2j.exe
+y.output
+yara-parser

Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.10
+
+RUN DEBIAN_FRONTEND=noninteractive \
+	apt update && apt install -y \
+		automake \
+		bison \
+		help2man \
+		m4 \
+		texinfo \
+		texlive
+
+RUN go get golang.org/x/tools/cmd/goyacc
+RUN go get github.com/pebbe/flexgo/...
+
+ENV FLEXGO=/go/src/github.com/pebbe/flexgo
+
+RUN cd ${FLEXGO} && ./configure && cd -
+RUN make -C ${FLEXGO} && make -C ${FLEXGO} install

LICENSES_THIRD_PARTIES

@@ -0,0 +1,23 @@
+- yara-parser (https://github.com/Northern-Lights/yara-parser)
+
+MIT License
+
+Copyright (c) 2018 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,21 @@
+all: grammar y2j
+
+builder:
+	docker build -t go-yara-parser-builder .
+
+grammar:
+	docker run --rm -v ${PWD}/grammar:/grammar go-yara-parser-builder bash -c 'flexgo -G -v -o /grammar/lexer.go /grammar/lexer.l && goyacc -p xx -o /grammar/parser.go /grammar/grammar.y'
+
+j2y:
+	go build github.com/VirusTotal/go-yara-parser/cmd/j2y
+
+y2j:
+	go build github.com/VirusTotal/go-yara-parser/cmd/y2j
+
+release: parser lexer
+	GOOS=linux go build -o y2j-linux github.com/VirusTotal/go-yara-parser/cmd/y2j
+	GOOS=darwin go build -o y2j-mac github.com/VirusTotal/go-yara-parser/cmd/y2j
+	GOOS=windows go build -o y2j.exe github.com/VirusTotal/go-yara-parser/cmd/y2j
+
+clean:
+	rm grammar/lexer.go grammar/parser.go y.output y2j

README.md

@@ -0,0 +1,84 @@
+# go-yara-parser
+
+`go-yara-parser` is a Go library for manipulating YARA rulesets.  Its key feature is that it uses the same grammar and lexer files as the original libyara to ensure that lexing and parsing work exactly like YARA.  The grammar and lexer files have been modified to fill Go data structures for ruleset manipulation instead of compiling rulesets for data matching.
+
+Using `go-yara-parser`, one will be able to read YARA rulesets to programatically change metadata, rule names, rule modifiers, tags, strings, and more.
+
+The ability to serialize rulesets to JSON for rule manipulation in other languages is provided with the `y2j` tool.  Similarly, `j2y` provides JSON-to-YARA conversion, but do see __Limitations__ below.
+
+## `y2j` Usage
+
+Command line usage for `y2j` looks like the following:
+
+```
+$ y2j --help            
+Usage of y2j: y2j [options] file.yar
+
+options:
+  -indent int
+        Set number of indent spaces (default 2)
+  -o string               
+        JSON output file
+```
+
+Note that the string types are as follows:
+
+| String type `int` code | Designation |
+| - | - |
+| 0 | string |
+| 1 | hex pair bytes |
+| 2 | regex |
+
+## Go Usage
+
+Sample usage for working with rulesets in Go looks like the following:
+
+```go
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/VirusTotal/go-yara-parser/grammar"
+)
+
+func main() {
+	input, err := os.Open(os.Args[1])   // Single argument: path to your file
+	if err != nil {
+		log.Fatalf("Error: %s\n", err)
+	}
+
+	ruleset, err := grammar.Parse(input, os.Stdout)
+	if err != nil {
+		log.Fatalf(`Parsing failed: "%s"`, err)
+	}
+
+    fmt.Printf("Ruleset:\n%v\n", ruleset)
+
+    // Manipulate the first rule
+    rule := ruleset.Rules[0]
+    rule.Identifier = "new_rule_name"
+    rule.Modifiers.Global = true
+    rule.Modifiers.Private = false
+}
+```
+
+## Development
+
+The included Dockerfile will build an image suitable for producing the parser and lexer using goyacc and flexgo.  There is a `builder` target in the `Makefile` to help you quickly get started with this.  Run the following to build the builder image:
+
+`make builder`
+
+This will provide you with a Docker image called `go-yara-parser-builder`.
+
+As you make changes to the grammar, you can then run `make grammar`.  The .go files will be output in the `grammar/` directory.
+
+## Limitations
+
+Currently, there are no guarantees with the library that modified rules will serialize back into a valid YARA ruleset.  For example, you can set `rule.Identifier = "123"`, but this would be invalid YARA. Additionally, adding or removing strings may cause a condition to become invalid, and conditions are currently treated only as text.  Comments also cannot be retained.
+
+## License and third party code
+
+This project uses code from [`yara-parser`](https://github.com/Northern-Lights/yara-parser) by [Northern-Lights](https://github.com/Northern-Lights), which is available under the MIT license (see `LICENSES_THIRD_PARTIES`).

cmd/j2y/errors.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+// perror writes a format string and args to stderr
+func perror(s string, a ...interface{}) {
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf(s, a...))
+	sb.WriteRune('\n')
+	os.Stderr.WriteString(sb.String())
+}
+
+// handleErr should be deferred to report any errors in deferred functions
+func handleErr(f func() error) {
+	err := f()
+	if err != nil {
+		perror(`Error: %s`, err)
+		os.Exit(127)
+	}
+}

cmd/j2y/main.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/VirusTotal/go-yara-parser/data"
+)
+
+// global options
+var opts options
+
+func main() {
+	opts = getopt()
+
+	jsonFile, err := os.Open(opts.Infile)
+	if err != nil {
+		perror(`Couldn't open JSON file "%s": %s`, opts.Infile, err)
+		os.Exit(2)
+	}
+	defer handleErr(jsonFile.Close)
+
+	var ruleset data.RuleSet
+	err = json.NewDecoder(jsonFile).Decode(&ruleset)
+	if err != nil {
+		perror(`Couldn't JSON decode file: %s`, err)
+		os.Exit(3)
+	}
+
+	// Set output to stdout if not specified; otherwise file
+	var out io.Writer
+	if opts.Outfile == "" {
+		out = os.Stdout
+	} else {
+		f, err := os.Create(opts.Outfile)
+		if err != nil {
+			perror(`Couldn't create output file "%s"`, opts.Outfile)
+			os.Exit(5)
+		}
+		defer handleErr(f.Close)
+		out = f
+	}
+
+	txt, err := ruleset.Serialize()
+	if err != nil {
+		perror(`Couldn't serialize ruleset: %s`, err)
+		os.Exit(6)
+	}
+
+	_, err = out.Write([]byte(txt))
+	if err != nil {
+		perror(`Error writing YARA: %s`, err)
+		os.Exit(6)
+	}
+}

cmd/j2y/opts.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"strings"
+)
+
+type options struct {
+	Indent  string
+	Infile  string
+	Outfile string
+}
+
+func getopt() options {
+	var (
+		o      options
+		indent int
+	)
+
+	flag.IntVar(&indent, "indent", 2, "Set number of indent spaces")
+	flag.StringVar(&o.Outfile, "o", "", "YARA output file")
+
+	flag.Parse()
+
+	// Set indent
+	var sb strings.Builder
+	for i := 0; i < indent; i++ {
+		sb.WriteRune(' ')
+	}
+	o.Indent = sb.String()
+
+	// The JSON file is the only positional argument
+	if n := flag.NArg(); n != 1 {
+		perror("Expected 1 input file; found %d", n)
+		os.Exit(1)
+	}
+
+	o.Infile = flag.Args()[0]
+
+	return o
+}

cmd/y2j/errors.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+// perror writes a format string and args to stderr
+func perror(s string, a ...interface{}) {
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf(s, a...))
+	sb.WriteRune('\n')
+	os.Stderr.WriteString(sb.String())
+}
+
+// handleErr should be deferred to report any errors in deferred functions
+func handleErr(f func() error) {
+	err := f()
+	if err != nil {
+		perror(`Error: %s`, err)
+		os.Exit(127)
+	}
+}

cmd/y2j/main.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/VirusTotal/go-yara-parser/grammar"
+)
+
+// global options
+var opts options
+
+func main() {
+	opts = getopt()
+
+	yaraFile, err := os.Open(opts.Infile)
+	if err != nil {
+		perror(`Couldn't open YARA file "%s": %s`, opts.Infile, err)
+		os.Exit(2)
+	}
+	defer handleErr(yaraFile.Close)
+
+	ruleset, err := grammar.Parse(yaraFile, os.Stdout)
+	if err != nil {
+		perror(`Couldn't parse YARA ruleset: %s`, err)
+		os.Exit(3)
+	}
+	ruleset.File = opts.Infile
+
+	// Set output to stdout if not specified; otherwise file
+	var out io.Writer
+	if opts.Outfile == "" {
+		out = os.Stdout
+	} else {
+		f, err := os.Create(opts.Outfile)
+		if err != nil {
+			perror(`Couldn't create output file "%s"`, opts.Outfile)
+			os.Exit(5)
+		}
+		defer handleErr(f.Close)
+		out = f
+	}
+
+	enc := json.NewEncoder(out)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", opts.Indent)
+	err = enc.Encode(&ruleset)
+	if err != nil {
+		perror(`Error writing JSON: %s`, err)
+		os.Exit(6)
+	}
+}