Closing steffenfritz#82: Added YARA support via flag. Fixed also a bu…

…g in EXIF metadata extraction

cmd/ftrove/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/hex"
 	"fmt"
+	yara_x "github.com/VirusTotal/yara-x/go"
 	"io"
 	"log/slog"
 	"os"
@@ -41,7 +42,7 @@ func main() {
 	// createThumbsImages :=
 	// createStillsVideo :=
 	// getTextfileIdea :=
-	// grepYARA :=
+	grepYARA := flag.StringP("yararules", "y", "", "Provide a YARA rule file and scan all files for matches.")
 	dublincore := flag.StringP("dublincore", "d", "", "Add DublinCore metadata as a JSON file for a session (not single files).")
 	exifData := flag.BoolP("exifdata", "e", false, "Get some EXIF metadata from image files.")
 	exportSessionToTSV := flag.StringP("export-tsv", "t", "", "Export a session from the database to a TSV file. Provide the session uuid.")
@@ -80,6 +81,10 @@ func main() {
 	if len(*dublincore) > 0 {
 		sessionmd.Dublincoreflag = "True"
 	}
+	if len(*grepYARA) > 0 {
+		sessionmd.Yaraflag = "True"
+		sessionmd.Yarasource = *grepYARA
+	}
 
 	// Print banner or version on startup
 	ft.PrintBanner()
@@ -208,10 +213,12 @@ func main() {
 			logger.Error("Error while exporting files from session to TSV file.", slog.String("error", err.Error()))
 			os.Exit(1)
 		}
-		// DOC: Value 6 MUST be the flag result of EXIF. We translate for clarity.
-		exifFlagSet := sessionValues[6]
-		// DOC: Value 7 MUST be the flag result of DublinCore. We translate for clarity.
-		dcFlagSet := sessionValues[7]
+		// DOC: Value 7 MUST be the flag result of EXIF. We translate for clarity.
+		exifFlagSet := sessionValues[7]
+		// DOC: Value 8 MUST be the flag result of DublinCore. We translate for clarity.
+		dcFlagSet := sessionValues[8]
+		// DOC: Value 9 MUST be the flag result of YARA. We translate for clarity.
+		yaraFlagSet := sessionValues[9]
 
 		err = ft.ExportSessionFilesTSV(*exportSessionToTSV)
 		if err != nil {
@@ -241,6 +248,15 @@ func main() {
 
 		}
 
+		if yaraFlagSet == "True" {
+			err = ft.ExportYaraTSV(*exportSessionToTSV)
+			if err != nil {
+				logger.Error("Error while exporting YARA identified files from session to TSV file.", slog.String("error", err.Error()))
+				os.Exit(1)
+			}
+
+		}
+
 		logger.Info("Export successful.")
 		return
 	}
@@ -322,6 +338,8 @@ func main() {
 
 		// Overwrite the NSRL counter
 		nsrlcount = ri.NSRLFiles
+
+		// ToDo: Get Yara information for resuming sessions
 	}
 
 	// Prepare statement to add file scan results to database
@@ -376,6 +394,23 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Compile YARA rule if flag provided, used later
+	var checkYara bool
+	var yaraRules *yara_x.Rules
+	prepYaraInsert, err := ft.PrepInsertYara(ftdb)
+	if err != nil {
+		logger.Error("Could not prepare an insert statement for YARA inserts.", slog.String("error", err.Error()))
+	}
+
+	if len(*grepYARA) > 0 {
+		checkYara = true
+		yaraRules, err = ft.YaraCompile(*grepYARA)
+		if err != nil {
+			logger.Error("Could not compile the YARA rules.", slog.String("error", err.Error()))
+			os.Exit(1)
+		}
+	}
+
 	// Inspect every file in filelist
 	// Set up the progress bar
 	bar := progressbar.Default(int64(len(filelist)))
@@ -504,6 +539,28 @@ func main() {
 			logger.Warn("Could not add file entry into FileTrove database. File: "+file, slog.String("warn", err.Error()))
 		}
 
+		// Check for YARA rules
+		if checkYara {
+			matches, err := ft.YaraScan(yaraRules, file)
+			if err != nil {
+				logger.Warn("Could not scan YARA rules", slog.String("error", err.Error()))
+			}
+
+			if len(matches) > 0 {
+				for _, match := range matches {
+					yarauuid, err := ft.CreateUUID()
+					if err != nil {
+						logger.Warn("Could not create UUID for YARA rule", slog.String("error", err.Error()))
+					}
+					_, err = prepYaraInsert.Exec(yarauuid, sessionmd.UUID, fileuuid, match.Identifier())
+					if err != nil {
+						logger.Warn("Could not add YARA identification", slog.String("error", err.Error()))
+					}
+				}
+			}
+
+		}
+
 		filecount += 1
 		bar.Add(1)
 	}
@@ -565,6 +622,8 @@ func main() {
 			logger.Error("Could not get image list from database.", slog.String("error", err.Error()))
 		}
 		for fileuuid, imagepath := range imagelist {
+			// debug
+			println(imagepath)
 
 			exifparsed, err := ft.ExifDecode(imagepath)
 			if err != nil {

database_schema.dbml

@@ -13,6 +13,8 @@ Table sessionsmd {
   mountpoint TEXT
   exifflag TEXT
   dublincoreflag TEXT
+  yaraflag TEXT
+  yarasource TEXT
   filetroveversion TEXT
   filetrovedbversion TEXT
   nsrlversion TEXT
@@ -93,8 +95,17 @@ TABLE exif{
   xpsubject TEXT
 }
 
+TABLE yara{
+  yaraentryuuid TEXT
+  sessionuuid TEXT
+  fileuuid TEXT
+  rulename TEXT
+}
+
 Ref: files.sessionuuid > sessionsmd.uuid
 Ref: exif.sessionuuid > sessionsmd.uuid
 Ref: exif.fileuuid > files.fileuuid
 Ref: directories.sessionuuid > sessionsmd.uuid
 Ref: dublincore.sessionuuid > sessionsmd.uuid
+Ref: yara.sessionuuid > sessionsmd.uuid
+Ref: yara.fileuuid > files.fileuuid

db.go

@@ -26,6 +26,8 @@ type SessionMD struct {
 	Pathseparator      string
 	ExifFlag           string
 	Dublincoreflag     string
+	Yaraflag           string
+	Yarasource         string
 	Filetroveversion   string
 	Nsrlversion        string
 	Sfversion          string
@@ -113,6 +115,8 @@ func CreateFileTroveDB(dbpath string, version string, initdate string) error {
 						pathseparator TEXT,
 					   	exifflag TEXT,
 					   	dublincoreflag TEXT,
+						yaraflag TEXT,
+						yarasource TEXT,
 						filetroveversion TEXT,
 						filetrovedbversion TEXT,
 						nsrlversion TEXT,
@@ -183,7 +187,11 @@ func CreateFileTroveDB(dbpath string, version string, initdate string) error {
                          xpauthor TEXT,
                          xpkeywords TEXT,
                          xpsubject TEXT
-                         );`
+                         );
+						CREATE TABLE yara(yaraentryuuid TEXT,
+						  sessionuuid TEXT,
+						  fileuuid TEXT,
+						  rulename TEXT);`
 
 	_, err = db.Exec(initstatements)
 	if err != nil {
@@ -211,8 +219,8 @@ func ConnectFileTroveDB(dbpath string) (*sql.DB, error) {
 
 // InsertSession adds session metadata to the database
 func InsertSession(db *sql.DB, s SessionMD) error {
-	_, err := db.Exec("INSERT INTO sessionsmd VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", s.UUID, s.Starttime, nil, s.Project,
-		s.Archivistname, s.Mountpoint, s.Pathseparator, s.ExifFlag, s.Dublincoreflag, s.Filetroveversion, s.Filetrovedbversion,
+	_, err := db.Exec("INSERT INTO sessionsmd VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", s.UUID, s.Starttime, nil, s.Project,
+		s.Archivistname, s.Mountpoint, s.Pathseparator, s.ExifFlag, s.Dublincoreflag, s.Yaraflag, s.Yarasource, s.Filetroveversion, s.Filetrovedbversion,
 		s.Nsrlversion, s.Sfversion, s.Goversion)
 
 	return err
@@ -241,6 +249,13 @@ func PrepInsertDir(db *sql.DB) (*sql.Stmt, error) {
 	return prepin, err
 }
 
+// PrepInsertYara prepares a statement for the addition of a matching YARA rule
+func PrepInsertYara(db *sql.DB) (*sql.Stmt, error) {
+	prepin, err := db.Prepare("INSERT INTO yara VALUES(?,?,?,?)")
+
+	return prepin, err
+}
+
 // ListSessions lists all sessions from the FileTrove database
 func ListSessions(db *sql.DB) error {
 	rows, err := db.Query("SELECT rowid, uuid, starttime, COALESCE(endtime, '') AS endtime, " +
@@ -621,7 +636,7 @@ func ExportSessionEXIFTSV(sessionuuid string) error {
 	return nil
 }
 
-// ExportSessionDCTSV exports all exif metadata from a session to a TSV file. Filtering is done by session UUID.
+// ExportSessionDCTSV exports all Dublin Core metadata from a session to a TSV file. Filtering is done by session UUID.
 func ExportSessionDCTSV(sessionuuid string) error {
 	db, err := sql.Open("sqlite3", filepath.Join("db", "filetrove.db"))
 	if err != nil {
@@ -697,12 +712,88 @@ func ExportSessionDCTSV(sessionuuid string) error {
 	return nil
 }
 
+// ExportYaraTSV exports all files that matched YARA rules to a TSV file. Filtering is done by session UUID.
+func ExportYaraTSV(sessionuuid string) error {
+	db, err := sql.Open("sqlite3", filepath.Join("db", "filetrove.db"))
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	outputFile, err := os.Create(sessionuuid + "_yara.tsv")
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
+
+	tsvWriter := csv.NewWriter(outputFile)
+	tsvWriter.Comma = '\t'
+	defer tsvWriter.Flush()
+
+	query := "SELECT * FROM yara WHERE sessionuuid=\"" + sessionuuid + "\""
+	rows, err := db.Query(query)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	columns, err := rows.Columns()
+	if err != nil {
+		return err
+	}
+
+	if err := tsvWriter.Write(columns); err != nil {
+		return err
+	}
+
+	// Loop to create TSV headings from row names
+	values := make([]interface{}, len(columns))
+	scanArgs := make([]interface{}, len(columns))
+	for i := range values {
+		scanArgs[i] = &values[i]
+	}
+
+	for rows.Next() {
+		if err := rows.Scan(scanArgs...); err != nil {
+			return err
+		}
+		var valueStrings []string
+		for _, value := range values {
+			if value == nil {
+				valueStrings = append(valueStrings, "")
+			} else {
+				//valueStrings = append(valueStrings, value.(string))
+				switch v := value.(type) {
+				case int64:
+					valueStrings = append(valueStrings, strconv.FormatInt(v, 10))
+				case string:
+					valueStrings = append(valueStrings, string(v))
+				case float64:
+					valueStrings = append(valueStrings, strconv.FormatFloat(v, 'E', -1, 32))
+				default:
+					log.Printf("Unexpected Type: %s", reflect.TypeOf(value))
+					valueStrings = append(valueStrings, "")
+				}
+			}
+		}
+		if err := tsvWriter.Write(valueStrings); err != nil {
+			return err
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // GetImageFiles queries all files that have mime type image from a session
 func GetImageFiles(db *sql.DB, sessionuuid string) (map[string]string, error) {
-	var filename string
+	var filepath string
 	var fileuuid string
 
-	query := "SELECT filename, fileuuid FROM files where sessionuuid=\"" + sessionuuid +
+	query := "SELECT filepath, fileuuid FROM files where sessionuuid=\"" + sessionuuid +
 		"\" AND filesfmime=\"image/jpeg\" OR filesfmime=\"image/tiff\""
 
 	rows, err := db.Query(query)
@@ -714,10 +805,10 @@ func GetImageFiles(db *sql.DB, sessionuuid string) (map[string]string, error) {
 	imagelist := make(map[string]string)
 
 	for rows.Next() {
-		if err := rows.Scan(&filename, &fileuuid); err != nil {
+		if err := rows.Scan(&filepath, &fileuuid); err != nil {
 			return imagelist, err
 		}
-		imagelist[fileuuid] = filename
+		imagelist[fileuuid] = filepath
 	}
 	if err = rows.Err(); err != nil {
 		return imagelist, err

filewalk_test.go

@@ -17,8 +17,8 @@ func TestCreateFileList(t *testing.T) {
 		wantErr bool
 	}{
 		{"testdata file list", args{"testdata"}, []string{"testdata/.hiddenfile",
-			"testdata/directory/A_PDF_File.pdf", "testdata/dublincore_ex.json", "testdata/emptyfile.txt", "testdata/images/screenshot_1.jpeg", "testdata/images/screenshot_1.jpeg_original", "testdata/images/screenshot_1.png", "testdata/images/screenshot_1.png_original", "testdata/images/screenshot_1.tiff", "testdata/images/screenshot_1.tiff_original", "testdata/noaccess.rtf", "testdata/noextension", "testdata/textfile.txt", "testdata/transparent.png", "testdata/white.jpg"},
-			[]string{"testdata", "testdata/.hiddendir", "testdata/directory", "testdata/images"}, false},
+			"testdata/directory/A_PDF_File.pdf", "testdata/dublincore_ex.json", "testdata/emptyfile.txt", "testdata/images/screenshot_1.jpeg", "testdata/images/screenshot_1.jpeg_original", "testdata/images/screenshot_1.png", "testdata/images/screenshot_1.png_original", "testdata/images/screenshot_1.tiff", "testdata/images/screenshot_1.tiff_original", "testdata/noaccess.rtf", "testdata/noextension", "testdata/textfile.txt", "testdata/transparent.png", "testdata/white.jpg", "testdata/yara/testrule.yara"},
+			[]string{"testdata", "testdata/.hiddendir", "testdata/directory", "testdata/images", "testdata/yara"}, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

go.mod

@@ -1,8 +1,11 @@
 module github.com/steffenfritz/FileTrove
 
-go 1.20
+go 1.21
+
+toolchain go1.22.3
 
 require (
+	github.com/VirusTotal/yara-x/go v0.3.0
 	github.com/djherbis/times v1.6.0
 	github.com/google/uuid v1.6.0
 	github.com/mattn/go-sqlite3 v1.14.22
@@ -28,4 +31,5 @@ require (
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -1,8 +1,12 @@
+github.com/VirusTotal/yara-x/go v0.3.0 h1:nkVmjJN3yUJLmxRb5tMbejen5AyNuC3x/lN578s3RJM=
+github.com/VirusTotal/yara-x/go v0.3.0/go.mod h1:lgXP/nkYX349MVowrtTtU5hzMdCOWQLv3+wKll9+0F8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
@@ -40,14 +44,16 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
 go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
 golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
 golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@@ -59,4 +65,9 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=