feat: implement connection for local fs and added mimetype detection

[skip ci]

checks/connection_filesystem.go

@@ -39,7 +39,7 @@ type FilesystemRW interface {
 	Write(ctx gocontext.Context, path string, data io.Reader) (os.FileInfo, error)
 }
 
-// s3FS implements connection.Filesystem for S3
+// s3FS implements FilesystemRW for S3
 type s3FS struct {
 	*s3.Client
 	Bucket string
@@ -130,7 +130,7 @@ func (t *s3FS) Write(ctx gocontext.Context, path string, data io.Reader) (os.Fil
 	return t.Stat(path)
 }
 
-// gcsFS implements connection.Filesystem for Google Cloud Storage
+// gcsFS implements FilesystemRW for Google Cloud Storage
 type gcsFS struct {
 	*gcs.Client
 	Bucket string
@@ -204,8 +204,59 @@ func (t *gcsFS) Write(ctx gocontext.Context, path string, data io.Reader) (os.Fi
 	return t.Stat(path)
 }
 
+// localFS implements FilesystemRW for local filesystem
+type localFS struct {
+	base string
+}
+
+func newLocalFS(base string) *localFS {
+	return &localFS{base: base}
+}
+
+func (t *localFS) Close() error {
+	return nil
+}
+
+func (t *localFS) ReadDir(name string) ([]os.FileInfo, error) {
+	return nil, nil // TODO:
+}
+
+func (t *localFS) Stat(name string) (os.FileInfo, error) {
+	return os.Stat(filepath.Join(t.base, name))
+}
+
+func (t *localFS) Read(ctx gocontext.Context, path string) (io.ReadCloser, error) {
+	return os.Open(filepath.Join(t.base, path))
+}
+
+func (t *localFS) Write(ctx gocontext.Context, path string, data io.Reader) (os.FileInfo, error) {
+	fullpath := filepath.Join(t.base, path)
+
+	// Ensure the directory exists
+	err := os.MkdirAll(filepath.Dir(fullpath), os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("error creating base directory: %w", err)
+	}
+
+	f, err := os.Create(fullpath)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = io.Copy(f, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return t.Stat(path)
+}
+
 func GetFSForConnection(ctx context.Context, c models.Connection) (FilesystemRW, error) {
 	switch c.Type {
+	case models.ConnectionTypeFolder:
+		path := c.Properties["path"]
+		return newLocalFS(path), nil
+
 	case models.ConnectionTypeAWS:
 		bucket := c.Properties["bucket"]
 		conn := connection.AWSConnection{

checks/runchecks.go

@@ -17,10 +17,13 @@ import (
 	"github.com/flanksource/canary-checker/pkg/utils"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty/models"
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/google/uuid"
 	gocache "github.com/patrickmn/go-cache"
 )
 
+const maxBytesForMimeDetection = 512 * 1024 // 512KB
+
 var checksCache = gocache.New(5*time.Minute, 5*time.Minute)
 
 var DisabledChecks []string
@@ -147,17 +150,25 @@ func saveArtifacts(ctx *context.Context, results pkg.Results) error {
 		}
 
 		for _, a := range r.Artifacts {
+			defer a.Content.Close()
+
 			checksum := sha256.New()
 			clonedReader := io.TeeReader(a.Content, checksum)
-			defer a.Content.Close()
+
+			mr := &mimeWriter{max: maxBytesForMimeDetection}
+			clonedReader2 := io.TeeReader(clonedReader, mr)
 
 			a.Path = filepath.Join("checks", checkID.String(), fmt.Sprintf("%d", r.Start.UnixNano()), a.Path)
-			info, err := fs.Write(ctx, a.Path, clonedReader)
+			info, err := fs.Write(ctx, a.Path, clonedReader2)
 			if err != nil {
 				logger.Errorf("error saving artifact to filestore: %v", err)
 				continue
 			}
 
+			if a.ContentType == "" {
+				a.ContentType = mr.Detect().String()
+			}
+
 			artifact := models.Artifact{
 				CheckID:      utils.Ptr(checkID),
 				CheckTime:    utils.Ptr(r.Start),
@@ -169,7 +180,7 @@ func saveArtifacts(ctx *context.Context, results pkg.Results) error {
 				Checksum:     hex.EncodeToString(checksum.Sum(nil)),
 			}
 			if err := ctx.DB().Create(&artifact).Error; err != nil {
-				logger.Errorf("error saving artifact to db: %v", err)
+				return fmt.Errorf("error saving artifact to db: %w", err)
 			}
 		}
 	}
@@ -260,3 +271,26 @@ func processTemplates(ctx *context.Context, r *pkg.CheckResult) *pkg.CheckResult
 
 	return r
 }
+
+// mimeWriter implements io.Writer with a limit on the number of bytes used for detection.
+type mimeWriter struct {
+	buffer []byte
+	max    int // max number of bytes to use from the source
+}
+
+func (t *mimeWriter) Write(bb []byte) (n int, err error) {
+	if len(t.buffer) > t.max {
+		return 0, nil
+	}
+
+	rem := t.max - len(t.buffer)
+	if rem > len(bb) {
+		rem = len(bb)
+	}
+	t.buffer = append(t.buffer, bb[:rem]...)
+	return rem, nil
+}
+
+func (t *mimeWriter) Detect() *mimetype.MIME {
+	return mimetype.Detect(t.buffer)
+}

go.mod

@@ -129,6 +129,7 @@ require (
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect

go.sum

@@ -834,6 +834,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
 github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=