add env variable for ingestor/grpc image

pkg/config/collector.go

@@ -50,6 +50,6 @@ type FileArchiveConfig struct {
 }
 
 type BlobConfig struct {
-	Bucket string `mapstructure:"bucket"` // Bucket to use to push k8s resources (e.g.: s3://<your_bucket>)
-	Region string `mapstructure:"region"` // Region to use for the bucket (only for s3)
+	BucketName string `mapstructure:"bucket_name"` // Bucket to use to push k8s resources (e.g.: s3://<your_bucket>)
+	Region     string `mapstructure:"region"`      // Region to use for the bucket (only for s3)
 }

pkg/config/config.go

@@ -108,12 +108,12 @@ func SetDefaultValues(v *viper.Viper) {
 	v.SetDefault(TelemetryEnabled, false)
 
 	// Default value for MongoDB
-	v.SetDefault("mongodb.url", DefaultMongoUrl)
-	v.SetDefault("mongodb.connection_timeout", DefaultConnectionTimeout)
+	v.SetDefault(MongoUrl, DefaultMongoUrl)
+	v.SetDefault(MongoConnectionTimeout, DefaultConnectionTimeout)
 
 	// Defaults values for JanusGraph
-	v.SetDefault("janusgraph.url", DefaultJanusGraphUrl)
-	v.SetDefault("janusgraph.connection_timeout", DefaultConnectionTimeout)
+	v.SetDefault(JanusGraphUrl, DefaultJanusGraphUrl)
+	v.SetDefault(JanusGrapTimeout, DefaultConnectionTimeout)
 
 	// Profiler values
 	v.SetDefault(TelemetryProfilerPeriod, DefaultProfilerPeriod)
@@ -149,6 +149,17 @@ func SetEnvOverrides(c *viper.Viper) {
 	res = multierror.Append(res, c.BindEnv("collector.file.directory", "KH_COLLECTOR_DIR"))
 	res = multierror.Append(res, c.BindEnv("collector.file.cluster", "KH_COLLECTOR_TARGET"))
 
+	res = multierror.Append(res, c.BindEnv(MongoUrl, "KH_MONGODB_URL"))
+	res = multierror.Append(res, c.BindEnv(JanusGraphUrl, "KH_JANUSGRAPH_URL"))
+
+	res = multierror.Append(res, c.BindEnv(IngestorAPIEndpoint, "KH_INGESTOR_API_ENDPOINT"))
+	res = multierror.Append(res, c.BindEnv(IngestorAPIInsecure, "KH_INGESTOR_API_INSECURE"))
+	res = multierror.Append(res, c.BindEnv(IngestorBlobBucketName, "KH_INGESTOR_BUCKET_NAME"))
+	res = multierror.Append(res, c.BindEnv(IngestorTempDir, "KH_INGESTOR_TEMP_DIR"))
+	res = multierror.Append(res, c.BindEnv(IngestorMaxArchiveSize, "KH_INGESTOR_MAX_ARCHIVE_SIZE"))
+	res = multierror.Append(res, c.BindEnv(IngestorArchiveName, "KH_INGESTOR_ARCHIVE_NAME"))
+	res = multierror.Append(res, c.BindEnv(IngestorBlobRegion, "KH_INGESTOR_REGION"))
+
 	if res.ErrorOrNil() != nil {
 		log.I.Fatalf("config environment override: %v", res.ErrorOrNil())
 	}
@@ -234,6 +245,8 @@ func NewEmbedConfig(v *viper.Viper, configPath string) (*KubehoundConfig, error)
 	v.SetConfigType(DefaultConfigType)
 	SetDefaultValues(v)
 
+	// Configure environment variable override
+	SetEnvOverrides(v)
 	data, err := embedconfig.F.ReadFile(configPath)
 	if err != nil {
 		return nil, fmt.Errorf("reading embed config: %w", err)

pkg/config/config_test.go

@@ -81,8 +81,8 @@ func TestMustLoadConfig(t *testing.T) {
 						Insecure: false,
 					},
 					Blob: &BlobConfig{
-						Bucket: "",
-						Region: "",
+						BucketName: "",
+						Region:     "",
 					},
 					TempDir:        "/tmp/kubehound",
 					ArchiveName:    "archive.tar.gz",
@@ -155,8 +155,8 @@ func TestMustLoadConfig(t *testing.T) {
 						Insecure: false,
 					},
 					Blob: &BlobConfig{
-						Bucket: "",
-						Region: "",
+						BucketName: "",
+						Region:     "",
 					},
 					TempDir:        "/tmp/kubehound",
 					ArchiveName:    "archive.tar.gz",

pkg/config/janusgraph.go

@@ -6,6 +6,9 @@ import (
 
 const (
 	DefaultJanusGraphUrl = "ws://localhost:8182/gremlin"
+
+	JanusGraphUrl    = "janusgraph.url"
+	JanusGrapTimeout = "janusgraph.connection_timeout"
 )
 
 // JanusGraphConfig configures JanusGraph specific parameters.

pkg/config/mongodb.go

@@ -6,6 +6,9 @@ import (
 
 const (
 	DefaultMongoUrl = "mongodb://localhost:27017"
+
+	MongoUrl               = "mongodb.url"
+	MongoConnectionTimeout = "mongodb.connection_timeout"
 )
 
 // MongoDBConfig configures mongodb specific parameters.

pkg/ingestor/puller/blob/blob.go

@@ -39,12 +39,12 @@ type BlobStore struct {
 var _ puller.DataPuller = (*BlobStore)(nil)
 
 func NewBlobStorage(cfg *config.KubehoundConfig, blobConfig *config.BlobConfig) (*BlobStore, error) {
-	if blobConfig.Bucket == "" {
+	if blobConfig.BucketName == "" {
 		return nil, ErrInvalidBucketName
 	}
 
 	return &BlobStore{
-		bucketName: blobConfig.Bucket,
+		bucketName: blobConfig.BucketName,
 		cfg:        cfg,
 		region:     blobConfig.Region,
 	}, nil

pkg/ingestor/puller/blob/blob_test.go

@@ -339,7 +339,7 @@ func TestNewBlobStorage(t *testing.T) {
 			name: "empty bucket name",
 			args: args{
 				blobConfig: &config.BlobConfig{
-					Bucket: "",
+					BucketName: "",
 				},
 				cfg: &config.KubehoundConfig{
 					Ingestor: config.IngestorConfig{
@@ -353,7 +353,7 @@ func TestNewBlobStorage(t *testing.T) {
 			name: "valid blob storage",
 			args: args{
 				blobConfig: &config.BlobConfig{
-					Bucket: "fakeBlobStorage",
+					BucketName: "fakeBlobStorage",
 				},
 				cfg: &config.KubehoundConfig{
 					Ingestor: config.IngestorConfig{

pkg/kubehound/storage/retrier.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/DataDog/KubeHound/pkg/config"
+	"github.com/DataDog/KubeHound/pkg/telemetry/log"
 )
 
 type Connector[T any] func(ctx context.Context, cfg *config.KubehoundConfig) (T, error)
@@ -13,6 +14,7 @@ func Retrier[T any](connector Connector[T], retries int, delay time.Duration) Co
 	return func(ctx context.Context, cfg *config.KubehoundConfig) (T, error) {
 		for r := 0; ; r++ {
 			var empty T
+			log.I.Warnf("Trying to connect [%d/%d]", r, retries)
 			provider, err := connector(ctx, cfg)
 			if err == nil || r >= retries {
 				return provider, err

test/system/kubehound_dump.yaml

@@ -29,11 +29,11 @@ builder:
 # Ingestor configuration (for KHaaS)
 ingestor:
   blob:
-    bucket: "" # (i.e.: s3://<your_bucket>)
+    bucket_name: "" # (i.e.: s3://<your_bucket>)
     region: "" # (i.e.: us-west-2)
   temp_dir: "/tmp/kubehound"
   archive_name: "archive.tar.gz"
   max_archive_size: 2147483648 # 2GB
   api: # GRPC endpoint for the ingestor
     endpoint: "127.0.0.1:9000"
-    insecure: true
+    insecure: true

test/system/setup_test.go

@@ -193,7 +193,7 @@ func RunGRPC(ctx context.Context, runArgs *runArgs, p *providers.ProvidersFactor
 		log.I.Fatal(err.Error())
 	}
 
-	khCfg.Ingestor.Blob.Bucket = fmt.Sprintf("file://%s", fileFolder)
+	khCfg.Ingestor.Blob.BucketName = fmt.Sprintf("file://%s", fileFolder)
 	log.I.Info("Creating Blob Storage provider")
 	puller, err := blob.NewBlobStorage(khCfg, khCfg.Ingestor.Blob)
 	if err != nil {