adding support for publishing bundle to Minio Object Storage (#5395)

go.mod

@@ -63,6 +63,7 @@ require (
 	github.com/jinzhu/gorm v1.9.16
 	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.24
+	github.com/minio/minio-go/v7 v7.0.82
 	github.com/mitchellh/cli v1.1.5
 	github.com/open-policy-agent/opa v0.70.0
 	github.com/prometheus/client_golang v1.20.5
@@ -175,6 +176,7 @@ require (
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-openapi/validate v0.24.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
@@ -215,14 +217,16 @@ require (
 	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -251,6 +255,7 @@ require (
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rs/xid v1.6.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect

go.sum

@@ -847,6 +847,8 @@ github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
 github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk=
@@ -1143,8 +1145,11 @@ github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKu
 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
@@ -1193,6 +1198,10 @@ github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
 github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
 github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
+github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
 github.com/mitchellh/cli v1.1.5 h1:OxRIeJXpAMztws/XHlN2vu6imG5Dpq+j61AzAX5fLng=
 github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -1326,6 +1335,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=

pkg/server/catalog/bundlepublisher.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spiffe/spire/pkg/server/plugin/bundlepublisher/awsrolesanywhere"
 	"github.com/spiffe/spire/pkg/server/plugin/bundlepublisher/awss3"
 	"github.com/spiffe/spire/pkg/server/plugin/bundlepublisher/gcpcloudstorage"
+	"github.com/spiffe/spire/pkg/server/plugin/bundlepublisher/minios3"
 )
 
 type bundlePublisherRepository struct {
@@ -29,6 +30,7 @@ func (repo *bundlePublisherRepository) BuiltIns() []catalog.BuiltIn {
 		awss3.BuiltIn(),
 		gcpcloudstorage.BuiltIn(),
 		awsrolesanywhere.BuiltIn(),
+		minios3.BuiltIn(),
 	}
 }
 

pkg/server/plugin/bundlepublisher/minios3/client.go

@@ -0,0 +1,26 @@
+package minios3
+
+import (
+	"context"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"io"
+)
+
+type minioService interface {
+	PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,
+		opts minio.PutObjectOptions,
+	) (info minio.UploadInfo, err error)
+}
+
+func newMinioClient(c Config) (minioService, error) {
+	// Initialize minio client object.
+	minioClient, err := minio.New(c.Endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(c.AccessKeyID, c.SecretAccessKey, ""),
+		Secure: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return minioClient, nil
+}

pkg/server/plugin/bundlepublisher/minios3/minios3.go

@@ -0,0 +1,228 @@
+package minios3
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/hcl"
+	"github.com/minio/minio-go/v7"
+	"github.com/spiffe/spire-plugin-sdk/pluginsdk/support/bundleformat"
+	bundlepublisherv1 "github.com/spiffe/spire-plugin-sdk/proto/spire/plugin/server/bundlepublisher/v1"
+	"github.com/spiffe/spire-plugin-sdk/proto/spire/plugin/types"
+	configv1 "github.com/spiffe/spire-plugin-sdk/proto/spire/service/common/config/v1"
+	"github.com/spiffe/spire/pkg/common/catalog"
+	"github.com/spiffe/spire/pkg/common/pluginconf"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+	"sync"
+)
+
+const (
+	pluginName = "minio_s3"
+)
+
+type pluginHooks struct {
+	newMinioClientFunc func(c Config) (minioService, error)
+}
+
+func BuiltIn() catalog.BuiltIn {
+	return builtin(New())
+}
+
+func New() *Plugin {
+	return newPlugin(newMinioClient)
+}
+
+type Config struct {
+	AccessKeyID     string `hcl:"access_key_id" json:"access_key_id"`
+	SecretAccessKey string `hcl:"secret_access_key" json:"secret_access_key"`
+	Endpoint        string `hcl:"endpoint" json:"endpoint"`
+	Bucket          string `hcl:"bucket" json:"bucket"`
+	ObjectKey       string `hcl:"object_key" json:"object_key"`
+	Format          string `hcl:"format" json:"format"`
+
+	// bundleFormat is used to store the content of Format, parsed
+	// as bundleformat.Format.
+	bundleFormat bundleformat.Format
+}
+
+func buildConfig(coreConfig catalog.CoreConfig, hclText string, status *pluginconf.Status) *Config {
+	newConfig := new(Config)
+
+	if err := hcl.Decode(newConfig, hclText); err != nil {
+		status.ReportErrorf("unable to decode configuration: %v", err)
+		return nil
+	}
+
+	if newConfig.Endpoint == "" {
+		status.ReportError("configuration is missing the endpoint")
+	}
+	if newConfig.Bucket == "" {
+		status.ReportError("configuration is missing the bucket name")
+	}
+	if newConfig.ObjectKey == "" {
+		status.ReportError("configuration is missing the object key")
+	}
+	if newConfig.Format == "" {
+		status.ReportError("configuration is missing the bundle format")
+	}
+
+	bundleFormat, err := bundleformat.FromString(newConfig.Format)
+	if err != nil {
+		status.ReportErrorf("could not parse bundle format from configuration: %v", err)
+	} else {
+		// This plugin only supports some bundleformats.
+		switch bundleFormat {
+		case bundleformat.JWKS:
+		case bundleformat.SPIFFE:
+		case bundleformat.PEM:
+		default:
+			status.ReportErrorf("bundle format %q is not supported", newConfig.Format)
+		}
+		newConfig.bundleFormat = bundleFormat
+	}
+
+	return newConfig
+}
+
+type Plugin struct {
+	bundlepublisherv1.UnsafeBundlePublisherServer
+	configv1.UnsafeConfigServer
+
+	config    *Config
+	configMtx sync.RWMutex
+
+	bundle    *types.Bundle
+	bundleMtx sync.RWMutex
+
+	hooks       pluginHooks
+	minioClient minioService
+	log         hclog.Logger
+}
+
+// SetLogger sets a logger in the plugin.
+func (p *Plugin) SetLogger(log hclog.Logger) {
+	p.log = log
+}
+
+// Configure configures the plugin.
+func (p *Plugin) Configure(ctx context.Context, request *configv1.ConfigureRequest) (*configv1.ConfigureResponse, error) {
+	newConfig, _, err := pluginconf.Build(request, buildConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	minioClient, err := p.hooks.newMinioClientFunc(*newConfig)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to create client: %v", err)
+	}
+	p.minioClient = minioClient
+
+	p.setConfig(newConfig)
+	p.setBundle(nil)
+
+	return &configv1.ConfigureResponse{}, nil
+}
+
+func (p *Plugin) Validate(ctx context.Context, req *configv1.ValidateRequest) (*configv1.ValidateResponse, error) {
+	_, notes, err := pluginconf.Build(req, buildConfig)
+
+	return &configv1.ValidateResponse{
+		Valid: err == nil,
+		Notes: notes,
+	}, err
+}
+
+// PublishBundle puts the bundle in the configured S3 bucket name and
+// object key.
+func (p *Plugin) PublishBundle(ctx context.Context, req *bundlepublisherv1.PublishBundleRequest) (*bundlepublisherv1.PublishBundleResponse, error) {
+	config, err := p.getConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	if req.Bundle == nil {
+		return nil, status.Error(codes.InvalidArgument, "missing bundle in request")
+	}
+	currentBundle := p.getBundle()
+	if proto.Equal(req.Bundle, currentBundle) {
+		// Bundle not changed. No need to publish.
+		return &bundlepublisherv1.PublishBundleResponse{}, nil
+	}
+
+	formatter := bundleformat.NewFormatter(req.Bundle)
+	bundleBytes, err := formatter.Format(config.bundleFormat)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "could not format bundle: %v", err.Error())
+	}
+
+	info, err := p.minioClient.PutObject(
+		ctx,
+		config.Bucket,
+		config.ObjectKey,
+		bytes.NewReader(bundleBytes),
+		int64(len(bundleBytes)),
+		minio.PutObjectOptions{},
+	)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to put object: %v", err)
+	}
+
+	fmt.Println(info.Key)
+	p.setBundle(req.Bundle)
+	p.log.Debug("Bundle published")
+	return &bundlepublisherv1.PublishBundleResponse{}, nil
+
+}
+
+// getBundle gets the latest bundle that the plugin has.
+func (p *Plugin) getBundle() *types.Bundle {
+	p.configMtx.RLock()
+	defer p.configMtx.RUnlock()
+
+	return p.bundle
+}
+
+// getConfig gets the configuration of the plugin.
+func (p *Plugin) getConfig() (*Config, error) {
+	p.configMtx.RLock()
+	defer p.configMtx.RUnlock()
+
+	if p.config == nil {
+		return nil, status.Error(codes.FailedPrecondition, "not configured")
+	}
+	return p.config, nil
+}
+
+// setBundle updates the current bundle in the plugin with the provided bundle.
+func (p *Plugin) setBundle(bundle *types.Bundle) {
+	p.bundleMtx.Lock()
+	defer p.bundleMtx.Unlock()
+
+	p.bundle = bundle
+}
+
+// setConfig sets the configuration for the plugin.
+func (p *Plugin) setConfig(config *Config) {
+	p.configMtx.Lock()
+	defer p.configMtx.Unlock()
+
+	p.config = config
+}
+
+// builtin creates a new BundlePublisher built-in plugin.
+func builtin(p *Plugin) catalog.BuiltIn {
+	return catalog.MakeBuiltIn(pluginName,
+		bundlepublisherv1.BundlePublisherPluginServer(p),
+		configv1.ConfigServiceServer(p),
+	)
+}
+
+// newPlugin returns a new plugin instance.
+func newPlugin(newMinioClientFunc func(c Config) (minioService, error)) *Plugin {
+	return &Plugin{
+		hooks: pluginHooks{newMinioClientFunc: newMinioClientFunc},
+	}
+}

pkg/server/plugin/bundlepublisher/minios3/minios3_test.go

@@ -0,0 +1 @@
+package minios3