Merge pull request #1168 from evan2645/prepare-for-0.8.2

Prepare for 0.8.2

.travis.yml

@@ -10,7 +10,7 @@ cache:
     - .cache
     - $HOME/.build
     # cache the Go module cache
-    - $HOME/gopath/pkg/mod
+    - $HOME/go/pkg/mod
     # cache `go install`-ed binaries across stages. build.sh unsets GOPATH
     # so this is hitting the default go env of $HOME/go
     - $HOME/go/bin

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [0.8.2] - 2019-10-10
+- Connection pool details in SQL DataStore plugin are now configurable (#1028)
+- SQL DataStore plugin now emits telemetry (#998)
+- The SPIFFE bundle endpoint now supports serving Web PKI via ACME (#1029)
+- Fix Workload API socket permissions when enclosing directory is automatically created (#1048)
+- The Kubernetes PSAT node attestor now emits node and pod label selectors (#1042)
+- SVIDs can now be created directly against SPIRE server using the new `mint` feature (#1036)
+- SPIRE agent behavior improved to more efficiently balance load across SPIRE servers (#1061)
+- Significant SQL DataStore performance improvements (#1069, #1079)
+- Kubernetes workload registrar now supports assigning SPIFFE IDs based on an annotation (#1047)
+- Registration entries with an expiry set are now automatically pruned from the datastore (#1056)
+- Fix bug that resulted in authorized workloads being denied SVIDs (#1103)
+
 ## [0.8.1] - 2019-07-19
 - Failure to obtain peer information from a Workload API connection no longer brings down the agent (#946)
 - Agent now detects expired cached SVID when it starts and will attempt to re-attest instead of failing (#1000)

CONTRIBUTING.md

@@ -114,6 +114,28 @@ interfaces
 Interfaces should be defined in their own file, named (in lowercase) after the name of the
 interface. eg. `foodata.go` implements `type FooData interface{}`
 
+## Metrics
+
+As much as possible, label names should be constants defined in the `telemetry` package.
+
+Labels added to metrics must be singular; that is, the value of a metrics label must not be an
+array or slice, and a label of some name must only be added once. Failure to follow this will
+make metrics less usable for non-tagging metrics libraries such as `statsd`.
+As counter examples, DO NOT do the following:
+```
+[]telemetry.Label{
+  {Name: "someName", "val1"},
+  {Name: "someName", "val2"},
+}
+```
+```
+var callCounter telemetry.CallCounter
+...
+callCounter.AddLabel("someName", "val1")
+...
+callCounter.AddLabel("someName", "val2")
+```
+
 ## Mocks
 
 Unit tests should avoid mock tests as much as possible. When necessary we should inject mocked

README.md

@@ -4,7 +4,7 @@
 
 ![SPIRE Logo](/doc/images/spire_logo.png)
 
-SPIRE (the [SPIFFE](https://github.com/spiffe/spiffe) Runtime Environment) is a tool-chain for establishing trust between software systems across a wide variety of hosting platforms. Concretely, SPIRE exposes the [SPIFFE Workload API](https://github.com/spiffe/spire/blob/master/proto/spire/api/workload/workload.proto), which can attest running software systems and issue [SPIFFE IDs](https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md) and [SVID](https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md)s to them. This in turn allows two workloads to establish trust between each other, for example by establishing an mTLS connection or by signing and verifying a JWT token.
+SPIRE (the [SPIFFE](https://github.com/spiffe/spiffe) Runtime Environment) is a tool-chain for establishing trust between software systems across a wide variety of hosting platforms. Concretely, SPIRE exposes the [SPIFFE Workload API](https://github.com/spiffe/go-spiffe/blob/master/proto/spiffe/workload/workload.proto), which can attest running software systems and issue [SPIFFE IDs](https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md) and [SVID](https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md)s to them. This in turn allows two workloads to establish trust between each other, for example by establishing an mTLS connection or by signing and verifying a JWT token.
 
 - [Get SPIRE](#get-spire)
 - [Getting started](#getting-started)

api/workload/v2/proto.go

@@ -6,7 +6,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/spiffe/spire/proto/spire/api/workload"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 )
 
 func protoToX509SVIDs(protoSVIDs *workload.X509SVIDResponse) (*X509SVIDs, error) {

api/workload/v2/proto_test.go

@@ -4,7 +4,7 @@ import (
 	"crypto/x509"
 	"testing"
 
-	"github.com/spiffe/spire/proto/spire/api/workload"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	"github.com/spiffe/spire/test/util"
 	"github.com/stretchr/testify/require"
 )

api/workload/v2/workload.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/spiffe/spire/proto/spire/api/workload"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"

api/workload/v2/workload_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/spiffe/spire/proto/spire/api/workload"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	"github.com/spiffe/spire/test/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -124,6 +124,17 @@ func TestStartStop(t *testing.T) {
 	})
 }
 
+func TestGetAgentAddress(t *testing.T) {
+	t.Run("default", func(t *testing.T) {
+		require.Equal(t, "unix:///tmp/agent.sock", GetAgentAddress())
+	})
+	t.Run("env", func(t *testing.T) {
+		os.Setenv(EnvVarAgentAddress, "/foo")
+		defer os.Unsetenv(EnvVarAgentAddress)
+		require.Equal(t, "/foo", GetAgentAddress())
+	})
+}
+
 type testWatcher struct {
 	t            *testing.T
 	X509SVIDs    []*X509SVIDs
@@ -234,13 +245,9 @@ func (m *mockHandler) WaitForCall() {
 }
 
 func (m *mockHandler) FetchX509SVID(_ *workload.X509SVIDRequest, stream workload.SpiffeWorkloadAPI_FetchX509SVIDServer) error {
-	m.t.Run("check security header", func(t *testing.T) {
-		md, ok := metadata.FromIncomingContext(stream.Context())
-		require.True(t, ok, "Request doesn't contain grpc metadata.")
-		require.Len(t, md.Get("workload.spiffe.io"), 1)
-		require.Equal(t, "true", md.Get("workload.spiffe.io")[0])
-	})
-
+	if err := checkHeader(stream.Context()); err != nil {
+		return err
+	}
 	for {
 		select {
 		case name := <-m.sendX509Response:
@@ -285,13 +292,18 @@ func (m *mockHandler) ValidateJWTSVID(context.Context, *workload.ValidateJWTSVID
 	return nil, errors.New("unimplemented")
 }
 
-func TestGetAgentAddress(t *testing.T) {
-	t.Run("default", func(t *testing.T) {
-		require.Equal(t, "unix:///tmp/agent.sock", GetAgentAddress())
-	})
-	t.Run("env", func(t *testing.T) {
-		os.Setenv(EnvVarAgentAddress, "/foo")
-		defer os.Unsetenv(EnvVarAgentAddress)
-		require.Equal(t, "/foo", GetAgentAddress())
-	})
+func checkHeader(ctx context.Context) error {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return errors.New("request context does not contain metadata")
+	}
+	value := md.Get("workload.spiffe.io")
+	if len(value) == 0 {
+		return errors.New("request does not have workload.spiffe.io metadata")
+	}
+	if value[0] != "true" {
+		return errors.New("request workload.spiffe.io metadata is not \"true\"")
+	}
+
+	return nil
 }

api/workload/x509_client.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/andres-erbsen/clock"
 	"github.com/sirupsen/logrus"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	"github.com/spiffe/spire/api/workload/dial"
-	"github.com/spiffe/spire/proto/spire/api/workload"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"

api/workload/x509_client_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/andres-erbsen/clock"
 	"github.com/gogo/protobuf/proto"
-	"github.com/spiffe/spire/proto/spire/api/workload"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	"github.com/spiffe/spire/test/fakes/fakeworkloadapi"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -23,7 +23,7 @@ const (
 	// something is broken is through failure to receive on channels. Using the
 	// following timeout to prevent having to wait for the default go test
 	// timeout (10 minutes) if this happens. The timeout should be large enough
-	// by a comfortable margin to accomodate for slower running platforms, like
+	// by a comfortable margin to accommodate for slower running platforms, like
 	// Travis CI.
 	testTimeout = time.Minute
 )

cmd/spire-agent/cli/api/common.go

@@ -6,10 +6,10 @@ import (
 	"net"
 	"time"
 
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	workload_dial "github.com/spiffe/spire/api/workload/dial"
 	"github.com/spiffe/spire/cmd/spire-agent/cli/common"
 	"github.com/spiffe/spire/pkg/common/cli"
-	"github.com/spiffe/spire/proto/spire/api/workload"
 	"google.golang.org/grpc/metadata"
 )
 

cmd/spire-agent/cli/api/fetch_jwt.go

@@ -7,8 +7,8 @@ import (
 	"fmt"
 
 	"github.com/mitchellh/cli"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	common_cli "github.com/spiffe/spire/pkg/common/cli"
-	"github.com/spiffe/spire/proto/spire/api/workload"
 )
 
 func NewFetchJWTCommand() cli.Command {

cmd/spire-agent/cli/api/fetch_x509.go

@@ -11,8 +11,8 @@ import (
 	"time"
 
 	"github.com/mitchellh/cli"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	common_cli "github.com/spiffe/spire/pkg/common/cli"
-	"github.com/spiffe/spire/proto/spire/api/workload"
 )
 
 func NewFetchX509Command() cli.Command {

cmd/spire-agent/cli/api/printer.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/spiffe/spire/proto/spire/api/workload"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 )
 
 func printX509SVIDResponse(resp *workload.X509SVIDResponse, respTime time.Duration) {

cmd/spire-agent/cli/api/validate_jwt.go

@@ -8,8 +8,8 @@ import (
 
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/mitchellh/cli"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	common_cli "github.com/spiffe/spire/pkg/common/cli"
-	"github.com/spiffe/spire/proto/spire/api/workload"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )

cmd/spire-agent/cli/healthcheck/healthcheck_test.go

@@ -5,8 +5,8 @@ import (
 	"testing"
 
 	"github.com/mitchellh/cli"
+	"github.com/spiffe/go-spiffe/proto/spiffe/workload"
 	common_cli "github.com/spiffe/spire/pkg/common/cli"
-	"github.com/spiffe/spire/proto/spire/api/workload"
 	"github.com/spiffe/spire/test/fakes/fakeworkloadapi"
 	"github.com/stretchr/testify/suite"
 	"google.golang.org/grpc/codes"

cmd/spire-agent/cli/run/run.go

@@ -19,6 +19,7 @@ import (
 	"github.com/spiffe/spire/pkg/agent"
 	"github.com/spiffe/spire/pkg/common/catalog"
 	"github.com/spiffe/spire/pkg/common/cli"
+	"github.com/spiffe/spire/pkg/common/health"
 	"github.com/spiffe/spire/pkg/common/idutil"
 	"github.com/spiffe/spire/pkg/common/log"
 	"github.com/spiffe/spire/pkg/common/telemetry"
@@ -36,9 +37,10 @@ const (
 
 // config contains all available configurables, arranged by section
 type config struct {
-	Agent     *agentConfig                `hcl:"agent"`
-	Plugins   *catalog.HCLPluginConfigMap `hcl:"plugins"`
-	Telemetry telemetry.FileConfig        `hcl:"telemetry"`
+	Agent        *agentConfig                `hcl:"agent"`
+	Plugins      *catalog.HCLPluginConfigMap `hcl:"plugins"`
+	Telemetry    telemetry.FileConfig        `hcl:"telemetry"`
+	HealthChecks health.Config               `hcl:"health_checks"`
 }
 
 type agentConfig struct {
@@ -96,6 +98,16 @@ func (*RunCLI) Run(args []string) int {
 		return 1
 	}
 
+	// Create uds dir and parents if not exists
+	dir := filepath.Dir(c.BindAddress.String())
+	if _, statErr := os.Stat(dir); os.IsNotExist(statErr) {
+		c.Log.WithField("dir", dir).Infof("Creating spire agent UDS directory")
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			return 1
+		}
+	}
+
 	// Set umask before starting up the agent
 	cli.SetUmask(c.Log)
 
@@ -202,7 +214,8 @@ func newAgentConfig(c *config) (*agent.Config, error) {
 		return nil, err
 	}
 
-	ac.ServerAddress = net.JoinHostPort(c.Agent.ServerAddress, strconv.Itoa(c.Agent.ServerPort))
+	serverHostPort := net.JoinHostPort(c.Agent.ServerAddress, strconv.Itoa(c.Agent.ServerPort))
+	ac.ServerAddress = fmt.Sprintf("dns:///%s", serverHostPort)
 
 	td, err := idutil.ParseSpiffeID("spiffe://"+c.Agent.TrustDomain, idutil.AllowAnyTrustDomain())
 	if err != nil {
@@ -241,6 +254,7 @@ func newAgentConfig(c *config) (*agent.Config, error) {
 
 	ac.PluginConfigs = *c.Plugins
 	ac.Telemetry = c.Telemetry
+	ac.HealthChecks = c.HealthChecks
 
 	return ac, nil
 }

cmd/spire-agent/cli/run/run_test.go

@@ -513,7 +513,7 @@ func TestNewAgentConfig(t *testing.T) {
 				c.Agent.ServerPort = 1337
 			},
 			test: func(t *testing.T, c *agent.Config) {
-				require.Equal(t, "192.168.1.1:1337", c.ServerAddress)
+				require.Equal(t, "dns:///192.168.1.1:1337", c.ServerAddress)
 			},
 		},
 		{

cmd/spire-server/cli/agent/evict_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/spiffe/spire/proto/spire/api/registration"
 
 	"github.com/golang/mock/gomock"
-	"github.com/spiffe/spire/test/mock/proto/api/registration"
+	mock_registration "github.com/spiffe/spire/test/mock/proto/api/registration"
 	"github.com/stretchr/testify/suite"
 )
 

cmd/spire-server/cli/agent/list_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/spiffe/spire/proto/spire/api/registration"
 	"github.com/spiffe/spire/proto/spire/common"
-	"github.com/spiffe/spire/test/mock/proto/api/registration"
+	mock_registration "github.com/spiffe/spire/test/mock/proto/api/registration"
 	"github.com/stretchr/testify/suite"
 )
 
@@ -38,7 +38,7 @@ func (s *ListTestSuite) TestRun() {
 	req := &registration.ListAgentsRequest{}
 	resp := &registration.ListAgentsResponse{
 		Nodes: []*common.AttestedNode{
-			&common.AttestedNode{SpiffeId: "spiffe://example.org/spire/agent/join_token/token_a"},
+			{SpiffeId: "spiffe://example.org/spire/agent/join_token/token_a"},
 		},
 	}
 	s.mockClient.EXPECT().ListAgents(gomock.Any(), req).Return(resp, nil)

cmd/spire-server/cli/bundle/bundle_test.go

@@ -340,7 +340,7 @@ func (s *BundleSuite) TestDeleteWithRestrictMode() {
 	})
 
 	s.Require().Equal(1, s.deleteCmd.Run([]string{"-id", "spiffe://domain1.test"}))
-	s.Require().Equal("rpc error: code = Unknown desc = cannot delete bundle; federated with 1 registration entries\n", s.stderr.String())
+	s.Require().Equal("rpc error: code = Internal desc = cannot delete bundle; federated with 1 registration entries\n", s.stderr.String())
 
 	_, err := s.ds.FetchBundle(context.Background(), &datastore.FetchBundleRequest{
 		TrustDomainId: "spiffe://domain1.test",

cmd/spire-server/cli/cli.go

@@ -8,8 +8,10 @@ import (
 	"github.com/spiffe/spire/cmd/spire-server/cli/bundle"
 	"github.com/spiffe/spire/cmd/spire-server/cli/entry"
 	"github.com/spiffe/spire/cmd/spire-server/cli/healthcheck"
+	"github.com/spiffe/spire/cmd/spire-server/cli/jwt"
 	"github.com/spiffe/spire/cmd/spire-server/cli/run"
 	"github.com/spiffe/spire/cmd/spire-server/cli/token"
+	"github.com/spiffe/spire/cmd/spire-server/cli/x509"
 	"github.com/spiffe/spire/pkg/common/version"
 )
 
@@ -65,6 +67,12 @@ func Run(args []string) int {
 		"healthcheck": func() (cli.Command, error) {
 			return healthcheck.NewHealthCheckCommand(), nil
 		},
+		"x509 mint": func() (cli.Command, error) {
+			return x509.NewMintCommand(), nil
+		},
+		"jwt mint": func() (cli.Command, error) {
+			return jwt.NewMintCommand(), nil
+		},
 	}
 
 	exitStatus, err := c.Run()