diff --git a/core/digest.go b/core/digest.go index 18ef9d6be..92851207f 100644 --- a/core/digest.go +++ b/core/digest.go @@ -23,11 +23,6 @@ import ( "strings" ) -const ( - // DigestEmptyTar is the sha256 digest of an empty tar file. - DigestEmptyTar = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" -) - // DigestList is a list of digests. type DigestList []Digest diff --git a/lib/backend/manager.go b/lib/backend/manager.go index 2606024d8..35069ab65 100644 --- a/lib/backend/manager.go +++ b/lib/backend/manager.go @@ -143,7 +143,7 @@ func (m *Manager) GetClient(namespace string) (Client, error) { return nil, ErrNamespaceNotFound } -// IsReady returns whether the backends are ready (reachable). +// CheckReadiness returns whether the backends are ready (available). // A backend must be explicitly configured as required for readiness to be checked. func (m *Manager) CheckReadiness() error { for _, b := range m.backends { diff --git a/origin/blobserver/server.go b/origin/blobserver/server.go index 03c22aa53..c905a3e55 100644 --- a/origin/blobserver/server.go +++ b/origin/blobserver/server.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -130,6 +130,7 @@ func (s *Server) Handler() http.Handler { // Public endpoints: r.Get("/health", handler.Wrap(s.healthCheckHandler)) + r.Get("/readiness", handler.Wrap(s.readinessCheckHandler)) r.Get("/blobs/{digest}/locations", handler.Wrap(s.getLocationsHandler)) @@ -179,6 +180,15 @@ func (s *Server) healthCheckHandler(w http.ResponseWriter, r *http.Request) erro return nil } +func (s *Server) readinessCheckHandler(w http.ResponseWriter, r *http.Request) error { + err := s.backends.CheckReadiness() + if err != nil { + return handler.Errorf("not ready to serve traffic: %s", err).Status(http.StatusServiceUnavailable) + } + fmt.Fprintln(w, "OK") + return nil +} + // statHandler returns blob info if it exists. func (s *Server) statHandler(w http.ResponseWriter, r *http.Request) error { checkLocal, err := strconv.ParseBool(httputil.GetQueryArg(r, "local", "false")) diff --git a/origin/blobserver/server_test.go b/origin/blobserver/server_test.go index 08ee08cb0..6711905f7 100644 --- a/origin/blobserver/server_test.go +++ b/origin/blobserver/server_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "testing" "time" @@ -26,6 +27,7 @@ import ( "github.com/stretchr/testify/require" "github.com/uber/kraken/core" + "github.com/uber/kraken/lib/backend" "github.com/uber/kraken/lib/backend/backenderrors" "github.com/uber/kraken/lib/persistedretry" "github.com/uber/kraken/lib/persistedretry/writeback" @@ -53,6 +55,60 @@ func TestHealth(t *testing.T) { require.Equal("OK\n", string(b)) } +func TestReadiness(t *testing.T) { + for _, tc := range []struct { + mockStatErr error + expectedErrMsg string + status int + }{ + { + mockStatErr: nil, + expectedErrMsg: "", + status: http.StatusOK, + }, + { + mockStatErr: backenderrors.ErrBlobNotFound, + expectedErrMsg: "", + status: http.StatusOK, + }, + { + mockStatErr: errors.New("failed due to backend error"), + expectedErrMsg: fmt.Sprintf("not ready to serve traffic: backend for namespace '%s' not ready: failed due to backend error", backend.ReadinessCheckNamespace), + status: http.StatusServiceUnavailable, + }, + } { + require := require.New(t) + + cp := newTestClientProvider() + + s := newTestServer(t, master1, hashRingMaxReplica(), cp) + defer s.cleanup() + + backendClient := s.backendClient(backend.ReadinessCheckNamespace, true) + mockStat := &core.BlobInfo{} + if tc.mockStatErr != nil { + mockStat = nil + } + + backendClient.EXPECT().Stat(backend.ReadinessCheckNamespace, backend.ReadinessCheckName).Return(mockStat, tc.mockStatErr) + + resp, err := httputil.Get( + fmt.Sprintf("http://%s/readiness", s.addr)) + + if tc.status == http.StatusOK { + defer resp.Body.Close() + require.Equal(tc.status, resp.StatusCode) + require.NoError(err) + b, _ := ioutil.ReadAll(resp.Body) + require.Equal("OK\n", string(b)) + } else { + require.True(httputil.IsStatus(err, tc.status)) + require.True(strings.Contains(err.Error(), tc.expectedErrMsg)) + require.Nil(resp) + } + } +} + func TestStatHandlerLocalNotFound(t *testing.T) { require := require.New(t) @@ -117,7 +173,7 @@ func TestStatHandlerNotFound(t *testing.T) { d := core.DigestFixture() namespace := core.TagFixture() - backendClient := s.backendClient(namespace) + backendClient := s.backendClient(namespace, false) backendClient.EXPECT().Stat(namespace, d.Hex()).Return(nil, backenderrors.ErrBlobNotFound) @@ -192,7 +248,7 @@ func TestDownloadBlobNotFound(t *testing.T) { d := core.DigestFixture() namespace := core.TagFixture() - backendClient := s.backendClient(namespace) + backendClient := s.backendClient(namespace, false) backendClient.EXPECT().Stat(namespace, d.Hex()).Return(nil, backenderrors.ErrBlobNotFound) err := cp.Provide(master1).DownloadBlob(namespace, d, ioutil.Discard) @@ -280,7 +336,7 @@ func TestGetMetaInfoDownloadsBlobAndReplicates(t *testing.T) { blob := computeBlobForHosts(ring, s1.host, s2.host) - backendClient := s1.backendClient(namespace) + backendClient := s1.backendClient(namespace, false) backendClient.EXPECT().Stat(namespace, blob.Digest.Hex()).Return(core.NewBlobInfo(int64(len(blob.Content))), nil).AnyTimes() backendClient.EXPECT().Download(namespace, blob.Digest.Hex(), mockutil.MatchWriter(blob.Content)).Return(nil) @@ -317,7 +373,7 @@ func TestGetMetaInfoBlobNotFound(t *testing.T) { d := core.DigestFixture() namespace := core.TagFixture() - backendClient := s.backendClient(namespace) + backendClient := s.backendClient(namespace, false) backendClient.EXPECT().Stat(namespace, d.Hex()).Return(nil, backenderrors.ErrBlobNotFound) mi, err := cp.Provide(master1).GetMetaInfo(namespace, d) @@ -582,7 +638,7 @@ func TestReplicateToRemoteWhenBlobInStorageBackend(t *testing.T) { blob := core.NewBlobFixture() namespace := core.TagFixture() - backendClient := s.backendClient(namespace) + backendClient := s.backendClient(namespace, false) backendClient.EXPECT().Stat(namespace, blob.Digest.Hex()).Return(core.NewBlobInfo(int64(len(blob.Content))), nil).AnyTimes() backendClient.EXPECT().Download(namespace, blob.Digest.Hex(), mockutil.MatchWriter(blob.Content)).Return(nil) diff --git a/origin/blobserver/testutils_test.go b/origin/blobserver/testutils_test.go index 9bed11b32..593328e71 100644 --- a/origin/blobserver/testutils_test.go +++ b/origin/blobserver/testutils_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -158,9 +158,9 @@ func newTestServer( } } -func (s *testServer) backendClient(namespace string) *mockbackend.MockClient { +func (s *testServer) backendClient(namespace string, mustReady bool) *mockbackend.MockClient { client := mockbackend.NewMockClient(s.ctrl) - if err := s.backendManager.Register(namespace, client, false); err != nil { + if err := s.backendManager.Register(namespace, client, mustReady); err != nil { panic(err) } return client