Merge pull request taskcluster#297 from taskcluster/url-hash-fetcher

Url hash fetcher

engines/qemu/fetcher.go

@@ -15,6 +15,8 @@ var imageFetcher = fetcher.Combine(
 	fetcher.Artifact,
 	// Allow fetching images from queue referenced by index namespace
 	fetcher.Index,
+	// Allow fetching images from URL + hash
+	fetcher.URLHash,
 )
 
 type fetchImageContext struct {

runtime/fetcher/artifactfetcher.go

@@ -91,7 +91,7 @@ func (r *artifactReference) Scopes() [][]string {
 	return [][]string{{"queue:get-artifact:" + r.Artifact}}
 }
 
-func (r *artifactReference) Fetch(ctx Context, target WriteSeekReseter) error {
+func (r *artifactReference) Fetch(ctx Context, target WriteReseter) error {
 	// Construct URL
 	var u string
 	if r.isPublic() {

runtime/fetcher/fetcher.go

@@ -25,13 +25,12 @@ type Context interface {
 	Progress(description string, percent float64)
 }
 
-// WriteSeekReseter is a io.Writer + io.Seeker + io.Closer with Reset()
+// WriteReseter is a io.Writer with Reset()
 // method that discards everything written and starts over from scratch.
 //
 // This is easily implemented by wrapping os.File with FileReseter.
-type WriteSeekReseter interface {
+type WriteReseter interface {
 	io.Writer
-	io.Seeker
 	Reset() error
 }
 
@@ -52,7 +51,7 @@ type Reference interface {
 	// Fetch a reference to a target, sending progress to Context as well
 	// as returning a human readable error message, if fetching fails.
 	// If the referenced resource doesn't exist it returns a BrokenReferenceError.
-	Fetch(context Context, target WriteSeekReseter) error
+	Fetch(context Context, target WriteReseter) error
 }
 
 // A Fetcher specifies a schema for references that it knows how to fetch.

runtime/fetcher/filereseter.go

@@ -2,7 +2,7 @@ package fetcher
 
 import "os"
 
-// FileReseter implements WriteSeekReseter for an *os.File instance
+// FileReseter implements WriteReseter for an *os.File instance
 type FileReseter struct {
 	*os.File
 }

runtime/fetcher/mocks_test.go

@@ -3,7 +3,6 @@ package fetcher
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"io"
 	"sync"
 	"testing"
@@ -36,12 +35,12 @@ func (c *mockContext) ProgressReports() []float64 {
 	return c.progressReports
 }
 
-type mockWriteSeekReseter struct {
+type mockWriteReseter struct {
 	offset int64
 	buffer []byte
 }
 
-func (w *mockWriteSeekReseter) Write(p []byte) (int, error) {
+func (w *mockWriteReseter) Write(p []byte) (int, error) {
 	offset := w.offset + int64(len(p))
 	if int64(len(w.buffer)) < offset {
 		w.buffer = append(w.buffer, make([]byte, offset-int64(len(w.buffer)))...)
@@ -51,41 +50,18 @@ func (w *mockWriteSeekReseter) Write(p []byte) (int, error) {
 	return len(p), nil
 }
 
-func (w *mockWriteSeekReseter) Seek(offset int64, whence int) (int64, error) {
-	switch whence {
-	case io.SeekStart:
-	case io.SeekCurrent:
-		offset += w.offset
-	case io.SeekEnd:
-		offset += int64(len(w.buffer))
-	default:
-		panic("whence value not supported")
-	}
-
-	// Check boundary
-	if offset < 0 {
-		return w.offset, fmt.Errorf("Can't seek to negative offset: %d", offset)
-	}
-	if offset > int64(len(w.buffer)) {
-		panic("Seeking past end of file is implementation defined behavior, don't!")
-	}
-	w.offset = offset
-
-	return w.offset, nil
-}
-
-func (w *mockWriteSeekReseter) Reset() error {
+func (w *mockWriteReseter) Reset() error {
 	w.offset = 0
 	w.buffer = nil
 	return nil
 }
 
-func (w *mockWriteSeekReseter) String() string {
+func (w *mockWriteReseter) String() string {
 	return string(w.buffer)
 }
 
 func TestMockWriteSeekReseter(t *testing.T) {
-	w := &mockWriteSeekReseter{}
+	w := &mockWriteReseter{}
 	_, err := io.Copy(w, bytes.NewBufferString("test"))
 	require.NoError(t, err)
 	require.Equal(t, w.String(), "test")
@@ -99,34 +75,4 @@ func TestMockWriteSeekReseter(t *testing.T) {
 	_, err = io.Copy(w, bytes.NewBufferString(" test again"))
 	require.NoError(t, err)
 	require.Equal(t, w.String(), "test again test again")
-
-	// Seek start
-	_, err = w.Seek(0, io.SeekStart)
-	require.NoError(t, err)
-	_, err = io.Copy(w, bytes.NewBufferString("TEST again"))
-	require.NoError(t, err)
-	require.Equal(t, w.String(), "TEST again test again")
-
-	// Seek end with offset
-	_, err = w.Seek(-5, io.SeekEnd)
-	require.NoError(t, err)
-	_, err = io.Copy(w, bytes.NewBufferString("AGAIN"))
-	require.NoError(t, err)
-	require.Equal(t, w.String(), "TEST again test AGAIN")
-
-	// Seek end
-	_, err = w.Seek(0, io.SeekEnd)
-	require.NoError(t, err)
-	_, err = io.Copy(w, bytes.NewBufferString("!"))
-	require.NoError(t, err)
-	require.Equal(t, w.String(), "TEST again test AGAIN!")
-
-	// Seek start with offset + seek current
-	_, err = w.Seek(0, io.SeekStart)
-	require.NoError(t, err)
-	_, err = w.Seek(5, io.SeekCurrent)
-	require.NoError(t, err)
-	_, err = io.Copy(w, bytes.NewBufferString("-----"))
-	require.NoError(t, err)
-	require.Equal(t, w.String(), "TEST ----- test AGAIN!")
 }

runtime/fetcher/urlfetcher.go

@@ -53,13 +53,13 @@ func (u *urlReference) Scopes() [][]string {
 	return [][]string{{}} // Set containing the empty-scope-set
 }
 
-func (u *urlReference) Fetch(ctx Context, target WriteSeekReseter) error {
+func (u *urlReference) Fetch(ctx Context, target WriteReseter) error {
 	return fetchURLWithRetries(ctx, u.url, u.url, target)
 }
 
 // fetchURLWithRetries will download URL u to target with retries, using subject
 // in error messages and progress updates
-func fetchURLWithRetries(ctx Context, subject, u string, target WriteSeekReseter) error {
+func fetchURLWithRetries(ctx Context, subject, u string, target WriteReseter) error {
 	retry := 0
 	for {
 		// Fetch URL, if no error then we're done

runtime/fetcher/urlfetcher_test.go

@@ -77,7 +77,7 @@ func TestUrlFetcher(t *testing.T) {
 
 	t.Run("status-ok", func(t *testing.T) {
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx, s.URL+"/ok")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx, w)
@@ -88,7 +88,7 @@ func TestUrlFetcher(t *testing.T) {
 
 	t.Run("streaming-ok", func(t *testing.T) {
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx, s.URL+"/streaming")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx, w)
@@ -107,7 +107,7 @@ func TestUrlFetcher(t *testing.T) {
 
 		ctx2 := &mockContext{Context: context.Background()}
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx2, s.URL+"/slow")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx2, w)
@@ -123,7 +123,7 @@ func TestUrlFetcher(t *testing.T) {
 
 	t.Run("client-error", func(t *testing.T) {
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx, s.URL+"/client-error")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx, w)
@@ -135,7 +135,7 @@ func TestUrlFetcher(t *testing.T) {
 
 	t.Run("unauthorized", func(t *testing.T) {
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx, s.URL+"/unauthorized")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx, w)
@@ -147,7 +147,7 @@ func TestUrlFetcher(t *testing.T) {
 
 	t.Run("forbidden", func(t *testing.T) {
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx, s.URL+"/forbidden")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx, w)
@@ -159,7 +159,7 @@ func TestUrlFetcher(t *testing.T) {
 
 	t.Run("not-found", func(t *testing.T) {
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx, s.URL+"/not-found")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx, w)
@@ -171,7 +171,7 @@ func TestUrlFetcher(t *testing.T) {
 
 	t.Run("server-error", func(t *testing.T) {
 		count = 0
-		w := &mockWriteSeekReseter{}
+		w := &mockWriteReseter{}
 		ref, err := URL.NewReference(ctx, s.URL+"/server-error")
 		require.NoError(t, err)
 		err = ref.Fetch(ctx, w)