Merge branch 'develop' into main

.goreleaser.yml

@@ -19,6 +19,7 @@ archives:
       amd64: x86_64
     files:
       - none*
+    format: binary
 checksum:
   name_template: 'checksums.txt'
 snapshot:

cmd/fanbox-dl/main.go

@@ -67,19 +67,21 @@ func main() {
 			CheckAllPosts:  c.Bool("all"),
 			DryRun:         c.Bool("dry-run"),
 		}
+
+		start := time.Now()
 		err := client.Run(c.Context)
 		if err != nil {
 			return fmt.Errorf("download error: %w", err)
 		}
+
+		log.Printf("Completed (after %v).", time.Now().Sub(start))
 		return nil
 	}
 
-	start := time.Now()
 	ctx := context.Background()
 	err := app.RunContext(ctx, os.Args)
 	if err != nil {
 		log.Fatalf("Pixiv FANBOX Downloader failed: %s", err)
 	}
-	log.Printf("Completed (after %v).", time.Now().Sub(start))
 	os.Exit(0)
 }

go.mod

@@ -3,6 +3,7 @@ module github.com/hareku/fanbox-dl
 go 1.15
 
 require (
+	github.com/cenkalti/backoff/v4 v4.1.0
 	github.com/hareku/filename v0.3.0
 	github.com/urfave/cli/v2 v2.3.0
 )

go.sum

@@ -1,4 +1,8 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/cenkalti/backoff v1.1.0 h1:QnvVp8ikKCDWOsFheytRCoYWYPO/ObCTBGxT19Hc+yE=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc=
+github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/hareku/filename v0.3.0 h1:MLvDN5HMapWr7VDIMaVU/bHR/PyeSRsrK2xV1+e5hOw=

pkg/fanbox/response.go → pkg/fanbox/api.go

@@ -1,6 +1,12 @@
 package fanbox
 
-// ListCreator is the response of https://api.fanbox.cc/post.listCreator.
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+// ListCreator represents the response of https://api.fanbox.cc/post.listCreator.
 type ListCreator struct {
 	Body ListCreatorBody `json:"body"`
 }
@@ -19,7 +25,8 @@ type Post struct {
 	Body              *PostBody `json:"body"`
 }
 
-// PostBody contains the body of a post.
+// PostBody represents a post's body.
+// PostBody has "Images" or "Blocks and ImageMap".
 type PostBody struct {
 	Blocks   *[]Block          `json:"blocks"`
 	Images   *[]Image          `json:"images"`
@@ -39,7 +46,7 @@ type Image struct {
 	OriginalURL string `json:"originalUrl"`
 }
 
-// OrderedImageMap returns ordered image map by PostBody.Blocks.
+// OrderedImageMap returns ordered images in ImageMap by PostBody.Blocks order.
 func (b *PostBody) OrderedImageMap() []Image {
 	if b.ImageMap == nil || b.Blocks == nil {
 		return nil
@@ -56,3 +63,23 @@ func (b *PostBody) OrderedImageMap() []Image {
 
 	return images
 }
+
+// Request sends a request to FANBOX with credentials.
+func Request(ctx context.Context, sessid string, url string) (*http.Response, error) {
+	client := http.Client{}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("http request building error: %w", err)
+	}
+
+	req.Header.Set("Cookie", fmt.Sprintf("FANBOXSESSID=%s", sessid))
+	req.Header.Set("Origin", "https://www.fanbox.cc") // If Origin header is not set, FANBOX returns HTTP 400 error.
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http response error: %w", err)
+	}
+
+	return resp, nil
+}

pkg/fanbox/api_test.go

@@ -0,0 +1,80 @@
+package fanbox_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/hareku/fanbox-dl/pkg/fanbox"
+)
+
+func stringPointer(str string) *string {
+	return &str
+}
+
+func TestPostBody_OrderedImageMap(t *testing.T) {
+	type fields struct {
+		Blocks   *[]fanbox.Block
+		Images   *[]fanbox.Image
+		ImageMap *map[string]fanbox.Image
+	}
+
+	tests := []struct {
+		name   string
+		fields fields
+		want   []fanbox.Image
+	}{
+		{
+			name: "sort images by blocks order",
+			fields: fields{
+				Images: nil,
+				Blocks: &[]fanbox.Block{
+					{
+						Type:    "image",
+						ImageID: stringPointer("first"),
+					},
+					{
+						Type:    "image",
+						ImageID: stringPointer("second"),
+					},
+				},
+				ImageMap: &map[string]fanbox.Image{
+					"second": {
+						ID: "second_image",
+					},
+					"first": {
+						ID: "first_image",
+					},
+				},
+			},
+			want: []fanbox.Image{
+				{
+					ID: "first_image",
+				},
+				{
+					ID: "second_image",
+				},
+			},
+		},
+		{
+			name: "return nil if Blocks and ImageMap are nil",
+			fields: fields{
+				Images:   &[]fanbox.Image{},
+				Blocks:   nil,
+				ImageMap: nil,
+			},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &fanbox.PostBody{
+				Blocks:   tt.fields.Blocks,
+				Images:   tt.fields.Images,
+				ImageMap: tt.fields.ImageMap,
+			}
+			if got := b.OrderedImageMap(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("PostBody.OrderedImageMap() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/fanbox/client.go

@@ -10,7 +10,8 @@ import (
 	"log"
 	"net/http"
 	"net/url"
-	"time"
+
+	backoff "github.com/cenkalti/backoff/v4"
 )
 
 // Client is the client which downloads images from
@@ -99,7 +100,7 @@ func (c *Client) buildFirstURL() string {
 	params.Set("creatorId", c.UserID)
 	params.Set("limit", "50")
 
-	return fmt.Sprintf("https://cc/post.listCreator?%s", params.Encode())
+	return fmt.Sprintf("https://api.fanbox.cc/post.listCreator?%s", params.Encode())
 }
 
 // request sends GET request with credentials.
@@ -113,32 +114,26 @@ func (c *Client) request(ctx context.Context, url string) (*http.Response, error
 }
 
 func (c *Client) downloadWithRetry(ctx context.Context, post Post, order int, img Image) error {
-	const maxRetry = 5
-	retry := 0
-	var err error
-
-	for {
-		if retry >= maxRetry {
-			break
-		}
-
-		err = c.download(ctx, post, order, img)
+	operation := func() error {
+		err := c.download(ctx, post, order, img)
 		if err == nil {
-			break
+			return nil
 		}
 
-		// HTTP body often disconnects and returns error io.ErrUnexpectedEOF.
-		// But if err is not io.ErrUnexpectedEOF, stop the retrying.
-		if !errors.Is(err, io.ErrUnexpectedEOF) {
-			break
+		// HTTP body often disconnects while downloading and returns error io.ErrUnexpectedEOF.
+		// But other errors may be permanent error, so return and wrap it.
+		if errors.Is(err, io.ErrUnexpectedEOF) {
+			return err
 		}
 
-		time.Sleep(time.Second)
-		retry++
+		return backoff.Permanent(err)
 	}
 
+	strategy := backoff.WithContext(backoff.NewExponentialBackOff(), ctx)
+
+	err := backoff.Retry(operation, strategy)
 	if err != nil {
-		return fmt.Errorf("failed to download with retry %d times: %w", retry, err)
+		return fmt.Errorf("failed to download with retry: %w", err)
 	}
 
 	return nil

pkg/fanbox/file.go

@@ -6,14 +6,11 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"regexp"
 	"time"
 
 	"github.com/hareku/filename"
 )
 
-var invalidFileChar = regexp.MustCompile(`[\/:*?"<>|]`)
-
 func (c *Client) makeFileName(post Post, order int, img Image) string {
 	date, err := time.Parse(time.RFC3339, post.PublishedDateTime)
 	if err != nil {