From 24cf933e22465e4eb03288fe1a200386742b8ff2 Mon Sep 17 00:00:00 2001
From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com>
Date: Fri, 29 Dec 2023 18:59:54 +0330
Subject: [PATCH 01/11] add unittest for utils
---
utils_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
create mode 100644 utils_test.go
diff --git a/utils_test.go b/utils_test.go
new file mode 100644
index 0000000..4e565da
--- /dev/null
+++ b/utils_test.go
@@ -0,0 +1,60 @@
+package obelisk
+
+import (
+ "fmt"
+ "net/url"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIsValitdURL(t *testing.T) {
+ dataURL := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAEklEQVQIW2P8z8AARAwMjDAGACwBA/+8RVWvAAAAAElFTkSuQmCC"
+ rawURL := "https://google.com/page#fragment?utm_source=google&utm_medium=cpc&utm_campaign=summer_sale"
+ expected := []byte("TextforTest")
+ // Parse the URL
+ parsedURL, err := url.Parse(rawURL)
+ if err != nil {
+ fmt.Println("Failed to parse URL:", err)
+ return
+ }
+ contentType := "text/plain"
+ expectedResult := "data:text/plain;base64,VGV4dGZvclRlc3Q="
+
+ t.Run("Test isvalidURL", func(t *testing.T) {
+ result := isValidURL("https://github.com/go-shiori/obelisk")
+ result2 := isValidURL("itIsNotAURL")
+ assert.True(t, result)
+ assert.False(t, result2)
+ })
+
+ t.Run("Test Create Absolute URL", func(t *testing.T) {
+
+ resultdataURL := createAbsoluteURL(dataURL, parsedURL)
+ resultRelativePath := createAbsoluteURL("/it/is/relarivepath", parsedURL)
+ resulacualturl := createAbsoluteURL("https://bing.com", parsedURL)
+ resulAcualtURLWithfragment := createAbsoluteURL(rawURL, parsedURL)
+ resulWithoutURL := createAbsoluteURL("", parsedURL)
+ resulWithfragment := createAbsoluteURL("#bar", parsedURL)
+
+ assert.Equal(t, dataURL, resultdataURL)
+ assert.Equal(t, "https://google.com/it/is/relarivepath", resultRelativePath)
+ assert.Equal(t, "https://bing.com", resulacualturl)
+ assert.Equal(t, "https://google.com/page%23fragment", resulAcualtURLWithfragment)
+ assert.Equal(t, "", resulWithoutURL)
+ assert.Equal(t, "#bar", resulWithfragment)
+
+ })
+ t.Run("Test create dataURL", func(t *testing.T) {
+ result := createDataURL(expected, contentType)
+ assert.Equal(t, expectedResult, result)
+ })
+ t.Run("s2b", func(t *testing.T) {
+ result := s2b("TextforTest")
+ assert.Equal(t, expected, result)
+ })
+ t.Run("b2s", func(t *testing.T) {
+ result := b2s(expected)
+ assert.Equal(t, "TextforTest", result)
+ })
+}
From f6e0defa2bfd6a91f77da4aba6ed1cee14d15936 Mon Sep 17 00:00:00 2001
From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com>
Date: Fri, 29 Dec 2023 19:21:09 +0330
Subject: [PATCH 02/11] add unit-test for sanitizeStyleURL
---
utils_test.go | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/utils_test.go b/utils_test.go
index 4e565da..64cad5f 100644
--- a/utils_test.go
+++ b/utils_test.go
@@ -58,3 +58,41 @@ func TestIsValitdURL(t *testing.T) {
assert.Equal(t, "TextforTest", result)
})
}
+
+func TestSanitizeStyleURL(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ expected string
+ }{
+ {
+ name: "Test with url()",
+ input: "url('https://example.com/')",
+ expected: "https://example.com/",
+ },
+ {
+ name: "Test with double quotes",
+ input: "\"https://example.com/\"",
+ expected: "https://example.com/",
+ },
+ {
+ name: "Test with single quotes",
+ input: "'https://example.com/'",
+ expected: "https://example.com/",
+ },
+ {
+ name: "Test with no quotes",
+ input: "https://example.com/",
+ expected: "https://example.com/",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ result := sanitizeStyleURL(test.input)
+ if result != test.expected {
+ t.Errorf("Expected %s, but got %s", test.expected, result)
+ }
+ })
+ }
+}
From a1ace8dbcdb28c85d04a3c78dd7fff7c70be9e1d Mon Sep 17 00:00:00 2001
From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com>
Date: Fri, 29 Dec 2023 22:42:04 +0330
Subject: [PATCH 03/11] add unit-test for archiver.go
---
archiver_test.go | 199 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 199 insertions(+)
create mode 100644 archiver_test.go
diff --git a/archiver_test.go b/archiver_test.go
new file mode 100644
index 0000000..63faab7
--- /dev/null
+++ b/archiver_test.go
@@ -0,0 +1,199 @@
+package obelisk
+
+import (
+ "context"
+ "net/http"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestArchiver_Validate(t *testing.T) {
+ arc := &Archiver{
+ Cache: nil,
+ UserAgent: "",
+ MaxConcurrentDownload: 0,
+ isValidated: false,
+ dlSemaphore: nil,
+ Transport: nil,
+ RequestTimeout: 0,
+ httpClient: nil,
+ }
+
+ arc.Validate()
+
+ if arc.Cache == nil {
+ t.Error("Cache should not be nil")
+ }
+
+ if arc.UserAgent == "" {
+ t.Error("UserAgent should not be empty")
+ }
+
+ if arc.MaxConcurrentDownload <= 0 {
+ t.Error("MaxConcurrentDownload should be greater than 0")
+ }
+
+ if !arc.isValidated {
+ t.Error("isValidated should be true")
+ }
+
+ if arc.dlSemaphore == nil {
+ t.Error("dlSemaphore should not be nil")
+ }
+
+ if arc.Transport == nil {
+ t.Error("Transport should not be nil")
+ }
+
+ if arc.httpClient == nil {
+ t.Error("httpClient should not be nil")
+ }
+}
+
+func TestArchiver_Archive(t *testing.T) {
+ archiver := &Archiver{
+ Cache: nil,
+ UserAgent: "",
+ MaxConcurrentDownload: 0,
+ isValidated: true,
+ dlSemaphore: nil,
+ Transport: nil,
+ RequestTimeout: 0,
+ httpClient: &http.Client{},
+ }
+
+ // Create a mock request
+ req := Request{
+ URL: "https://example.com",
+ }
+
+ // Call the Archive method and capture the result
+ result, contentType, err := archiver.Archive(context.Background(), req)
+
+ // Check if there was an error
+ if err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ }
+
+ // Check if the result is not empty
+ if len(result) == 0 {
+ t.Errorf("Empty result")
+ }
+
+ // Check if the content type is not empty
+ if contentType == "" {
+ t.Errorf("Empty content type")
+ }
+
+ t.Run("Test isvalidURL", func(t *testing.T) {
+ archiver.isValidated = false
+ result, contentType, err = archiver.Archive(context.Background(), req)
+ assert.Equal(t, []byte(nil), result)
+ assert.Equal(t, "", contentType)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "archiver hasn't been validated")
+ })
+ t.Run("Test url is empty", func(t *testing.T) {
+ archiver.isValidated = true
+ req.URL = ""
+ result, contentType, err = archiver.Archive(context.Background(), req)
+ assert.Equal(t, []byte(nil), result)
+ assert.Equal(t, "", contentType)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "request url is not specified")
+ })
+ t.Run("Test not valid url ", func(t *testing.T) {
+ req.URL = "notValidURL"
+ result, contentType, err = archiver.Archive(context.Background(), req)
+ assert.Equal(t, []byte(nil), result)
+ assert.Equal(t, "", contentType)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "url \"notValidURL\" is not valid")
+ })
+}
+
+func TestTransform(t *testing.T) {
+ // Create a new instance of the Archiver struct
+ //arc := &Archiver{}
+
+ arc := &Archiver{
+ Cache: nil,
+ UserAgent: "",
+ MaxConcurrentDownload: 0,
+ isValidated: true,
+ dlSemaphore: nil,
+ Transport: nil,
+ RequestTimeout: 0,
+ httpClient: &http.Client{},
+ }
+ // Test case 1: No WrapDirectory specified
+ uri := "https://raw.githubusercontent.com/go-shiori/obelisk/master/docs/readme/logo.png"
+ content := []byte("image content")
+ contentType := "image/jpeg"
+
+ result := arc.transform(uri, content, contentType)
+ expected := createDataURL(content, contentType)
+
+ if result != expected {
+ t.Errorf("Expected %s, but got %s", expected, result)
+ }
+
+ // Test case 2: WrapDirectory specified
+ arc.WrapDirectory = "/path/to/directory"
+
+ result = arc.transform(uri, content, contentType)
+ expected = "data:image/jpeg;base64,aW1hZ2UgY29udGVudA=="
+
+ if result != expected {
+ t.Errorf("Expected %s, but got %s", expected, result)
+ }
+}
+
+func TestStore(t *testing.T) {
+ arc := &Archiver{
+ WrapDirectory: "/tmp/some",
+ }
+
+ // Test case 1: Empty URI
+ path, rel, err := arc.store("")
+ if err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ }
+ if path != "" || rel != "" {
+ t.Errorf("Expected empty path and rel, got path: %s, rel: %s", path, rel)
+ }
+
+ // Test case 2: Valid URI
+ uri := "http://example.com/statics/css/foo.css"
+ path, rel, err = arc.store(uri)
+ if err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ }
+ expectedPath := "/tmp/some/statics/css/foo.css"
+ expectedRel := "/statics/css/foo.css"
+ if path != expectedPath || rel != expectedRel {
+ t.Errorf("Expected path: %s, rel: %s, got path: %s, rel: %s", expectedPath, expectedRel, path, rel)
+ }
+
+ // Test case 3: Invalid URI
+ uri = "invalid uri"
+ path, rel, err = arc.store(uri)
+ if err == nil {
+ t.Errorf("Expected error, got nil")
+ }
+ if path != "" || rel != "" {
+ t.Errorf("Expected empty path and rel, got path: %s, rel: %s", path, rel)
+ }
+
+ // Test case 4: Error creating directory
+ arc.WrapDirectory = "/nonexistent"
+ uri = "http://example.com/statics/css/foo.css"
+ path, rel, err = arc.store(uri)
+ if err == nil {
+ t.Errorf("Expected error, got nil")
+ }
+ if path != "" || rel != "" {
+ t.Errorf("Expected empty path and rel, got path: %s, rel: %s", path, rel)
+ }
+}
From e991c4cff498168ff64932b55b48590347f7378c Mon Sep 17 00:00:00 2001
From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com>
Date: Sat, 30 Dec 2023 15:29:58 +0330
Subject: [PATCH 04/11] serve file from localhost
---
archiver_test.go | 13 ++++++++++++-
testdata/index.html | 46 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+), 1 deletion(-)
create mode 100644 testdata/index.html
diff --git a/archiver_test.go b/archiver_test.go
index 63faab7..c884fbe 100644
--- a/archiver_test.go
+++ b/archiver_test.go
@@ -3,11 +3,15 @@ package obelisk
import (
"context"
"net/http"
+ "net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
)
+func servefiles() {
+}
+
func TestArchiver_Validate(t *testing.T) {
arc := &Archiver{
Cache: nil,
@@ -52,6 +56,13 @@ func TestArchiver_Validate(t *testing.T) {
}
func TestArchiver_Archive(t *testing.T) {
+ fs := http.FileServer(http.Dir("./testdata/"))
+
+ // start a test server with the file server handler
+ server := httptest.NewServer(fs)
+
+ defer server.Close()
+
archiver := &Archiver{
Cache: nil,
UserAgent: "",
@@ -65,7 +76,7 @@ func TestArchiver_Archive(t *testing.T) {
// Create a mock request
req := Request{
- URL: "https://example.com",
+ URL: server.URL,
}
// Call the Archive method and capture the result
diff --git a/testdata/index.html b/testdata/index.html
new file mode 100644
index 0000000..098f596
--- /dev/null
+++ b/testdata/index.html
@@ -0,0 +1,46 @@
+
+
+ Example Domain
+
+
+
+
+
+
+
+
+
+
Example Domain
+
This domain is for use in illustrative examples in documents. You may use this
+ domain in literature without prior coordination or asking for permission.