From 9a878026303dbd06fada7e384b93dbb02d85e8ab Mon Sep 17 00:00:00 2001 From: t-takahashi Date: Tue, 16 Jul 2024 23:04:47 +0900 Subject: [PATCH 1/4] add isTruncated --- .github/workflows/go.yml | 2 +- core/amazon/awssdk/listobjects_v2.go | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index db3e764..49aa950 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.22.4' + go-version: '1.22.5' - name: Build run: go build -v ./... diff --git a/core/amazon/awssdk/listobjects_v2.go b/core/amazon/awssdk/listobjects_v2.go index ea55d9f..7220714 100644 --- a/core/amazon/awssdk/listobjects_v2.go +++ b/core/amazon/awssdk/listobjects_v2.go @@ -10,11 +10,12 @@ import ( ) type ListBucketResult struct { - XMLName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - Prefix string `xml:"Prefix"` - Marker string `xml:"Marker"` - Items []Item `xml:"Contents"` + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Marker string `xml:"Marker"` + Items []Item `xml:"Contents"` + IsTruncated bool `xml:"IsTruncated"` } type Item struct { @@ -25,7 +26,7 @@ type Item struct { func ListObjectsV2(w http.ResponseWriter, r *http.Request) { slog.Info("ListObjectsV2 is called.") - path := strings.Split(r.URL.Path, "?list-type=2")[0] // It has been confirmed in the previous process controller.go that `?list-type=2` is included. + path := strings.Split(r.URL.Path, "?list-type=2")[0] dir := strings.TrimPrefix(path, "/") if dir == "" { slog.Error("No directory specified in the query parameter") @@ -33,7 +34,6 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { return } - // Define the target directory rootDir := filepath.Join("upload", dir) var items []Item @@ -43,7 +43,7 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { } if !info.IsDir() { items = append(items, Item{ - Key: filepath.ToSlash(path[len("upload/"):]), // Convert to a relative path + Key: filepath.ToSlash(path[len("upload/"):]), Size: info.Size(), }) } @@ -57,9 +57,17 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { slog.Info("Files are below", "files", items) + isTruncated := false + // Add logic to determine if the result is truncated + if len(items) > 1000 { + isTruncated = true + items = items[:1000] + } + response := ListBucketResult{ - Name: dir, - Items: items, + Name: dir, + Items: items, + IsTruncated: isTruncated, } w.Header().Set("Content-Type", "application/xml") From 7fc7a80dee90e0ca9a98f0acfe0f91dd0ae777f7 Mon Sep 17 00:00:00 2001 From: t-takahashi Date: Wed, 17 Jul 2024 07:20:52 +0900 Subject: [PATCH 2/4] fix listObjctsv2 --- core/amazon/awssdk/listobjects_v2.go | 53 +++++++++++++---------- core/amazon/awssdk/listobjects_v2_test.go | 36 +++++++++++++++ 2 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 core/amazon/awssdk/listobjects_v2_test.go diff --git a/core/amazon/awssdk/listobjects_v2.go b/core/amazon/awssdk/listobjects_v2.go index 7220714..81e5fb7 100644 --- a/core/amazon/awssdk/listobjects_v2.go +++ b/core/amazon/awssdk/listobjects_v2.go @@ -9,7 +9,7 @@ import ( "strings" ) -type ListBucketResult struct { +type ListObjectsResult struct { XMLName xml.Name `xml:"ListBucketResult"` Name string `xml:"Name"` Prefix string `xml:"Prefix"` @@ -26,7 +26,7 @@ type Item struct { func ListObjectsV2(w http.ResponseWriter, r *http.Request) { slog.Info("ListObjectsV2 is called.") - path := strings.Split(r.URL.Path, "?list-type=2")[0] + path := strings.Split(r.URL.Path, "?list-type=2")[0] // It has been confirmed in the previous process controller.go that `?list-type=2` is included. dir := strings.TrimPrefix(path, "/") if dir == "" { slog.Error("No directory specified in the query parameter") @@ -36,19 +36,7 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { rootDir := filepath.Join("upload", dir) - var items []Item - err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - items = append(items, Item{ - Key: filepath.ToSlash(path[len("upload/"):]), - Size: info.Size(), - }) - } - return nil - }) + items, err := ListObjects(rootDir) if err != nil { slog.Error("Failed to list objects", "error", err) http.Error(w, "Failed to list objects", http.StatusInternalServerError) @@ -57,14 +45,9 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { slog.Info("Files are below", "files", items) - isTruncated := false - // Add logic to determine if the result is truncated - if len(items) > 1000 { - isTruncated = true - items = items[:1000] - } + isTruncated, items := IsTruncated(items) - response := ListBucketResult{ + response := ListObjectsResult{ Name: dir, Items: items, IsTruncated: isTruncated, @@ -76,3 +59,29 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { http.Error(w, "Failed to encode response", http.StatusInternalServerError) } } + +func ListObjects(rootDir string) ([]Item, error) { + var items []Item + err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + items = append(items, Item{ + Key: filepath.ToSlash(path[len("upload/"):]), + Size: info.Size(), + }) + } + return nil + }) + + return items, err +} + +func IsTruncated(items []Item) (bool, []Item) { + if len(items) > 1000 { + return true, items[:1000] + } else { + return false, items + } +} diff --git a/core/amazon/awssdk/listobjects_v2_test.go b/core/amazon/awssdk/listobjects_v2_test.go new file mode 100644 index 0000000..84b8097 --- /dev/null +++ b/core/amazon/awssdk/listobjects_v2_test.go @@ -0,0 +1,36 @@ +package awssdk + +import ( + "testing" + + "github.com/go-playground/assert/v2" +) + +func TestIsTruncated(t *testing.T) { + items1 := make([]Item, 1) + items999 := make([]Item, 999) + items1000 := make([]Item, 1000) + items1001 := make([]Item, 1001) + items1200 := make([]Item, 1200) + + isTruncated1, actualItems1 := IsTruncated(items1) + assert.Equal(t, isTruncated1, false) + assert.Equal(t, actualItems1, items1) + + isTruncated999, actualItems999 := IsTruncated(items999) + assert.Equal(t, isTruncated999, false) + assert.Equal(t, actualItems999, items999) + + isTruncated1000, actualItems1000 := IsTruncated(items1000) + assert.Equal(t, isTruncated1000, false) + assert.Equal(t, actualItems1000, items1000) + + isTruncated1001, actualItems1001 := IsTruncated(items1001) + assert.Equal(t, isTruncated1001, true) + assert.Equal(t, actualItems1001, items1001[:1000]) + + isTruncated1200, actualItems1200 := IsTruncated(items1200) + assert.Equal(t, isTruncated1200, true) + assert.Equal(t, actualItems1200, items1200[:1000]) + +} From 87cf719ba7f890d5fe551c23328680484fb62156 Mon Sep 17 00:00:00 2001 From: t-takahashi Date: Wed, 17 Jul 2024 07:57:07 +0900 Subject: [PATCH 3/4] add TestListObjects --- core/amazon/awssdk/listobjects_v2.go | 10 ++--- core/amazon/awssdk/listobjects_v2_test.go | 52 +++++++++++++++++++++++ web/constant.go | 3 +- web/controller.go | 2 +- web/dir_controller.go | 4 +- web/rename_controller.go | 4 +- web/rm_controller.go | 2 +- web/s3_controller.go | 4 +- web/upload_controller.go | 2 +- 9 files changed, 68 insertions(+), 15 deletions(-) diff --git a/core/amazon/awssdk/listobjects_v2.go b/core/amazon/awssdk/listobjects_v2.go index 81e5fb7..ac05147 100644 --- a/core/amazon/awssdk/listobjects_v2.go +++ b/core/amazon/awssdk/listobjects_v2.go @@ -23,7 +23,7 @@ type Item struct { Size int64 `xml:"Size"` } -func ListObjectsV2(w http.ResponseWriter, r *http.Request) { +func ListObjectsV2(w http.ResponseWriter, r *http.Request, uploadDirName string) { slog.Info("ListObjectsV2 is called.") path := strings.Split(r.URL.Path, "?list-type=2")[0] // It has been confirmed in the previous process controller.go that `?list-type=2` is included. @@ -34,9 +34,9 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { return } - rootDir := filepath.Join("upload", dir) + rootDir := filepath.Join(uploadDirName, dir) - items, err := ListObjects(rootDir) + items, err := ListObjects(rootDir, uploadDirName) if err != nil { slog.Error("Failed to list objects", "error", err) http.Error(w, "Failed to list objects", http.StatusInternalServerError) @@ -60,7 +60,7 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { } } -func ListObjects(rootDir string) ([]Item, error) { +func ListObjects(rootDir string, uploadDirName string) ([]Item, error) { var items []Item err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -68,7 +68,7 @@ func ListObjects(rootDir string) ([]Item, error) { } if !info.IsDir() { items = append(items, Item{ - Key: filepath.ToSlash(path[len("upload/"):]), + Key: filepath.ToSlash(path[len(uploadDirName+"/"):]), Size: info.Size(), }) } diff --git a/core/amazon/awssdk/listobjects_v2_test.go b/core/amazon/awssdk/listobjects_v2_test.go index 84b8097..bb3ab64 100644 --- a/core/amazon/awssdk/listobjects_v2_test.go +++ b/core/amazon/awssdk/listobjects_v2_test.go @@ -1,11 +1,63 @@ package awssdk import ( + "os" + "path/filepath" "testing" "github.com/go-playground/assert/v2" ) +func TestListObjects(t *testing.T) { + // Create a temporary directory for testing + tempDir := "test-upload" + err := os.Mkdir(tempDir, 0777) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Create test files and directories + err = os.MkdirAll(filepath.Join(tempDir, "dir1"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(tempDir, "file1.txt"), []byte("file1 content"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(tempDir, "dir1", "file2.txt"), []byte("file2 content"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + // Call the ListObjects function + items, err := ListObjects(filepath.Join(tempDir), "test-upload") + if err != nil { + t.Fatalf("ListObjects returned an error: %v", err) + } + + // Define expected results + expectedItems := []Item{ + {Key: "dir1/file2.txt", Size: int64(len("file2 content"))}, + {Key: "file1.txt", Size: int64(len("file1 content"))}, + } + + // Check the number of items + if len(items) != len(expectedItems) { + t.Fatalf("Expected %d items, got %d", len(expectedItems), len(items)) + } + + // Check each item + for i, item := range items { + if item.Key != expectedItems[i].Key || item.Size != expectedItems[i].Size { + t.Errorf("Expected item %v, got %v", expectedItems[i], item) + } + } +} + func TestIsTruncated(t *testing.T) { items1 := make([]Item, 1) items999 := make([]Item, 999) diff --git a/web/constant.go b/web/constant.go index bc400f6..27964bc 100644 --- a/web/constant.go +++ b/web/constant.go @@ -1,5 +1,6 @@ package web const ( - UPLOAD_DIR = "./upload" + UPLOAD_DIR_PATH = "./upload" + UPLOAD_DIR_NAME = "upload" ) diff --git a/web/controller.go b/web/controller.go index 86b3f16..2bc66bf 100644 --- a/web/controller.go +++ b/web/controller.go @@ -27,7 +27,7 @@ func CliSdkHandler(w http.ResponseWriter, r *http.Request) { if strings.Contains(userAgent, "aws-sdk") { if r.Method == "GET" { if r.URL.Query().Get("list-type") == "2" { - awssdk.ListObjectsV2(w, r) + awssdk.ListObjectsV2(w, r, UPLOAD_DIR_NAME) } else { awssdk.Get(w, r) } diff --git a/web/dir_controller.go b/web/dir_controller.go index 6c42db2..6b7ff75 100644 --- a/web/dir_controller.go +++ b/web/dir_controller.go @@ -15,7 +15,7 @@ func MkdirHandler(w http.ResponseWriter, r *http.Request) { currentPath := r.FormValue("currentPath") dirname := r.FormValue("dirname") - dir := filepath.Join(UPLOAD_DIR, currentPath, dirname) + dir := filepath.Join(UPLOAD_DIR_PATH, currentPath, dirname) err := os.Mkdir(dir, os.ModePerm) if err != nil { slog.Error("failed to mkdir", "target directory", dir, "error", err) @@ -34,7 +34,7 @@ func RmdirHandler(w http.ResponseWriter, r *http.Request) { } dirname := r.FormValue("dirname") - err := os.Remove(filepath.Join(UPLOAD_DIR, dirname)) + err := os.Remove(filepath.Join(UPLOAD_DIR_PATH, dirname)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/web/rename_controller.go b/web/rename_controller.go index 63e48a5..8f1ea76 100644 --- a/web/rename_controller.go +++ b/web/rename_controller.go @@ -14,7 +14,7 @@ func RenameHandler(w http.ResponseWriter, r *http.Request) { oldFilename := r.FormValue("oldFilename") newFilename := r.FormValue("newFilename") - err := os.Rename(filepath.Join(UPLOAD_DIR, oldFilename), filepath.Join(UPLOAD_DIR, newFilename)) + err := os.Rename(filepath.Join(UPLOAD_DIR_PATH, oldFilename), filepath.Join(UPLOAD_DIR_PATH, newFilename)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -31,7 +31,7 @@ func RenamedirHandler(w http.ResponseWriter, r *http.Request) { oldDirname := r.FormValue("oldDirname") newDirname := r.FormValue("newDirname") - err := os.Rename(filepath.Join(UPLOAD_DIR, oldDirname), filepath.Join(UPLOAD_DIR, newDirname)) + err := os.Rename(filepath.Join(UPLOAD_DIR_PATH, oldDirname), filepath.Join(UPLOAD_DIR_PATH, newDirname)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/web/rm_controller.go b/web/rm_controller.go index efcc9a8..efeaf87 100644 --- a/web/rm_controller.go +++ b/web/rm_controller.go @@ -16,7 +16,7 @@ func RemoveHandler(w http.ResponseWriter, r *http.Request) { } path := r.FormValue("path")[len("/s3/"):] - err := os.Remove(filepath.Join(UPLOAD_DIR, path)) + err := os.Remove(filepath.Join(UPLOAD_DIR_PATH, path)) if err != nil { slog.Error("Failed to remove.", "error", err, "path", path) http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/web/s3_controller.go b/web/s3_controller.go index 4241034..fe59487 100644 --- a/web/s3_controller.go +++ b/web/s3_controller.go @@ -16,7 +16,7 @@ func S3Handler(w http.ResponseWriter, r *http.Request) { slog.Info("S3Handler is called.") path := r.URL.Path[len("/s3/"):] - s3Objects, err := util.GenerateS3Objects(r, UPLOAD_DIR, GetDirPath(path)) + s3Objects, err := util.GenerateS3Objects(r, UPLOAD_DIR_PATH, GetDirPath(path)) if err != nil { slog.Error("GenerateS3Objects error", "error", err) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -59,7 +59,7 @@ func S3Handler(w http.ResponseWriter, r *http.Request) { func download(w http.ResponseWriter, path string) (n int64, httpStatus int, err error) { slog.Info("Start downloading file", "path", path) - file, err := os.Open(filepath.Join(UPLOAD_DIR, path)) + file, err := os.Open(filepath.Join(UPLOAD_DIR_PATH, path)) if err != nil { slog.Error("File open error", "error", err) return 0, http.StatusNotFound, err diff --git a/web/upload_controller.go b/web/upload_controller.go index e33a16a..93e58b0 100644 --- a/web/upload_controller.go +++ b/web/upload_controller.go @@ -37,7 +37,7 @@ func UploadHandler(w http.ResponseWriter, r *http.Request) { } defer file.Close() - dst, err := os.Create(filepath.Join(UPLOAD_DIR, path, header.Filename)) + dst, err := os.Create(filepath.Join(UPLOAD_DIR_PATH, path, header.Filename)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return From e2d26ddfb0cd3f698b21a0b65d80210711832efb Mon Sep 17 00:00:00 2001 From: t-takahashi Date: Wed, 17 Jul 2024 08:03:26 +0900 Subject: [PATCH 4/4] fix test --- core/amazon/awssdk/listobjects_v2_test.go | 24 +++++++++-------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/core/amazon/awssdk/listobjects_v2_test.go b/core/amazon/awssdk/listobjects_v2_test.go index bb3ab64..f66a859 100644 --- a/core/amazon/awssdk/listobjects_v2_test.go +++ b/core/amazon/awssdk/listobjects_v2_test.go @@ -9,7 +9,7 @@ import ( ) func TestListObjects(t *testing.T) { - // Create a temporary directory for testing + // Create a temporary directory and file for testing tempDir := "test-upload" err := os.Mkdir(tempDir, 0777) if err != nil { @@ -17,7 +17,6 @@ func TestListObjects(t *testing.T) { } defer os.RemoveAll(tempDir) - // Create test files and directories err = os.MkdirAll(filepath.Join(tempDir, "dir1"), os.ModePerm) if err != nil { t.Fatal(err) @@ -33,28 +32,23 @@ func TestListObjects(t *testing.T) { t.Fatal(err) } - // Call the ListObjects function - items, err := ListObjects(filepath.Join(tempDir), "test-upload") + // actual + actualItems, err := ListObjects(filepath.Join(tempDir), "test-upload") if err != nil { t.Fatalf("ListObjects returned an error: %v", err) } - // Define expected results + // expected expectedItems := []Item{ {Key: "dir1/file2.txt", Size: int64(len("file2 content"))}, {Key: "file1.txt", Size: int64(len("file1 content"))}, } - // Check the number of items - if len(items) != len(expectedItems) { - t.Fatalf("Expected %d items, got %d", len(expectedItems), len(items)) - } - - // Check each item - for i, item := range items { - if item.Key != expectedItems[i].Key || item.Size != expectedItems[i].Size { - t.Errorf("Expected item %v, got %v", expectedItems[i], item) - } + // assertion + assert.Equal(t, len(actualItems), len(expectedItems)) + for i, item := range actualItems { + assert.Equal(t, item.Key, expectedItems[i].Key) + assert.Equal(t, item.Size, expectedItems[i].Size) } }