From 6b97e8454b816dbaecc941a74e170441a560aa2e Mon Sep 17 00:00:00 2001 From: TOL Date: Wed, 17 Jul 2024 08:04:19 +0900 Subject: [PATCH] Add isTruncated & add test code (#15) * add isTruncated * fix listObjctsv2 * add TestListObjects * fix test --- .github/workflows/go.yml | 2 +- core/amazon/awssdk/listobjects_v2.go | 67 +++++++++++------- core/amazon/awssdk/listobjects_v2_test.go | 82 +++++++++++++++++++++++ web/constant.go | 3 +- web/controller.go | 2 +- web/dir_controller.go | 4 +- web/rename_controller.go | 4 +- web/rm_controller.go | 2 +- web/s3_controller.go | 4 +- web/upload_controller.go | 2 +- 10 files changed, 136 insertions(+), 36 deletions(-) create mode 100644 core/amazon/awssdk/listobjects_v2_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index db3e764..49aa950 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.22.4' + go-version: '1.22.5' - name: Build run: go build -v ./... diff --git a/core/amazon/awssdk/listobjects_v2.go b/core/amazon/awssdk/listobjects_v2.go index ea55d9f..ac05147 100644 --- a/core/amazon/awssdk/listobjects_v2.go +++ b/core/amazon/awssdk/listobjects_v2.go @@ -9,12 +9,13 @@ import ( "strings" ) -type ListBucketResult struct { - XMLName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - Prefix string `xml:"Prefix"` - Marker string `xml:"Marker"` - Items []Item `xml:"Contents"` +type ListObjectsResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Marker string `xml:"Marker"` + Items []Item `xml:"Contents"` + IsTruncated bool `xml:"IsTruncated"` } type Item struct { @@ -22,7 +23,7 @@ type Item struct { Size int64 `xml:"Size"` } -func ListObjectsV2(w http.ResponseWriter, r *http.Request) { +func ListObjectsV2(w http.ResponseWriter, r *http.Request, uploadDirName string) { slog.Info("ListObjectsV2 is called.") path := strings.Split(r.URL.Path, "?list-type=2")[0] // It has been confirmed in the previous process controller.go that `?list-type=2` is included. @@ -33,22 +34,9 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { return } - // Define the target directory - rootDir := filepath.Join("upload", dir) + rootDir := filepath.Join(uploadDirName, dir) - var items []Item - err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - items = append(items, Item{ - Key: filepath.ToSlash(path[len("upload/"):]), // Convert to a relative path - Size: info.Size(), - }) - } - return nil - }) + items, err := ListObjects(rootDir, uploadDirName) if err != nil { slog.Error("Failed to list objects", "error", err) http.Error(w, "Failed to list objects", http.StatusInternalServerError) @@ -57,9 +45,12 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { slog.Info("Files are below", "files", items) - response := ListBucketResult{ - Name: dir, - Items: items, + isTruncated, items := IsTruncated(items) + + response := ListObjectsResult{ + Name: dir, + Items: items, + IsTruncated: isTruncated, } w.Header().Set("Content-Type", "application/xml") @@ -68,3 +59,29 @@ func ListObjectsV2(w http.ResponseWriter, r *http.Request) { http.Error(w, "Failed to encode response", http.StatusInternalServerError) } } + +func ListObjects(rootDir string, uploadDirName string) ([]Item, error) { + var items []Item + err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + items = append(items, Item{ + Key: filepath.ToSlash(path[len(uploadDirName+"/"):]), + Size: info.Size(), + }) + } + return nil + }) + + return items, err +} + +func IsTruncated(items []Item) (bool, []Item) { + if len(items) > 1000 { + return true, items[:1000] + } else { + return false, items + } +} diff --git a/core/amazon/awssdk/listobjects_v2_test.go b/core/amazon/awssdk/listobjects_v2_test.go new file mode 100644 index 0000000..f66a859 --- /dev/null +++ b/core/amazon/awssdk/listobjects_v2_test.go @@ -0,0 +1,82 @@ +package awssdk + +import ( + "os" + "path/filepath" + "testing" + + "github.com/go-playground/assert/v2" +) + +func TestListObjects(t *testing.T) { + // Create a temporary directory and file for testing + tempDir := "test-upload" + err := os.Mkdir(tempDir, 0777) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + err = os.MkdirAll(filepath.Join(tempDir, "dir1"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(tempDir, "file1.txt"), []byte("file1 content"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile(filepath.Join(tempDir, "dir1", "file2.txt"), []byte("file2 content"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + + // actual + actualItems, err := ListObjects(filepath.Join(tempDir), "test-upload") + if err != nil { + t.Fatalf("ListObjects returned an error: %v", err) + } + + // expected + expectedItems := []Item{ + {Key: "dir1/file2.txt", Size: int64(len("file2 content"))}, + {Key: "file1.txt", Size: int64(len("file1 content"))}, + } + + // assertion + assert.Equal(t, len(actualItems), len(expectedItems)) + for i, item := range actualItems { + assert.Equal(t, item.Key, expectedItems[i].Key) + assert.Equal(t, item.Size, expectedItems[i].Size) + } +} + +func TestIsTruncated(t *testing.T) { + items1 := make([]Item, 1) + items999 := make([]Item, 999) + items1000 := make([]Item, 1000) + items1001 := make([]Item, 1001) + items1200 := make([]Item, 1200) + + isTruncated1, actualItems1 := IsTruncated(items1) + assert.Equal(t, isTruncated1, false) + assert.Equal(t, actualItems1, items1) + + isTruncated999, actualItems999 := IsTruncated(items999) + assert.Equal(t, isTruncated999, false) + assert.Equal(t, actualItems999, items999) + + isTruncated1000, actualItems1000 := IsTruncated(items1000) + assert.Equal(t, isTruncated1000, false) + assert.Equal(t, actualItems1000, items1000) + + isTruncated1001, actualItems1001 := IsTruncated(items1001) + assert.Equal(t, isTruncated1001, true) + assert.Equal(t, actualItems1001, items1001[:1000]) + + isTruncated1200, actualItems1200 := IsTruncated(items1200) + assert.Equal(t, isTruncated1200, true) + assert.Equal(t, actualItems1200, items1200[:1000]) + +} diff --git a/web/constant.go b/web/constant.go index bc400f6..27964bc 100644 --- a/web/constant.go +++ b/web/constant.go @@ -1,5 +1,6 @@ package web const ( - UPLOAD_DIR = "./upload" + UPLOAD_DIR_PATH = "./upload" + UPLOAD_DIR_NAME = "upload" ) diff --git a/web/controller.go b/web/controller.go index 86b3f16..2bc66bf 100644 --- a/web/controller.go +++ b/web/controller.go @@ -27,7 +27,7 @@ func CliSdkHandler(w http.ResponseWriter, r *http.Request) { if strings.Contains(userAgent, "aws-sdk") { if r.Method == "GET" { if r.URL.Query().Get("list-type") == "2" { - awssdk.ListObjectsV2(w, r) + awssdk.ListObjectsV2(w, r, UPLOAD_DIR_NAME) } else { awssdk.Get(w, r) } diff --git a/web/dir_controller.go b/web/dir_controller.go index 6c42db2..6b7ff75 100644 --- a/web/dir_controller.go +++ b/web/dir_controller.go @@ -15,7 +15,7 @@ func MkdirHandler(w http.ResponseWriter, r *http.Request) { currentPath := r.FormValue("currentPath") dirname := r.FormValue("dirname") - dir := filepath.Join(UPLOAD_DIR, currentPath, dirname) + dir := filepath.Join(UPLOAD_DIR_PATH, currentPath, dirname) err := os.Mkdir(dir, os.ModePerm) if err != nil { slog.Error("failed to mkdir", "target directory", dir, "error", err) @@ -34,7 +34,7 @@ func RmdirHandler(w http.ResponseWriter, r *http.Request) { } dirname := r.FormValue("dirname") - err := os.Remove(filepath.Join(UPLOAD_DIR, dirname)) + err := os.Remove(filepath.Join(UPLOAD_DIR_PATH, dirname)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/web/rename_controller.go b/web/rename_controller.go index 63e48a5..8f1ea76 100644 --- a/web/rename_controller.go +++ b/web/rename_controller.go @@ -14,7 +14,7 @@ func RenameHandler(w http.ResponseWriter, r *http.Request) { oldFilename := r.FormValue("oldFilename") newFilename := r.FormValue("newFilename") - err := os.Rename(filepath.Join(UPLOAD_DIR, oldFilename), filepath.Join(UPLOAD_DIR, newFilename)) + err := os.Rename(filepath.Join(UPLOAD_DIR_PATH, oldFilename), filepath.Join(UPLOAD_DIR_PATH, newFilename)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -31,7 +31,7 @@ func RenamedirHandler(w http.ResponseWriter, r *http.Request) { oldDirname := r.FormValue("oldDirname") newDirname := r.FormValue("newDirname") - err := os.Rename(filepath.Join(UPLOAD_DIR, oldDirname), filepath.Join(UPLOAD_DIR, newDirname)) + err := os.Rename(filepath.Join(UPLOAD_DIR_PATH, oldDirname), filepath.Join(UPLOAD_DIR_PATH, newDirname)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/web/rm_controller.go b/web/rm_controller.go index efcc9a8..efeaf87 100644 --- a/web/rm_controller.go +++ b/web/rm_controller.go @@ -16,7 +16,7 @@ func RemoveHandler(w http.ResponseWriter, r *http.Request) { } path := r.FormValue("path")[len("/s3/"):] - err := os.Remove(filepath.Join(UPLOAD_DIR, path)) + err := os.Remove(filepath.Join(UPLOAD_DIR_PATH, path)) if err != nil { slog.Error("Failed to remove.", "error", err, "path", path) http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/web/s3_controller.go b/web/s3_controller.go index 4241034..fe59487 100644 --- a/web/s3_controller.go +++ b/web/s3_controller.go @@ -16,7 +16,7 @@ func S3Handler(w http.ResponseWriter, r *http.Request) { slog.Info("S3Handler is called.") path := r.URL.Path[len("/s3/"):] - s3Objects, err := util.GenerateS3Objects(r, UPLOAD_DIR, GetDirPath(path)) + s3Objects, err := util.GenerateS3Objects(r, UPLOAD_DIR_PATH, GetDirPath(path)) if err != nil { slog.Error("GenerateS3Objects error", "error", err) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -59,7 +59,7 @@ func S3Handler(w http.ResponseWriter, r *http.Request) { func download(w http.ResponseWriter, path string) (n int64, httpStatus int, err error) { slog.Info("Start downloading file", "path", path) - file, err := os.Open(filepath.Join(UPLOAD_DIR, path)) + file, err := os.Open(filepath.Join(UPLOAD_DIR_PATH, path)) if err != nil { slog.Error("File open error", "error", err) return 0, http.StatusNotFound, err diff --git a/web/upload_controller.go b/web/upload_controller.go index e33a16a..93e58b0 100644 --- a/web/upload_controller.go +++ b/web/upload_controller.go @@ -37,7 +37,7 @@ func UploadHandler(w http.ResponseWriter, r *http.Request) { } defer file.Close() - dst, err := os.Create(filepath.Join(UPLOAD_DIR, path, header.Filename)) + dst, err := os.Create(filepath.Join(UPLOAD_DIR_PATH, path, header.Filename)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return