From 0a126c5291411301d97826949309a313d6b3e04b Mon Sep 17 00:00:00 2001 From: jojo Date: Wed, 11 Oct 2023 14:24:46 -0300 Subject: [PATCH 1/5] :sparkles: feat: read/write operations --- metrica_test.go | 20 +------------------- storage.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ storage_test.go | 34 +++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 storage.go create mode 100644 storage_test.go diff --git a/metrica_test.go b/metrica_test.go index d6ebad5..f763665 100644 --- a/metrica_test.go +++ b/metrica_test.go @@ -8,25 +8,7 @@ import ( "testing" ) -func TestHandlerCount(t *testing.T) { - req := httptest.NewRequest("GET", "/count", nil) - w := httptest.NewRecorder() - c := AtomicCounter{} - mux := Handler(&c) - mux.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Errorf("expected status OK; got %v", w.Code) - } - - var got countResponse - if err := json.NewDecoder(w.Body).Decode(&got); err != nil { - t.Errorf("unable to decode body: %v", err) - } - - assert(t, 1, got.Count) -} - -func TestHandlerCount_Multiple(t *testing.T) { +func TestHandlerCount_Sequential(t *testing.T) { c := NewAtomicCounter() mux := Handler(c) diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..17c2eb8 --- /dev/null +++ b/storage.go @@ -0,0 +1,50 @@ +package metrica + +import ( + "fmt" + "io" + "os" + "strings" + "time" +) + +type Counter struct { + Datetime string +} + +func Write(f *os.File, c Counter) error { + _, err := f.WriteString(fmt.Sprintf("%v\n", c.Datetime)) + if err != nil { + return fmt.Errorf("error writing file: %v", err) + } + return nil +} + +func Read(filename string) ([]time.Time, error) { + f, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("error opening file: %v", err) + } + defer f.Close() + + fileData, err := io.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("error reading file: %v", err) + } + + datetimes := strings.Split(string(fileData), "\n") + if datetimes[len(datetimes)-1] == "" { + datetimes = datetimes[:len(datetimes)-1] + } + + var parsedDatetime []time.Time + for _, datetime := range datetimes { + t, err := time.Parse(time.RFC3339Nano, datetime) + if err != nil { + return nil, fmt.Errorf("error parsing datetime: %v", err) + } + parsedDatetime = append(parsedDatetime, t) + + } + return parsedDatetime, nil +} diff --git a/storage_test.go b/storage_test.go new file mode 100644 index 0000000..fdaba66 --- /dev/null +++ b/storage_test.go @@ -0,0 +1,34 @@ +package metrica + +import ( + "os" + "testing" + "time" +) + +func TestWriteRead(t *testing.T) { + c := Counter{ + Datetime: time.Now().Format(time.RFC3339Nano), + } + + f, err := os.CreateTemp("", "test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer os.Remove(f.Name()) + + if err := Write(f, c); err != nil { + t.Fatalf("error writing to file: %v", err) + } + + got, err := Read(f.Name()) + if err != nil { + t.Fatalf("error reading file: %v", err) + } + + if len(got) == 1 { + assert(t, got[0].Format(time.RFC3339Nano), c.Datetime) + } else { + t.Fatalf("expected 1; got %v", len(got)) + } +} From fc154c4566223c4f3ac12da26db50a13fb4197d2 Mon Sep 17 00:00:00 2001 From: jojo Date: Wed, 11 Oct 2023 14:38:40 -0300 Subject: [PATCH 2/5] :bug: chore: lint --- Makefile | 2 +- go.mod | 2 +- storage.go | 8 +++++++- storage_test.go | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 26bc7ae..374a8ca 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -export GO_VERSION=1.21.1 +export GO_VERSION=1.21.3 export GOLANGCI_LINT_VERSION=v1.54.0 # configuration/aliases diff --git a/go.mod b/go.mod index 59cbd87..2c73531 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/perebaj/metrica -go 1.21.0 +go 1.21.3 require ( github.com/golangci/golangci-lint v1.54.2 diff --git a/storage.go b/storage.go index 17c2eb8..ab91ad9 100644 --- a/storage.go +++ b/storage.go @@ -8,10 +8,12 @@ import ( "time" ) +// Counter is a struct that holds the datetime of the request type Counter struct { Datetime string } +// Write writes the datetime to the file func Write(f *os.File, c Counter) error { _, err := f.WriteString(fmt.Sprintf("%v\n", c.Datetime)) if err != nil { @@ -20,12 +22,16 @@ func Write(f *os.File, c Counter) error { return nil } +// Read reads the datetime from the file func Read(filename string) ([]time.Time, error) { f, err := os.Open(filename) if err != nil { return nil, fmt.Errorf("error opening file: %v", err) } - defer f.Close() + + defer func() { + _ = f.Close() + }() fileData, err := io.ReadAll(f) if err != nil { diff --git a/storage_test.go b/storage_test.go index fdaba66..a14e500 100644 --- a/storage_test.go +++ b/storage_test.go @@ -15,7 +15,9 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatalf("error creating temp file: %v", err) } - defer os.Remove(f.Name()) + defer func() { + _ = os.Remove(f.Name()) + }() if err := Write(f, c); err != nil { t.Fatalf("error writing to file: %v", err) From fe7a8411a6a7a871d42cda8f3948a5d0803aca99 Mon Sep 17 00:00:00 2001 From: jojo Date: Wed, 11 Oct 2023 19:40:31 -0300 Subject: [PATCH 3/5] :sparkles: feaT: concurrent fs operations --- Makefile | 2 +- cmd/metrica/main.go | 4 +- handler.go | 69 ++++++++++++++++++++++++ handler_test.go | 125 ++++++++++++++++++++++++++++++++++++++++++++ metrica.go | 36 ------------- metrica_test.go | 59 --------------------- storage.go | 75 ++++++++++++++++++++++---- storage_test.go | 32 +++++++++--- 8 files changed, 288 insertions(+), 114 deletions(-) create mode 100644 handler.go create mode 100644 handler_test.go delete mode 100644 metrica.go delete mode 100644 metrica_test.go diff --git a/Makefile b/Makefile index 374a8ca..6259b97 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ devrun=docker run $(devrunopts) --rm \ ## run isolated tests .PHONY: test test: - go test -run="$(testcase)" ./... -timeout 10s -race -shuffle on + go test -run="$(testcase)" ./... -cover -race -shuffle on ## Run lint .PHONY: lint diff --git a/cmd/metrica/main.go b/cmd/metrica/main.go index 023b798..6330521 100644 --- a/cmd/metrica/main.go +++ b/cmd/metrica/main.go @@ -4,14 +4,16 @@ package main import ( "log/slog" "net/http" + "sync" "github.com/perebaj/metrica" ) func main() { c := metrica.NewAtomicCounter() + fs := metrica.NewFileStorage(&sync.Mutex{}, "counters.txt") + mux := metrica.Handler(c, fs) - mux := metrica.Handler(c) slog.Info("Starting server", "port", 8080) err := http.ListenAndServe(":8080", mux) diff --git a/handler.go b/handler.go new file mode 100644 index 0000000..0810c4e --- /dev/null +++ b/handler.go @@ -0,0 +1,69 @@ +package metrica + +import ( + "encoding/json" + "net/http" + "sync" + "time" + + "log/slog" +) + +type countResponse struct { + Count int64 +} + +// Handler returns a http.Handler for new endpoints. +func Handler(c *AtomicCounter, fs *FileStorage) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/count", func(w http.ResponseWriter, r *http.Request) { + counter(w, r, c) + }) + mux.HandleFunc("/countfs", func(w http.ResponseWriter, r *http.Request) { + counterFs(w, r, fs) + }) + return mux +} + +func counter(w http.ResponseWriter, _ *http.Request, c *AtomicCounter) { + c.Inc(1) + send(w, http.StatusOK, countResponse{Count: c.Value()}) +} + +func counterFs(w http.ResponseWriter, _ *http.Request, fs *FileStorage) { + var wg sync.WaitGroup + var counter int64 + + c := Counter(time.Now()) + + for i := 0; i < 1; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := fs.Write(c) + if err != nil { + send(w, http.StatusInternalServerError, err) + return + } + + counters, err := fs.Read() + if err != nil { + send(w, http.StatusInternalServerError, err) + return + } + counter = counters.Count60sec() + }() + } + wg.Wait() + send(w, http.StatusOK, countResponse{Count: counter}) +} + +func send(w http.ResponseWriter, statusCode int, body interface{}) { + const jsonContentType = "application/json; charset=utf-8" + + w.Header().Set("Content-Type", jsonContentType) + w.WriteHeader(statusCode) + if err := json.NewEncoder(w).Encode(body); err != nil { + slog.Error("Unable to encode body as JSON", "error", err) + } +} diff --git a/handler_test.go b/handler_test.go new file mode 100644 index 0000000..ddae6ef --- /dev/null +++ b/handler_test.go @@ -0,0 +1,125 @@ +package metrica + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "sync" + "testing" +) + +func TestHandlerCount_Sequential(t *testing.T) { + c := NewAtomicCounter() + mux := Handler(c, nil) + + for i := 0; i < 100; i++ { + req := httptest.NewRequest("GET", "/count", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected status OK; got %v", w.Code) + } + + var gotRes countResponse + if err := json.NewDecoder(w.Body).Decode(&gotRes); err != nil { + t.Errorf("unable to decode body: %v", err) + } + + wantCount := int64(i + 1) + assert(t, wantCount, gotRes.Count) + } +} + +func TestHandlerCount_Concurrent(t *testing.T) { + c := NewAtomicCounter() + mux := Handler(c, nil) + var wg sync.WaitGroup + + for i := 0; i < 1000; i++ { + wg.Add(1) + go func() { + req := httptest.NewRequest("GET", "/count", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected status OK; got %v", w.Code) + } + defer wg.Done() + }() + } + wg.Wait() + + assert(t, int64(1000), c.Value()) +} + +func TestHandlerCountFS_Sequential(t *testing.T) { + f, err := os.Create("test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer func() { + _ = os.Remove(f.Name()) + }() + + fs := NewFileStorage(&sync.Mutex{}, f.Name()) + mux := Handler(nil, fs) + + for i := 0; i < 100; i++ { + req := httptest.NewRequest("GET", "/countfs", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected status OK; got %v", w.Code) + } + + var gotRes countResponse + if err := json.NewDecoder(w.Body).Decode(&gotRes); err != nil { + t.Errorf("unable to decode body: %v", err) + } + + wantCount := int64(i + 1) + assert(t, wantCount, gotRes.Count) + } +} + +func TestHandlerCountFS_Concurrent(t *testing.T) { + f, err := os.Create("test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer func() { + _ = os.Remove(f.Name()) + }() + + fs := NewFileStorage(&sync.Mutex{}, f.Name()) + + mux := Handler(nil, fs) + var wg sync.WaitGroup + + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + req := httptest.NewRequest("GET", "/countfs", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected status OK; got %v", w.Code) + } + defer wg.Done() + }() + } + wg.Wait() + + got, err := fs.Read() + if err != nil { + t.Fatalf("error reading file: %v", err) + } + assert(t, int64(100), got.Count60sec()) +} + +func assert(t *testing.T, want interface{}, got interface{}) { + if want != got { + t.Errorf("expected %v; got %v", want, got) + } +} diff --git a/metrica.go b/metrica.go deleted file mode 100644 index 9a392e7..0000000 --- a/metrica.go +++ /dev/null @@ -1,36 +0,0 @@ -package metrica - -import ( - "encoding/json" - "net/http" - - "log/slog" -) - -type countResponse struct { - Count int -} - -// Handler returns a http.Handler for new endpoints. -func Handler(c *AtomicCounter) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/count", func(w http.ResponseWriter, r *http.Request) { - counter(w, r, c) - }) - return mux -} - -func counter(w http.ResponseWriter, _ *http.Request, c *AtomicCounter) { - c.Inc(1) - send(w, http.StatusOK, countResponse{Count: int(c.Value())}) -} - -func send(w http.ResponseWriter, statusCode int, body interface{}) { - const jsonContentType = "application/json; charset=utf-8" - - w.Header().Set("Content-Type", jsonContentType) - w.WriteHeader(statusCode) - if err := json.NewEncoder(w).Encode(body); err != nil { - slog.Error("Unable to encode body as JSON", "error", err) - } -} diff --git a/metrica_test.go b/metrica_test.go deleted file mode 100644 index f763665..0000000 --- a/metrica_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package metrica - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "sync" - "testing" -) - -func TestHandlerCount_Sequential(t *testing.T) { - c := NewAtomicCounter() - mux := Handler(c) - - for i := 0; i < 100; i++ { - req := httptest.NewRequest("GET", "/count", nil) - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Errorf("expected status OK; got %v", w.Code) - } - - var gotRes countResponse - if err := json.NewDecoder(w.Body).Decode(&gotRes); err != nil { - t.Errorf("unable to decode body: %v", err) - } - - wantCount := i + 1 - assert(t, wantCount, gotRes.Count) - } -} - -func TestHandlerCount_Concurrent(t *testing.T) { - c := NewAtomicCounter() - mux := Handler(c) - var wg sync.WaitGroup - - for i := 0; i < 1000; i++ { - wg.Add(1) - go func() { - req := httptest.NewRequest("GET", "/count", nil) - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Errorf("expected status OK; got %v", w.Code) - } - defer wg.Done() - }() - } - wg.Wait() - - assert(t, int64(1000), c.Value()) -} - -func assert(t *testing.T, want interface{}, got interface{}) { - if want != got { - t.Errorf("expected %v; got %v", want, got) - } -} diff --git a/storage.go b/storage.go index ab91ad9..95277c5 100644 --- a/storage.go +++ b/storage.go @@ -1,3 +1,4 @@ +// Package metrica (storage.go) implements a version of a storage using the filesystem package metrica import ( @@ -5,17 +6,41 @@ import ( "io" "os" "strings" + "sync" "time" ) -// Counter is a struct that holds the datetime of the request -type Counter struct { - Datetime string +// Counter is an alias for time.Time +type Counter time.Time + +// Counters is a list of Counter +type Counters []Counter + +// FileStorage is a struct that implements the Storage interface +type FileStorage struct { + mu *sync.Mutex + fileName string +} + +// NewFileStorage initializes a new FileStorage +func NewFileStorage(mu *sync.Mutex, fileName string) *FileStorage { + return &FileStorage{ + mu: mu, + fileName: fileName, + } } // Write writes the datetime to the file -func Write(f *os.File, c Counter) error { - _, err := f.WriteString(fmt.Sprintf("%v\n", c.Datetime)) +func (m *FileStorage) Write(c Counter) error { + m.mu.Lock() + defer m.mu.Unlock() + + f, err := os.OpenFile(m.fileName, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("error opening file: %v", err) + } + + _, err = f.WriteString(fmt.Sprintf("%v\n", c.Format())) if err != nil { return fmt.Errorf("error writing file: %v", err) } @@ -23,8 +48,11 @@ func Write(f *os.File, c Counter) error { } // Read reads the datetime from the file -func Read(filename string) ([]time.Time, error) { - f, err := os.Open(filename) +func (m *FileStorage) Read() (Counters, error) { + m.mu.Lock() + defer m.mu.Unlock() + + f, err := os.Open(m.fileName) if err != nil { return nil, fmt.Errorf("error opening file: %v", err) } @@ -43,14 +71,39 @@ func Read(filename string) ([]time.Time, error) { datetimes = datetimes[:len(datetimes)-1] } - var parsedDatetime []time.Time + return parseDatetimesToCounters(datetimes) +} + +// Format returns the datetime in RFC3339Nano format +func (c Counter) Format() string { + return time.Time(c).Format(time.RFC3339Nano) +} + +// Count60sec returns the number of datetimes in the last 60 seconds +func (cs Counters) Count60sec() int64 { + currentTime := time.Now() + var count int64 + + lowerLimit := currentTime.Add(-60 * time.Second) + for _, c := range cs { + t := time.Time(c) + + if t.After(lowerLimit) && t.Before(currentTime) { + count++ + } + } + + return count +} + +func parseDatetimesToCounters(datetimes []string) (Counters, error) { + var counters []Counter for _, datetime := range datetimes { t, err := time.Parse(time.RFC3339Nano, datetime) if err != nil { return nil, fmt.Errorf("error parsing datetime: %v", err) } - parsedDatetime = append(parsedDatetime, t) - + counters = append(counters, Counter(t)) } - return parsedDatetime, nil + return counters, nil } diff --git a/storage_test.go b/storage_test.go index a14e500..c623e88 100644 --- a/storage_test.go +++ b/storage_test.go @@ -2,14 +2,13 @@ package metrica import ( "os" + "sync" "testing" "time" ) func TestWriteRead(t *testing.T) { - c := Counter{ - Datetime: time.Now().Format(time.RFC3339Nano), - } + c := Counter(time.Now()) f, err := os.CreateTemp("", "test") if err != nil { @@ -19,18 +18,39 @@ func TestWriteRead(t *testing.T) { _ = os.Remove(f.Name()) }() - if err := Write(f, c); err != nil { + fs := NewFileStorage(&sync.Mutex{}, f.Name()) + + if err := fs.Write(c); err != nil { t.Fatalf("error writing to file: %v", err) } - got, err := Read(f.Name()) + got, err := fs.Read() if err != nil { t.Fatalf("error reading file: %v", err) } if len(got) == 1 { - assert(t, got[0].Format(time.RFC3339Nano), c.Datetime) + assert(t, c.Format(), got[0].Format()) } else { t.Fatalf("expected 1; got %v", len(got)) } } + +func TestCountersCount60sec(t *testing.T) { + currentTime := time.Now() + counters := Counters{ + Counter(currentTime.Add(-time.Second * 10)), + Counter(currentTime.Add(-time.Second * 10)), + Counter(currentTime.Add(-time.Second * 20)), + Counter(currentTime.Add(-time.Second * 30)), + Counter(currentTime.Add(-time.Second * 40)), + Counter(currentTime.Add(-time.Second * 50)), + Counter(currentTime.Add(-time.Second * 60)), + } + want := int64(6) + got := counters.Count60sec() + if got != want { + t.Fatalf("expected %v; got %v", want, got) + } + +} From 58c62f73303c56e9b886954936d1f3d1a6282a7f Mon Sep 17 00:00:00 2001 From: jojo Date: Wed, 11 Oct 2023 19:41:45 -0300 Subject: [PATCH 4/5] :bug: chore: reference --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 78d9280..528c739 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,4 @@ Some resources that were useful to solve it - [Mutex](https://golangbot.com/mutex/) - [Atomic Counters](https://gobyexample.com/atomic-counters) - [Bjorn Rabenstein - Prometheus: Designing and Implementing a Modern Monitoring Solution in Go](https://www.youtube.com/watch?v=1V7eJ0jN8-E) +- [go file mutex](https://echorand.me/posts/go-file-mutex/) From 0ea55247f5627de7ffddbdc5d09deb239260ebbe Mon Sep 17 00:00:00 2001 From: jojo Date: Wed, 11 Oct 2023 19:50:32 -0300 Subject: [PATCH 5/5] :bug: chore: minors --- cmd/metrica/main.go | 2 +- storage.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/metrica/main.go b/cmd/metrica/main.go index 6330521..471d4fe 100644 --- a/cmd/metrica/main.go +++ b/cmd/metrica/main.go @@ -11,7 +11,7 @@ import ( func main() { c := metrica.NewAtomicCounter() - fs := metrica.NewFileStorage(&sync.Mutex{}, "counters.txt") + fs := metrica.NewFileStorage(&sync.Mutex{}, "metrica.txt") mux := metrica.Handler(c, fs) slog.Info("Starting server", "port", 8080) diff --git a/storage.go b/storage.go index 95277c5..dd9a078 100644 --- a/storage.go +++ b/storage.go @@ -13,10 +13,10 @@ import ( // Counter is an alias for time.Time type Counter time.Time -// Counters is a list of Counter +// Counters is a list of Counters type Counters []Counter -// FileStorage is a struct that implements the Storage interface +// FileStorage is a struct that implements the Storage methods type FileStorage struct { mu *sync.Mutex fileName string @@ -30,7 +30,7 @@ func NewFileStorage(mu *sync.Mutex, fileName string) *FileStorage { } } -// Write writes the datetime to the file +// Write save a Counter in the file func (m *FileStorage) Write(c Counter) error { m.mu.Lock() defer m.mu.Unlock() @@ -47,7 +47,7 @@ func (m *FileStorage) Write(c Counter) error { return nil } -// Read reads the datetime from the file +// Read returns a list of Counters func (m *FileStorage) Read() (Counters, error) { m.mu.Lock() defer m.mu.Unlock()