Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor admin server desing #117

Merged
merged 1 commit into from
Aug 1, 2022
Merged

refactor admin server desing #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion cmds/admin_server/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,34 @@ func (l *Log) ToStorageLog() storage.Log {
}
}

func toServerLog(l *storage.Log) Log {
return Log{
JobID: l.JobID,
LogData: l.LogData,
Date: l.Date,
LogLevel: l.LogLevel,
}
}

type Result struct {
Logs []Log `json:"logs"`
Count uint64 `json:"count"`
Page uint `json:"page"`
PageSize uint `json:"page_size"`
}

func toServerResult(r *storage.Result) Result {
var result Result
result.Count = r.Count
result.Page = r.Page
result.PageSize = r.PageSize

for _, log := range r.Logs {
result.Logs = append(result.Logs, toServerLog(&log))
}
return result
}

type RouteHandler struct {
storage storage.Storage
log logger.Logger
Expand Down Expand Up @@ -125,7 +153,7 @@ func (r *RouteHandler) getLogs(c *gin.Context) {
return
}

c.JSON(http.StatusOK, result)
c.JSON(http.StatusOK, toServerResult(result))
}

func initRouter(ctx xcontext.Context, rh RouteHandler) *gin.Engine {
Expand Down
42 changes: 37 additions & 5 deletions cmds/admin_server/storage/mongo/mongo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strings"
"time"

"github.com/linuxboot/contest/cmds/admin_server/storage"
"github.com/linuxboot/contest/pkg/xcontext"
Expand Down Expand Up @@ -63,7 +64,7 @@ func toMongoQuery(query storage.Query) bson.D {

if query.Text != nil {
q = append(q, bson.E{
Key: "logdata",
Key: "log_data",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note that this is a breaking change, youre changing a db field and consumers may already be using the last version of the field names in their mongo deployments

i will approve this this time, as the project state is technically "not launched", but you should consider a migration mechanism for these updates

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you might want to consider a db migration/versioning scheme later on to address this

Value: bson.M{
"$regex": primitive.Regex{Pattern: *query.Text, Options: "ig"},
},
Expand Down Expand Up @@ -94,7 +95,7 @@ func toMongoQuery(query storage.Query) bson.D {
levels := strings.Split(*query.LogLevel, ",")
q = append(q,
bson.E{
Key: "loglevel",
Key: "log_level",
Value: bson.M{
"$in": levels,
},
Expand All @@ -106,7 +107,8 @@ func toMongoQuery(query storage.Query) bson.D {
}

func (s *MongoStorage) StoreLog(ctx xcontext.Context, log storage.Log) error {
_, err := s.collection.InsertOne(ctx, log)
mongoLog := toMongoLog(&log)
_, err := s.collection.InsertOne(ctx, mongoLog)
if err != nil {
// for better debugging
ctx.Errorf("Error while inserting into the db: %v", err)
Expand Down Expand Up @@ -134,15 +136,20 @@ func (s *MongoStorage) GetLogs(ctx xcontext.Context, query storage.Query) (*stor
return nil, storage.ErrQuery
}

var logs []storage.Log
var logs []Log
err = cur.All(ctx, &logs)
if err != nil {
ctx.Errorf("Error while reading query result from db: %v", err)
return nil, storage.ErrQuery
}
// convert to storage logs
storageLogs := make([]storage.Log, 0, len(logs))
for _, log := range logs {
storageLogs = append(storageLogs, log.toStorageLog())
}

return &storage.Result{
Logs: logs,
Logs: storageLogs,
Count: uint64(count),
Page: query.Page,
PageSize: query.PageSize,
Expand All @@ -152,3 +159,28 @@ func (s *MongoStorage) GetLogs(ctx xcontext.Context, query storage.Query) (*stor
func (s *MongoStorage) Close(ctx xcontext.Context) error {
return s.dbClient.Disconnect(ctx)
}

type Log struct {
JobID uint64 `bson:"job_id"`
LogData string `bson:"log_data"`
Date time.Time `bson:"date"`
LogLevel string `bson:"log_level"`
}

func (l *Log) toStorageLog() storage.Log {
mimir-d marked this conversation as resolved.
Show resolved Hide resolved
return storage.Log{
JobID: l.JobID,
LogData: l.LogData,
Date: l.Date,
LogLevel: l.LogLevel,
}
}

func toMongoLog(l *storage.Log) Log {
return Log{
JobID: l.JobID,
LogData: l.LogData,
Date: l.Date,
LogLevel: l.LogLevel,
}
}
16 changes: 8 additions & 8 deletions cmds/admin_server/storage/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ type Storage interface {

// Log defines the basic log info pushed by the server
type Log struct {
JobID uint64 `json:"jobID"`
LogData string `json:"logData"`
Date time.Time `json:"date"`
LogLevel string `json:"logLevel"`
JobID uint64
LogData string
Date time.Time
LogLevel string
}

// Query defines the different options to filter with
Expand All @@ -46,8 +46,8 @@ type Query struct {

//Result defines the expected result returned from the db
type Result struct {
Logs []Log `json:"logs"`
Count uint64 `json:"count"`
Page uint `json:"page"`
PageSize uint `json:"pageSize"`
Logs []Log
Count uint64
Page uint
PageSize uint
}
12 changes: 6 additions & 6 deletions tests/integ/admin_server/getlogs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func generateRandomLogs(db *mongo.Client, count int) error {
level := logLevels[rand.Intn(len(logLevels))]
data := randomText()

logs = append(logs, storage.Log{
logs = append(logs, mongoStorage.Log{
JobID: uint64(jobID),
LogLevel: level,
LogData: data,
Expand Down Expand Up @@ -231,7 +231,7 @@ func TestLogQuery(t *testing.T) {
PageSize: &pageSize,
},
dbQuery: bson.M{
"logdata": bson.M{"$regex": primitive.Regex{Pattern: "a", Options: "ig"}},
"log_data": bson.M{"$regex": primitive.Regex{Pattern: "a", Options: "ig"}},
},
},
{
Expand All @@ -243,7 +243,7 @@ func TestLogQuery(t *testing.T) {
PageSize: &pageSize,
},
dbQuery: bson.M{
"loglevel": "info",
"log_level": "info",
},
},
{
Expand All @@ -255,7 +255,7 @@ func TestLogQuery(t *testing.T) {
PageSize: &pageSize,
},
dbQuery: bson.M{
"loglevel": bson.M{"$in": []string{"info", "debug"}},
"log_level": bson.M{"$in": []string{"info", "debug"}},
},
},
{
Expand Down Expand Up @@ -313,12 +313,12 @@ func TestLogQuery(t *testing.T) {
PageSize: &pageSize,
},
dbQuery: bson.M{
"logdata": bson.M{"$regex": primitive.Regex{Pattern: "a", Options: "ig"}},
"log_data": bson.M{"$regex": primitive.Regex{Pattern: "a", Options: "ig"}},
"date": bson.M{
"$lte": primitive.NewDateTimeFromTime(endDate),
"$gte": primitive.NewDateTimeFromTime(startDate),
},
"loglevel": "info",
"log_level": "info",
},
},
}
Expand Down
7 changes: 3 additions & 4 deletions tests/integ/admin_server/logendpoint_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"time"

"github.com/linuxboot/contest/cmds/admin_server/server"
"github.com/linuxboot/contest/cmds/admin_server/storage"
mongoStorage "github.com/linuxboot/contest/cmds/admin_server/storage/mongo"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
Expand Down Expand Up @@ -58,17 +57,17 @@ func submitLog(addr string, log server.Log) error {
return err
}

func getAllLogs(t *testing.T, db *mongo.Client) []storage.Log {
func getAllLogs(t *testing.T, db *mongo.Client) []mongoStorage.Log {
cur, err := db.Database(mongoStorage.DefaultDB).
Collection(mongoStorage.DefaultCollection).
Find(context.Background(), bson.D{{}})
if err != nil {
t.Fatal(err)
}

var dbLogs []storage.Log
var dbLogs []mongoStorage.Log
for cur.Next(context.Background()) {
var log storage.Log
var log mongoStorage.Log
err := cur.Decode(&log)
if err != nil {
t.Fatal(err)
Expand Down