Skip to content

Commit

Permalink
Metrics (#66)
Browse files Browse the repository at this point in the history 
* Added counter for inserted and read record count

fixes #64

* Small adjustments to docker config files

* Added sql query end counter

fixes #65

* add to metric after error check
  • Loading branch information
@kristinapathak
kristinapathak authored May 9, 2019
1 parent bc5dbec commit 71f0ece
Show file tree
Hide file tree
Showing 10 changed files with 76 additions and 29 deletions.
9 changes: 6 additions & 3 deletions db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,7 @@ func (db *Connection) GetRecords(deviceID string, limit int) ([]Record, error) {
db.measures.SQLQueryFailureCount.With(typeLabel, readType).Add(1.0)
return []Record{}, emperror.WrapWith(err, "Getting records from database failed", "device id", deviceID)
}
db.measures.SQLReadRecords.Add(float64(len(deviceInfo)))
db.measures.SQLQuerySuccessCount.With(typeLabel, readType).Add(1.0)
return deviceInfo, nil
}
Expand All @@ -282,6 +283,7 @@ func (db *Connection) GetRecordsOfType(deviceID string, limit int, eventType Eve
db.measures.SQLQueryFailureCount.With(typeLabel, readType).Add(1.0)
return []Record{}, emperror.WrapWith(err, "Getting records from database failed", "device id", deviceID)
}
db.measures.SQLReadRecords.Add(float64(len(deviceInfo)))
db.measures.SQLQuerySuccessCount.With(typeLabel, readType).Add(1.0)
return deviceInfo, nil
}
Expand All @@ -300,7 +302,7 @@ func (db *Connection) GetBlacklist() (list []blacklist.BlackListedItem, err erro
// PruneRecords removes records past their deathdate.
func (db *Connection) PruneRecords(t int64) error {
rowsAffected, err := db.deleter.delete(&Record{}, db.pruneLimit, "death_date < ?", t)
db.measures.SQLDeletedRows.Add(float64(rowsAffected))
db.measures.SQLDeletedRecords.Add(float64(rowsAffected))
if err != nil {
db.measures.SQLQueryFailureCount.With(typeLabel, deleteType).Add(1.0)
return emperror.WrapWith(err, "Prune records failed", "time", t)
Expand All @@ -311,7 +313,8 @@ func (db *Connection) PruneRecords(t int64) error {

// InsertEvent adds a record to the table.
func (db *Connection) InsertRecords(records ...Record) error {
err := db.mutliInsert.insert(records)
rowsAffected, err := db.mutliInsert.insert(records)
db.measures.SQLInsertedRecords.Add(float64(rowsAffected))
if err != nil {
db.measures.SQLQueryFailureCount.With(typeLabel, insertType).Add(1.0)
return emperror.Wrap(err, "Inserting records failed")
Expand Down Expand Up @@ -361,7 +364,7 @@ func doEvery(d time.Duration, f func()) chan struct{} {
// RemoveAll removes everything in the events table. Used for testing.
func (db *Connection) RemoveAll() error {
rowsAffected, err := db.deleter.delete(&Record{}, 0)
db.measures.SQLDeletedRows.Add(float64(rowsAffected))
db.measures.SQLDeletedRecords.Add(float64(rowsAffected))
if err != nil {
db.measures.SQLQueryFailureCount.With(typeLabel, deleteType).Add(1.0)
return emperror.Wrap(err, "Removing all records from database failed")
Expand Down
16 changes: 10 additions & 6 deletions db/db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,13 @@ func TestGetRecordsOfType(t *testing.T) {
}
p.Assert(t, SQLQuerySuccessCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryFailureCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLReadRecordsCounter)(xmetricstest.Value(0.0))

records, err := dbConnection.GetRecordsOfType(tc.deviceID, 5, tc.eventType)
mockObj.AssertExpectations(t)
p.Assert(t, SQLQuerySuccessCounter, typeLabel, readType)(xmetricstest.Value(tc.expectedSuccessMetric))
p.Assert(t, SQLQueryFailureCounter, typeLabel, readType)(xmetricstest.Value(tc.expectedFailureMetric))
p.Assert(t, SQLReadRecordsCounter)(xmetricstest.Value(float64(len(tc.expectedRecords))))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand All @@ -172,7 +174,7 @@ func TestGetRecordsOfType(t *testing.T) {
}
}

func TestUpdateHistory(t *testing.T) {
func TestPruneRecords(t *testing.T) {
pruneTestErr := errors.New("test prune history error")
tests := []struct {
description string
Expand Down Expand Up @@ -211,13 +213,13 @@ func TestUpdateHistory(t *testing.T) {
mockObj.On("delete", mock.Anything, 3, mock.Anything).Return(6, tc.pruneErr).Once()
p.Assert(t, SQLQuerySuccessCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryFailureCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLDeletedRowsCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLDeletedRecordsCounter)(xmetricstest.Value(0.0))

err := dbConnection.PruneRecords(tc.time.Unix())
mockObj.AssertExpectations(t)
p.Assert(t, SQLQuerySuccessCounter, typeLabel, deleteType)(xmetricstest.Value(tc.expectedSuccessMetric))
p.Assert(t, SQLQueryFailureCounter, typeLabel, deleteType)(xmetricstest.Value(tc.expectedFailureMetric))
p.Assert(t, SQLDeletedRowsCounter)(xmetricstest.Value(6.0))
p.Assert(t, SQLDeletedRecordsCounter)(xmetricstest.Value(6.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down Expand Up @@ -269,15 +271,17 @@ func TestMultiInsertEvent(t *testing.T) {
mutliInsert: mockObj,
}
if tc.expectedCalls > 0 {
mockObj.On("insert", mock.Anything).Return(tc.createErr).Times(tc.expectedCalls)
mockObj.On("insert", mock.Anything).Return(3, tc.createErr).Times(tc.expectedCalls)
}
p.Assert(t, SQLQuerySuccessCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryFailureCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLInsertedRecordsCounter)(xmetricstest.Value(0.0))

err := dbConnection.InsertRecords(tc.records...)
mockObj.AssertExpectations(t)
p.Assert(t, SQLQuerySuccessCounter, typeLabel, insertType)(xmetricstest.Value(tc.expectedSuccessMetric))
p.Assert(t, SQLQueryFailureCounter, typeLabel, insertType)(xmetricstest.Value(tc.expectedFailureMetric))
p.Assert(t, SQLInsertedRecordsCounter)(xmetricstest.Value(3.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down Expand Up @@ -318,13 +322,13 @@ func TestRemoveAll(t *testing.T) {
mockObj.On("delete", mock.Anything, 0, mock.Anything).Return(6, tc.expectedErr).Once()
p.Assert(t, SQLQuerySuccessCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryFailureCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLDeletedRowsCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLDeletedRecordsCounter)(xmetricstest.Value(0.0))

err := dbConnection.RemoveAll()
mockObj.AssertExpectations(t)
p.Assert(t, SQLQuerySuccessCounter, typeLabel, deleteType)(xmetricstest.Value(tc.expectedSuccessMetric))
p.Assert(t, SQLQueryFailureCounter, typeLabel, deleteType)(xmetricstest.Value(tc.expectedFailureMetric))
p.Assert(t, SQLDeletedRowsCounter)(xmetricstest.Value(6.0))
p.Assert(t, SQLDeletedRecordsCounter)(xmetricstest.Value(6.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down
16 changes: 9 additions & 7 deletions db/executer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ import (

"database/sql"
"fmt"
"github.com/Comcast/codex/blacklist"
"strings"

"github.com/Comcast/codex/blacklist"

"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/postgres"
)
Expand All @@ -37,7 +38,7 @@ type (
findBlacklist(out *[]blacklist.BlackListedItem) error
}
multiinserter interface {
insert(records []Record) error
insert(records []Record) (int64, error)
}
deleter interface {
delete(value *Record, limit int, where ...interface{}) (int64, error)
Expand Down Expand Up @@ -67,9 +68,9 @@ func (b *dbDecorator) findBlacklist(out *[]blacklist.BlackListedItem) error {
return db.Error
}

func (b *dbDecorator) insert(records []Record) error {
func (b *dbDecorator) insert(records []Record) (int64, error) {
if len(records) == 0 {
return errNoEvents
return 0, errNoEvents
}
mainScope := b.DB.NewScope(records[0])
mainFields := mainScope.Fields()
Expand Down Expand Up @@ -108,10 +109,11 @@ func (b *dbDecorator) insert(records []Record) error {
strings.Join(placeholdersArr, ", "),
))

if _, err := mainScope.SQLDB().Exec(mainScope.SQL, mainScope.SQLVars...); err != nil {
return err
result, err := mainScope.SQLDB().Exec(mainScope.SQL, mainScope.SQLVars...)
if err != nil {
return 0, err
}
return nil
return result.RowsAffected()
}

func (b *dbDecorator) delete(value *Record, limit int, where ...interface{}) (int64, error) {
Expand Down
33 changes: 29 additions & 4 deletions db/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,10 @@ const (
SQLQuerySuccessCounter = "sql_query_success_count"
SQLQueryFailureCounter = "sql_query_failure_count"
SQLQueryRetryCounter = "sql_query_retry_count"
SQLDeletedRowsCounter = "sql_deleted_rows_count"
SQLQueryEndCounter = "sql_query_end_counter"
SQLInsertedRecordsCounter = "sql_inserted_rows_count"
SQLReadRecordsCounter = "sql_read_rows_count"
SQLDeletedRecordsCounter = "sql_deleted_rows_count"
)

//Metrics returns the Metrics relevant to this package
Expand Down Expand Up @@ -110,7 +113,23 @@ func Metrics() []xmetrics.Metric {
LabelNames: []string{typeLabel},
},
{
Name: SQLDeletedRowsCounter,
Name: SQLQueryEndCounter,
Type: "counter",
Help: "the total number of SQL queries that are done, no more retrying",
LabelNames: []string{typeLabel},
},
{
Name: SQLInsertedRecordsCounter,
Type: "counter",
Help: "The total number of rows inserted",
},
{
Name: SQLReadRecordsCounter,
Type: "counter",
Help: "The total number of rows read",
},
{
Name: SQLDeletedRecordsCounter,
Type: "counter",
Help: "The total number of rows deleted",
},
Expand All @@ -130,7 +149,10 @@ type Measures struct {
SQLQuerySuccessCount metrics.Counter
SQLQueryFailureCount metrics.Counter
SQLQueryRetryCount metrics.Counter
SQLDeletedRows metrics.Counter
SQLQueryEndCount metrics.Counter
SQLInsertedRecords metrics.Counter
SQLReadRecords metrics.Counter
SQLDeletedRecords metrics.Counter
}

func NewMeasures(p provider.Provider) Measures {
Expand All @@ -147,6 +169,9 @@ func NewMeasures(p provider.Provider) Measures {
SQLQuerySuccessCount: p.NewCounter(SQLQuerySuccessCounter),
SQLQueryFailureCount: p.NewCounter(SQLQueryFailureCounter),
SQLQueryRetryCount: p.NewCounter(SQLQueryRetryCounter),
SQLDeletedRows: p.NewCounter(SQLDeletedRowsCounter),
SQLQueryEndCount: p.NewCounter(SQLQueryEndCounter),
SQLInsertedRecords: p.NewCounter(SQLInsertedRecordsCounter),
SQLReadRecords: p.NewCounter(SQLReadRecordsCounter),
SQLDeletedRecords: p.NewCounter(SQLDeletedRecordsCounter),
}
}
4 changes: 2 additions & 2 deletions db/mocks_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ type mockMultiInsert struct {
mock.Mock
}

func (c *mockMultiInsert) insert(records []Record) error {
func (c *mockMultiInsert) insert(records []Record) (int64, error) {
args := c.Called(records)
return args.Error(0)
return int64(args.Int(0)), args.Error(1)
}

type mockDeleter struct {
Expand Down
5 changes: 5 additions & 0 deletions db/retry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ func (ri RetryInsertService) InsertRecords(records ...Record) error {
}
}

ri.config.measures.SQLQueryEndCount.With(typeLabel, insertType).Add(1.0)
return err
}

Expand Down Expand Up @@ -149,6 +150,7 @@ func (ru RetryUpdateService) PruneRecords(t int64) error {
}
}

ru.config.measures.SQLQueryEndCount.With(typeLabel, deleteType).Add(1.0)
return err
}

Expand Down Expand Up @@ -188,6 +190,7 @@ func (ltg RetryListGService) GetBlacklist() (list []blacklist.BlackListedItem, e
}
}

ltg.config.measures.SQLQueryEndCount.With(typeLabel, listReadType).Add(1.0)
return
}

Expand Down Expand Up @@ -237,6 +240,7 @@ func (rtg RetryRGService) GetRecords(deviceID string, limit int) ([]Record, erro
}
}

rtg.config.measures.SQLQueryEndCount.With(typeLabel, readType).Add(1.0)
return record, err
}

Expand All @@ -261,6 +265,7 @@ func (rtg RetryRGService) GetRecordsOfType(deviceID string, limit int, eventType
}
}

rtg.config.measures.SQLQueryEndCount.With(typeLabel, readType).Add(1.0)
return record, err
}

Expand Down
10 changes: 10 additions & 0 deletions db/retry_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,11 @@ func TestRetryInsertRecords(t *testing.T) {
},
}
p.Assert(t, SQLQueryRetryCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryEndCounter)(xmetricstest.Value(0.0))
err := retryInsertService.InsertRecords(Record{})
mockObj.AssertExpectations(t)
p.Assert(t, SQLQueryRetryCounter, typeLabel, insertType)(xmetricstest.Value(tc.expectedRetryMetric))
p.Assert(t, SQLQueryEndCounter, typeLabel, insertType)(xmetricstest.Value(1.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down Expand Up @@ -193,9 +195,11 @@ func TestRetryPruneRecords(t *testing.T) {
},
}
p.Assert(t, SQLQueryRetryCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryEndCounter)(xmetricstest.Value(0.0))
err := retryInsertService.PruneRecords(time.Now().Unix())
mockObj.AssertExpectations(t)
p.Assert(t, SQLQueryRetryCounter, typeLabel, deleteType)(xmetricstest.Value(tc.expectedRetryMetric))
p.Assert(t, SQLQueryEndCounter, typeLabel, deleteType)(xmetricstest.Value(1.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down Expand Up @@ -290,9 +294,11 @@ func TestRetryGetBlacklist(t *testing.T) {
},
}
p.Assert(t, SQLQueryRetryCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryEndCounter)(xmetricstest.Value(0.0))
_, err := retryListGService.GetBlacklist()
mockObj.AssertExpectations(t)
p.Assert(t, SQLQueryRetryCounter, typeLabel, listReadType)(xmetricstest.Value(tc.expectedRetryMetric))
p.Assert(t, SQLQueryEndCounter, typeLabel, listReadType)(xmetricstest.Value(1.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down Expand Up @@ -371,9 +377,11 @@ func TestRetryGetRecords(t *testing.T) {
},
}
p.Assert(t, SQLQueryRetryCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryEndCounter)(xmetricstest.Value(0.0))
_, err := retryRGService.GetRecords("", 5)
mockObj.AssertExpectations(t)
p.Assert(t, SQLQueryRetryCounter, typeLabel, readType)(xmetricstest.Value(tc.expectedRetryMetric))
p.Assert(t, SQLQueryEndCounter, typeLabel, readType)(xmetricstest.Value(1.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down Expand Up @@ -452,9 +460,11 @@ func TestRetryGetRecordsOfType(t *testing.T) {
},
}
p.Assert(t, SQLQueryRetryCounter)(xmetricstest.Value(0.0))
p.Assert(t, SQLQueryEndCounter)(xmetricstest.Value(0.0))
_, err := retryRGService.GetRecordsOfType("", 5, 0)
mockObj.AssertExpectations(t)
p.Assert(t, SQLQueryRetryCounter, typeLabel, readType)(xmetricstest.Value(tc.expectedRetryMetric))
p.Assert(t, SQLQueryEndCounter, typeLabel, readType)(xmetricstest.Value(1.0))
if tc.expectedErr == nil || err == nil {
assert.Equal(tc.expectedErr, err)
} else {
Expand Down
6 changes: 3 additions & 3 deletions deploy/docker-compose/docFiles/fenrir.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ log:
level: "DEBUG"
json: true

pruneInterval: 10s
pruneRetries: 0
retryInterval: 0s
pruneInterval: 5m
pruneRetries: 2
retryInterval: 30s
db:
server: "db:26257"
username: "roachadmin"
Expand Down
4 changes: 2 additions & 2 deletions deploy/docker-compose/docFiles/gungnir.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ health:
port: ":7001"
endpoint: "/health"

getLimit: 2
getLimit: 200
getRetries: 0
retryInterval: 0s
blacklistInterval: 1m
Expand All @@ -46,4 +46,4 @@ cipher:
keys:
privateKey: "/etc/gungnir/private.pem"
- type: none
kid: none
kid: none
2 changes: 0 additions & 2 deletions deploy/docker-compose/docFiles/svalinn.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ maxBatchWaitTime: 10ms
metadataMaxSize: 1000
payloadMaxSize: 1000
insertRetries: 0
pruneRetries: 0
pruneInterval: 0s
retryInterval: 10s
defaultTTL: 1h
blacklistInterval: 1m
Expand Down

0 comments on commit 71f0ece

Please sign in to comment.