Update metrics 2

.travis.yml

@@ -19,6 +19,8 @@ branches:
   only:
     - master
 
-script:
+install:
   - go get -v github.com/Masterminds/glide
+
+script:
   - make build style codecov

CHANGELOG.md

@@ -22,11 +22,11 @@ fixed build upload
 
 
 
-## [0.1.3] - 2019-02-27
+## [v0.1.3] - 2019-02-27
 ### Changed
 - Fix for [issue 126](https://github.com/Comcast/caduceus/issues/126)
 
-## [0.1.2] - 2019-02-21
+## [v0.1.2] - 2019-02-21
 ### Added
 - Fix for delivering events as json or msgpack based events [issue 113](https://github.com/Comcast/caduceus/issues/113)
 
@@ -56,8 +56,10 @@ fixed build upload
 ### Added
 - Initial creation
 
-[Unreleased]: https://github.com/Comcast/caduceus/compare/0.1.3...HEAD
-[0.1.3]: https://github.com/Comcast/caduceus/compare/0.1.2...0.1.3
-[0.1.2]: https://github.com/Comcast/caduceus/compare/0.1.1...0.1.2
+[Unreleased]: https://github.com/Comcast/caduceus/compare/v0.1.5...HEAD
+[v0.1.5]: https://github.com/Comcast/caduceus/compare/v0.1.4...v0.1.5
+[v0.1.4]: https://github.com/Comcast/caduceus/compare/v0.1.3...v0.1.4
+[v0.1.3]: https://github.com/Comcast/caduceus/compare/v0.1.2...v0.1.3
+[v0.1.2]: https://github.com/Comcast/caduceus/compare/0.1.1...v0.1.2
 [0.1.1]: https://github.com/Comcast/caduceus/compare/0.0.1...0.1.1
-[0.0.1]: https://github.com/Comcast/caduceus/compare/0.0.0...0.0.1
+[0.0.1]: https://github.com/Comcast/caduceus/compare/v0.0.0...0.0.1

Dockerfile

@@ -1,24 +1,30 @@
 FROM golang:alpine as builder
 MAINTAINER Jack Murdock <[email protected]>
 
-# build the binary
-WORKDIR /go/src
+ENV build_gopath /go/src/caduceus
+
+# fetch needed software
 RUN apk add --update --repository https://dl-3.alpinelinux.org/alpine/edge/testing/ git curl
 RUN curl https://glide.sh/get | sh
-COPY src/ /go/src/
 
+COPY src/ ${build_gopath}/src
+
+WORKDIR ${build_gopath}/src
+ENV GOPATH ${build_gopath} 
+
+# fetch golang dependencies
 RUN glide -q install --strip-vendor
-RUN go build -o caduceus_linux_amd64 caduceus
 
-EXPOSE 6000 6001 6002
-RUN mkdir -p /etc/caduceus
-VOLUME /etc/caduceus
+# build the binary
+WORKDIR ${build_gopath}/src/caduceus
+RUN go build -o caduceus_linux_amd64 caduceus
 
-# the actual image
+# prep the actual image
 FROM alpine:latest
 RUN apk --no-cache add ca-certificates
+EXPOSE 6100 6101 6102
 RUN mkdir -p /etc/caduceus
 VOLUME /etc/caduceus
 WORKDIR /root/
-COPY --from=builder /go/src/caduceus_linux_amd64 .
+COPY --from=builder /go/src/caduceus/src/caduceus/caduceus_linux_amd64 .
 ENTRYPOINT ["./caduceus_linux_amd64"]

Dockerfile.local

@@ -1,21 +1,23 @@
 FROM golang:alpine as builder
 MAINTAINER Jack Murdock <[email protected]>
 
-# build the binary
-WORKDIR /go/src
-COPY src/ /go/src/
+ENV build_gopath /go/src/caduceus
 
-RUN go build -o caduceus_linux_amd64 caduceus
+COPY src/ ${build_gopath}/src
 
-EXPOSE 6000 6001 6002
-RUN mkdir -p /etc/caduceus
-VOLUME /etc/caduceus
+WORKDIR ${build_gopath}/src
+ENV GOPATH ${build_gopath}
+
+# build the binary
+WORKDIR ${build_gopath}/src/caduceus
+RUN go build -o caduceus_linux_amd64 caduceus
 
-# the actual image
+# prep the actual image
 FROM alpine:latest
 RUN apk --no-cache add ca-certificates
+EXPOSE 6100 6101 6102
 RUN mkdir -p /etc/caduceus
 VOLUME /etc/caduceus
 WORKDIR /root/
-COPY --from=builder /go/src/caduceus_linux_amd64 .
+COPY --from=builder /go/src/caduceus/src/caduceus/caduceus_linux_amd64 .
 ENTRYPOINT ["./caduceus_linux_amd64"]

Makefile

@@ -63,7 +63,6 @@ docker:
 # build docker without running modules
 .PHONY: local-docker
 local-docker:
-	GOOS=linux  GOARCH=amd64 go build -o $(APP)_linux_amd64 $(APP)
 	docker build -f ./deploy/Dockerfile.local -t $(APP):local .
 
 .PHONY: style

docs/getting_started.md

@@ -0,0 +1,8 @@
+---
+title: Getting started
+sort_rank: 1
+---
+
+# Getting started
+
+This is a sample to get our documentation system started.  More will come later.

src/caduceus/metrics.go

@@ -5,17 +5,24 @@ import (
 )
 
 const (
-	ErrorRequestBodyCounter       = "error_request_body_count"
-	EmptyRequestBodyCounter       = "empty_request_body_count"
-	DeliveryCounter               = "delivery_count"
-	DeliveryRetryCounter          = "delivery_retry_count"
-	SlowConsumerDroppedMsgCounter = "slow_consumer_dropped_message_count"
-	SlowConsumerCounter           = "slow_consumer_cut_off_count"
-	IncomingQueueDepth            = "incoming_queue_depth"
-	IncomingContentTypeCounter    = "incoming_content_type_count"
-	IncomingEventTypeCounter      = "incoming_event_type_count"
-	DropsDueToInvalidPayload      = "drops_due_to_invalid_payload"
-	OutgoingQueueDepth            = "outgoing_queue_depths"
+	ErrorRequestBodyCounter         = "error_request_body_count"
+	EmptyRequestBodyCounter         = "empty_request_body_count"
+	DeliveryCounter                 = "delivery_count"
+	DeliveryRetryCounter            = "delivery_retry_count"
+	DeliveryRetryMaxGauge           = "delivery_retry_max"
+	SlowConsumerDroppedMsgCounter   = "slow_consumer_dropped_message_count"
+	SlowConsumerCounter             = "slow_consumer_cut_off_count"
+	IncomingQueueDepth              = "incoming_queue_depth"
+	IncomingContentTypeCounter      = "incoming_content_type_count"
+	IncomingEventTypeCounter        = "incoming_event_type_count"
+	DropsDueToInvalidPayload        = "drops_due_to_invalid_payload"
+	OutgoingQueueDepth              = "outgoing_queue_depths"
+	DropsDueToPanic                 = "drops_due_to_panic"
+	ConsumerRenewalTimeGauge        = "consumer_renewal_time"
+	ConsumerDeliverUntilGauge       = "consumer_deliver_until"
+	ConsumerDropUntilGauge          = "consumer_drop_until"
+	ConsumerDeliveryWorkersGauge    = "consumer_delivery_workers"
+	ConsumerMaxDeliveryWorkersGauge = "consumer_delivery_workers_max"
 )
 
 func Metrics() []xmetrics.Metric {
@@ -52,6 +59,12 @@ func Metrics() []xmetrics.Metric {
 			Type:       "counter",
 			LabelNames: []string{"url", "event"},
 		},
+		{
+			Name:       DeliveryRetryMaxGauge,
+			Help:       "Maximum number of delivery retries attempted",
+			Type:       "gauge",
+			LabelNames: []string{"url"},
+		},
 		{
 			Name:       DeliveryCounter,
 			Help:       "Count of delivered messages to a url with a status code",
@@ -82,18 +95,61 @@ func Metrics() []xmetrics.Metric {
 			Type:       "counter",
 			LabelNames: []string{"event"},
 		},
+		{
+			Name: DropsDueToPanic,
+			Help: "The outgoing message delivery pipeline panicked.",
+			Type: "counter",
+		},
+		{
+			Name:       ConsumerRenewalTimeGauge,
+			Help:       "Time when the consumer data was updated.",
+			Type:       "gauge",
+			LabelNames: []string{"url"},
+		},
+		{
+			Name:       ConsumerDeliverUntilGauge,
+			Help:       "Time when the consumer's registration expires and events will be dropped.",
+			Type:       "gauge",
+			LabelNames: []string{"url"},
+		},
+		{
+			Name:       ConsumerDropUntilGauge,
+			Help:       "The time after which events going to a customer will be delivered.",
+			Type:       "gauge",
+			LabelNames: []string{"url"},
+		},
+		{
+			Name:       ConsumerDeliveryWorkersGauge,
+			Help:       "The number of active delivery workers for a particular customer.",
+			Type:       "gauge",
+			LabelNames: []string{"url"},
+		},
+		{
+			Name:       ConsumerMaxDeliveryWorkersGauge,
+			Help:       "The maximum number of delivery workers available for a particular customer.",
+			Type:       "gauge",
+			LabelNames: []string{"url"},
+		},
 	}
 }
 
 func CreateOutbounderMetrics(m CaduceusMetricsRegistry, c *CaduceusOutboundSender) {
 	c.deliveryCounter = m.NewCounter(DeliveryCounter)
 	c.deliveryRetryCounter = m.NewCounter(DeliveryRetryCounter)
+	c.deliveryRetryMaxGauge = m.NewGauge(DeliveryRetryMaxGauge)
 	c.cutOffCounter = m.NewCounter(SlowConsumerCounter).With("url", c.id)
 	c.droppedQueueFullCounter = m.NewCounter(SlowConsumerDroppedMsgCounter).With("url", c.id, "reason", "queue_full")
 	c.droppedExpiredCounter = m.NewCounter(SlowConsumerDroppedMsgCounter).With("url", c.id, "reason", "expired")
+	c.droppedExpiredBeforeQueueCounter = m.NewCounter(SlowConsumerDroppedMsgCounter).With("url", c.id, "reason", "expired_before_queueing")
 	c.droppedCutoffCounter = m.NewCounter(SlowConsumerDroppedMsgCounter).With("url", c.id, "reason", "cut_off")
 	c.droppedInvalidConfig = m.NewCounter(SlowConsumerDroppedMsgCounter).With("url", c.id, "reason", "invalid_config")
 	c.droppedNetworkErrCounter = m.NewCounter(SlowConsumerDroppedMsgCounter).With("url", c.id, "reason", "network_err")
+	c.droppedPanic = m.NewCounter(DropsDueToPanic).With("url", c.id)
 	c.queueDepthGauge = m.NewGauge(OutgoingQueueDepth).With("url", c.id)
 	c.contentTypeCounter = m.NewCounter(IncomingContentTypeCounter)
+	c.renewalTimeGauge = m.NewGauge(ConsumerRenewalTimeGauge).With("url", c.id)
+	c.deliverUntilGauge = m.NewGauge(ConsumerDeliverUntilGauge).With("url", c.id)
+	c.dropUntilGauge = m.NewGauge(ConsumerDropUntilGauge).With("url", c.id)
+	c.currentWorkersGauge = m.NewGauge(ConsumerDeliveryWorkersGauge).With("url", c.id)
+	c.maxWorkersGauge = m.NewGauge(ConsumerMaxDeliveryWorkersGauge).With("url", c.id)
 }

src/caduceus/mocks_test.go

@@ -19,6 +19,7 @@ package main
 import (
 	"net/http"
 	"time"
+	"unicode/utf8"
 
 	"github.com/Comcast/webpa-common/health"
 	"github.com/Comcast/webpa-common/webhook"
@@ -99,6 +100,11 @@ func (m *mockCounter) Add(delta float64) {
 }
 
 func (m *mockCounter) With(labelValues ...string) metrics.Counter {
+	for _, v := range labelValues {
+		if !utf8.ValidString(v) {
+			panic("not UTF-8")
+		}
+	}
 	args := m.Called(labelValues)
 	return args.Get(0).(metrics.Counter)
 }
@@ -113,10 +119,17 @@ func (m *mockGauge) Add(delta float64) {
 }
 
 func (m *mockGauge) Set(value float64) {
-	m.Called(value)
+	// We're setting time values & the ROI seems pretty low with this level
+	// of validation...
+	//m.Called(value)
 }
 
 func (m *mockGauge) With(labelValues ...string) metrics.Gauge {
+	for _, v := range labelValues {
+		if !utf8.ValidString(v) {
+			panic("not UTF-8")
+		}
+	}
 	args := m.Called(labelValues)
 	return args.Get(0).(metrics.Gauge)
 }