From 64bdc6ee2a01922335382fc5b344040df4eecd59 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Fri, 23 Jun 2023 17:05:03 +0200 Subject: [PATCH 01/37] Add latency package (#14) * Add latency package and its tests. * Rename Measurements -> Packets in result structs. * Add docstrings. * Move timeout ctx into the sendLoop func and call cancel(). * Improve error handling in latency.go Stop memoryless.Run() by canceling the context. * Move time.Now() before conn.WriteTo() and add comments. * Only record send time on successful write and send kickoff packet * Lock/unlock mutexes before accessing map items. * Keep all the RTTs * Rename RTTs to Results * Update unit tests * Change results format and allow filenames without subtest * Remove session from cache and save to disk after a call to /result * Fix seqN check and unit tests * Remove debug binary * Update comments. * Use a slice instead of a map for send times. * Add comments. * Add logging and comments * Move recvTime snapshot one instruction earlier. * Add src field to debug message * Set Archivalata.Version to TODO. * Fix received packets count. * Restore original persistence/ package and use "network" as subtest. * Update go.mod/sum. * Remove msak-latency client added by mistake. * s/network/transport/ and s/src/addr * Rename RTT to Packet * Update go.mod/sum * Make the session's mutexes values instead of references * s/Packet/RoundTrip/ * Revert changes to msak-server's main * s/transport/application/ * Rename latency -> latency1 globally * Remove unnecessary rw.WriteHeader after failed Write --- go.mod | 1 + go.sum | 8 + internal/handler/handler.go | 6 +- internal/latency1/latency1.go | 297 +++++++++++++++++++++++++++++ internal/latency1/latency1_test.go | 272 ++++++++++++++++++++++++++ pkg/latency1/model/result.go | 167 ++++++++++++++++ 6 files changed, 748 insertions(+), 3 deletions(-) create mode 100644 internal/latency1/latency1.go create mode 100644 internal/latency1/latency1_test.go create mode 100644 pkg/latency1/model/result.go diff --git a/go.mod b/go.mod index a38bb90..cae1031 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/charmbracelet/log v0.2.1 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.5.0 + github.com/jellydator/ttlcache/v3 v3.0.1 github.com/m-lab/access v0.0.11 github.com/m-lab/go v0.1.58 github.com/m-lab/ndt-server v0.20.17 diff --git a/go.sum b/go.sum index 3d65ebc..2f59bfe 100644 --- a/go.sum +++ b/go.sum @@ -190,12 +190,15 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/google-cloud-go-testing v0.0.0-20191008195207-8e1d251e947d h1:YBqybTXA//1pltKcwyntNQdgDw6AnA5oHZCXFOiZhoo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jellydator/ttlcache/v3 v3.0.1 h1:cHgCSMS7TdQcoprXnWUptJZzyFsqs18Lt8VVhRuZYVU= +github.com/jellydator/ttlcache/v3 v3.0.1/go.mod h1:WwTaEmcXQ3MTjOm4bsZoDFiCu/hMvNWLO1w67RXz6h4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -219,8 +222,10 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/m-lab/access v0.0.11 h1:i2aoal7zgdzXAA7pGL5mXpM8yybURDJGZLwBMmA4Le8= @@ -292,6 +297,7 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -322,6 +328,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -621,6 +628,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9f60af4..e4af0f2 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -68,7 +68,7 @@ func (h *Handler) Upload(rw http.ResponseWriter, req *http.Request) { func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.ResponseWriter, req *http.Request) { - mid, err := getMIDFromRequest(req) + mid, err := GetMIDFromRequest(req) if err != nil { ClientConnections.WithLabelValues(string(kind), "missing-mid").Inc() log.Info("Received request without mid", "source", req.RemoteAddr, @@ -253,13 +253,13 @@ func (h *Handler) writeResult(uuid string, kind model.TestDirection, result *mod } } -// getMIDFromRequest extracts the measurement id ("mid") from a given HTTP +// GetMIDFromRequest extracts the measurement id ("mid") from a given HTTP // request, if present. // // A measurement ID can be specified in two ways: via a "mid" querystring // parameter (when access tokens are not required) or via the ID field // in the JWT access token. -func getMIDFromRequest(req *http.Request) (string, error) { +func GetMIDFromRequest(req *http.Request) (string, error) { // If the request includes a valid JWT token, the claim and the ID are in // the request's context already. claims := controller.GetClaim(req.Context()) diff --git a/internal/latency1/latency1.go b/internal/latency1/latency1.go new file mode 100644 index 0000000..010f287 --- /dev/null +++ b/internal/latency1/latency1.go @@ -0,0 +1,297 @@ +package latency1 + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/http" + "sync" + "time" + + "github.com/charmbracelet/log" + "github.com/jellydator/ttlcache/v3" + "github.com/m-lab/go/memoryless" + "github.com/m-lab/go/rtx" + "github.com/m-lab/msak/internal/handler" + "github.com/m-lab/msak/internal/persistence" + "github.com/m-lab/msak/pkg/latency1/model" +) + +const sendDuration = 5 * time.Second + +var ( + errorUnauthorized = errors.New("unauthorized") + errorInvalidSeqN = errors.New("invalid sequence number") +) + +// Handler is the handler for latency tests. +type Handler struct { + dataDir string + sessions *ttlcache.Cache[string, *model.Session] + sessionsMu sync.Mutex +} + +// NewHandler returns a new handler for the UDP latency test. +// It sets up a cache for sessions that writes the results to disk on item +// eviction. +func NewHandler(dir string, cacheTTL time.Duration) *Handler { + + cache := ttlcache.New( + ttlcache.WithTTL[string, *model.Session](cacheTTL), + ttlcache.WithDisableTouchOnHit[string, *model.Session](), + ) + cache.OnEviction(func(ctx context.Context, + er ttlcache.EvictionReason, + i *ttlcache.Item[string, *model.Session]) { + log.Debug("Session expired", "id", i.Value().ID, "reason", er) + + // Save data to disk when the session expires. + archive := i.Value().Archive() + archive.EndTime = time.Now() + _, err := persistence.WriteDataFile(dir, "latency1", "application", archive.ID, archive) + if err != nil { + log.Error("failed to write latency result", "mid", archive.ID, "error", err) + return + } + }) + + go cache.Start() + return &Handler{ + dataDir: dir, + sessions: cache, + } +} + +// Authorize verifies that the request includes a valid JWT, extracts its jti +// and adds a new empty session to the sessions cache. +// It returns a valid kickoff LatencyPacket for this new session in the +// response body. +func (h *Handler) Authorize(rw http.ResponseWriter, req *http.Request) { + mid, err := handler.GetMIDFromRequest(req) + if err != nil { + log.Info("Received request without mid", "source", req.RemoteAddr, + "error", err) + rw.WriteHeader(http.StatusUnauthorized) + rw.Header().Set("Connection", "Close") + return + } + + // Create a new session for this mid. + session := model.NewSession(mid) + h.sessionsMu.Lock() + h.sessions.Set(mid, session, ttlcache.DefaultTTL) + h.sessionsMu.Unlock() + + log.Debug("session created", "id", mid) + + // Create a valid kickoff packet for this session and send it in the + // response body. + kickoff := &model.LatencyPacket{ + Type: "c2s", + ID: mid, + Seq: 0, + } + + b, err := json.Marshal(kickoff) + // This should never happen. + rtx.Must(err, "cannot marshal LatencyPacket") + + _, err = rw.Write(b) + if err != nil { + // TODO: add Prometheus metric for write errors. + return + } + +} + +// Result returns a result for a given measurement id. Possible status codes +// are: +// - 400 if the request does not contain a mid +// - 404 if the mid is not found in the sessions cache +// - 500 if the session JSON cannot be marshalled +func (h *Handler) Result(rw http.ResponseWriter, req *http.Request) { + mid, err := handler.GetMIDFromRequest(req) + if err != nil { + log.Info("Received request without mid", "source", req.RemoteAddr, + "error", err) + rw.WriteHeader(http.StatusBadRequest) + rw.Header().Set("Connection", "Close") + return + } + + h.sessionsMu.Lock() + cachedResult := h.sessions.Get(mid) + h.sessionsMu.Unlock() + if cachedResult == nil { + rw.WriteHeader(http.StatusNotFound) + return + } + + session := cachedResult.Value() + b, err := json.Marshal(session.Summarize()) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + return + } + + _, err = rw.Write(b) + if err != nil { + // TODO: add Prometheus metric for write errors. + return + } + + // Remove this session from the cache. + h.sessions.Delete(mid) +} + +// sendLoop sends UDP pings with progressive sequence numbers until the context +// expires or is canceled. +func (h *Handler) sendLoop(ctx context.Context, conn net.PacketConn, + remoteAddr net.Addr, session *model.Session, duration time.Duration) error { + seq := 0 + var err error + + timeout, cancel := context.WithTimeout(ctx, duration) + defer cancel() + + memoryless.Run(timeout, func() { + b, marshalErr := json.Marshal(&model.LatencyPacket{ + ID: session.ID, + Type: "s2c", + Seq: seq, + LastRTT: int(session.LastRTT.Load()), + }) + + // This should never happen, since we should always be able to marshal + // a LatencyPacket struct. + rtx.Must(marshalErr, "cannot marshal LatencyPacket") + + // Call time.Now() just before writing to the socket. The RTT will + // include the ping packet's write time. This is intentional. + sendTime := time.Now() + // As the kernel's socket buffers are usually much larger than the + // packets we send here, calling conn.WriteTo is expected to take a + // negligible time. + n, writeErr := conn.WriteTo(b, remoteAddr) + if writeErr != nil { + err = writeErr + cancel() + return + } + if n != len(b) { + err = errors.New("partial write") + cancel() + return + } + + // Update the SendTimes map after a successful write. + session.SendTimesMu.Lock() + session.SendTimes = append(session.SendTimes, sendTime) + session.SendTimesMu.Unlock() + + // Add this packet to the Results slice. Results are "lost" until a + // reply is received from the server. + session.RoundTrips = append(session.RoundTrips, model.RoundTrip{ + Lost: true, + }) + + seq++ + + log.Debug("packet sent", "len", n, "id", session.ID, "seq", seq) + + }, memoryless.Config{ + // Using randomized intervals allows to detect cyclic network + // behaviors where a fixed interval could align to the cycle. + Expected: 25 * time.Millisecond, + Min: 10 * time.Millisecond, + Max: 40 * time.Millisecond, + }) + return err +} + +// processPacket processes a single UDP latency packet. +func (h *Handler) processPacket(conn net.PacketConn, remoteAddr net.Addr, + packet []byte, recvTime time.Time) error { + // Attempt to unmarshal the packet. + var m model.LatencyPacket + err := json.Unmarshal(packet, &m) + if err != nil { + return err + } + + // Check if this is a known session. + h.sessionsMu.Lock() + cachedResult := h.sessions.Get(m.ID) + h.sessionsMu.Unlock() + if cachedResult == nil { + return errorUnauthorized + } + + session := cachedResult.Value() + + // If this message's type is s2c, it was a server ping echoed back by the + // client. Store it in the session's result and compute the RTT. + if m.Type == "s2c" { + session.SendTimesMu.Lock() + defer session.SendTimesMu.Unlock() + if m.Seq >= len(session.SendTimes) { + // TODO: Add Prometheus metric. + log.Info("received packet with valid mid and invalid seq number", + "mid", m.ID, + "seq", m.Seq, + "addr", remoteAddr.String()) + return errorInvalidSeqN + } + + rtt := recvTime.Sub(session.SendTimes[m.Seq]).Microseconds() + session.LastRTT.Store(rtt) + session.RoundTrips[m.Seq].RTT = int(rtt) + session.RoundTrips[m.Seq].Lost = false + + log.Debug("received pong, updating result", "mid", session.ID, + "result", session.RoundTrips[m.Seq]) + // TODO: prometheus metric + return nil + } + + // If this message's type is c2s, it's a kickoff packet. Record + // local/remote addresses and trigger the send loop. + if m.Type == "c2s" { + session.StartedMu.Lock() + defer session.StartedMu.Unlock() + if !session.Started { + session.Started = true + session.Client = remoteAddr.String() + session.Server = conn.LocalAddr().String() + go h.sendLoop(context.Background(), conn, remoteAddr, session, + sendDuration) + } + } + + return nil +} + +// ProcessPacketLoop is the main packet processing loop. For each incoming +// packet, it records its timestamp and acts depending on the packet type. +func (h *Handler) ProcessPacketLoop(conn net.PacketConn) { + log.Info("Accepting UDP packets...") + buf := make([]byte, 1024) + for { + n, addr, err := conn.ReadFrom(buf) + if err != nil { + log.Error("error while reading UDP packet", "err", err) + continue + } + // The receive time should be recorded as soon as possible after + // reading the packet, to improve accuracy. + recvTime := time.Now() + log.Debug("received UDP packet", "addr", addr, "n", n, "data", string(buf[:n])) + err = h.processPacket(conn, addr, buf[:n], recvTime) + if err != nil { + log.Debug("failed to process packet", + "err", err, + "addr", addr.String()) + } + } +} diff --git a/internal/latency1/latency1_test.go b/internal/latency1/latency1_test.go new file mode 100644 index 0000000..f3e5943 --- /dev/null +++ b/internal/latency1/latency1_test.go @@ -0,0 +1,272 @@ +package latency1 + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/jellydator/ttlcache/v3" + "github.com/m-lab/msak/pkg/latency1/model" +) + +func TestNewHandler(t *testing.T) { + tempDir := t.TempDir() + h := NewHandler(tempDir, 2*time.Second) + if h.dataDir != tempDir || h.sessions == nil { + t.Errorf("NewHandler(): invalid handler returned") + } + // Add an item to the cache and check its TTL is the configured one. + h.sessions.Set("test", &model.Session{}, ttlcache.DefaultTTL) + ttl := h.sessions.Get("test").TTL() + if ttl != 2*time.Second { + t.Errorf("cached item has invalid TTL %s", ttl) + } + + // Check that cache cleanup goroutine can be stopped. If it hasn't been + // started, calling Stop() will block indefinitely. + completed := make(chan bool) + go func() { + h.sessions.Stop() + completed <- true + }() + + select { + case <-time.After(1 * time.Second): + t.Fatalf("failed to stop cache cleanup goroutine") + case <-completed: + // NOTHING - happy path + } +} + +func TestOnEviction(t *testing.T) { + // Create a cache with a very low TTL + tempDir := t.TempDir() + h := NewHandler(tempDir, 1*time.Millisecond) + h.sessions.Set("test", model.NewSession("test"), ttlcache.DefaultTTL) + + // Wait for the TTL to expire. + <-time.After(100 * time.Millisecond) + + files, err := os.ReadDir(tempDir) + if err != nil { + t.Fatalf("cannot read temp data folder: %v\n", err) + } + if len(files) == 0 { + t.Errorf("cache expired but no file written") + } +} + +func TestHandler_Authorize(t *testing.T) { + tempDir := t.TempDir() + h := NewHandler(tempDir, 5*time.Second) + + rw := httptest.NewRecorder() + req, err := http.NewRequest(http.MethodGet, "/latency/v1/authorize", nil) + if err != nil { + t.Fatalf("cannot create request: %v", err) + } + + // Valid authorization request. + req.URL.RawQuery = "mid=test" + h.Authorize(rw, req) + if rw.Result().StatusCode != http.StatusOK { + t.Errorf("invalid HTTP status code %d (expected 200)\n", + rw.Result().StatusCode) + } + + // No mid provided on the querystring. + rw = httptest.NewRecorder() + req.URL.RawQuery = "" + h.Authorize(rw, req) + if rw.Result().StatusCode != http.StatusUnauthorized { + t.Errorf("invalid HTTP status code %d (expected %d)\n", + rw.Result().StatusCode, 401) + } +} + +func TestHandler_Result(t *testing.T) { + tempDir := t.TempDir() + h := NewHandler(tempDir, 5*time.Second) + rw := httptest.NewRecorder() + req, err := http.NewRequest(http.MethodGet, "/latency/v1/authorize", nil) + if err != nil { + t.Fatalf("cannot create request: %v", err) + } + req.URL.RawQuery = "mid=test" + h.Authorize(rw, req) + if rw.Result().StatusCode != http.StatusOK { + t.Errorf("invalid HTTP status code %d (expected 200)", + rw.Result().StatusCode) + } + + // Verify that there is an in-memory session for mid=test. + rw = httptest.NewRecorder() + req, err = http.NewRequest(http.MethodGet, "/latency/v1/result", nil) + if err != nil { + t.Fatalf("cannot create request: %v", err) + } + req.URL.RawQuery = "mid=test" + h.Result(rw, req) + if rw.Result().StatusCode != http.StatusOK { + t.Errorf("invalid HTTP status code %d (expected 200)", + rw.Result().StatusCode) + } + // Unmarshal response body. + b, err := io.ReadAll(rw.Body) + if err != nil { + t.Errorf("cannot read response body: %v", err) + } + var summary model.Summary + err = json.Unmarshal(b, &summary) + if err != nil { + t.Errorf("cannot unmarshal response body: %v", err) + } + if summary.ID != "test" { + t.Errorf("invalid ID in summary") + } + + // Do not provide any mid. + rw = httptest.NewRecorder() + req.URL.RawQuery = "" + h.Result(rw, req) + if rw.Result().StatusCode != http.StatusBadRequest { + t.Errorf("invalid HTTP status code %d (expected 400)", + rw.Result().StatusCode) + } + + // Request the summary for a mid that does not exist in the session cache. + rw = httptest.NewRecorder() + req.URL.RawQuery = "mid=doesnotexist" + h.Result(rw, req) + if rw.Result().StatusCode != http.StatusNotFound { + t.Errorf("invalid HTTP status code %d (expected 404)", + rw.Result().StatusCode) + } + +} + +func TestHandler_processPacket(t *testing.T) { + serverConn, err := net.ListenUDP("udp", nil) + if err != nil { + t.Fatalf("cannot create test socket") + } + + tempDir := t.TempDir() + h := NewHandler(tempDir, 5*time.Second) + + clientConn, err := net.Dial("udp", serverConn.LocalAddr().String()) + if err != nil { + t.Fatalf("cannot connect to test socket") + } + + invalidPayload := []byte("test") + err = h.processPacket(serverConn, clientConn.LocalAddr(), + invalidPayload, time.Now()) + if err == nil { + t.Errorf("expected error on invalid payload, got nil.") + } + + invalidSession := []byte(`{"ID":"invalid"}`) + err = h.processPacket(serverConn, clientConn.LocalAddr(), + invalidSession, time.Now()) + if err != errorUnauthorized { + t.Errorf("wrong error: expected %v, got %v", errorUnauthorized, err) + } + + // Add a session to the cache + h.sessions.Set("test", model.NewSession("test"), ttlcache.DefaultTTL) + // Send a kickoff message + validKickoff := []byte(`{"ID":"test","Type":"c2s"}`) + err = h.processPacket(serverConn, clientConn.LocalAddr(), validKickoff, + time.Now()) + if err != nil { + t.Errorf("unexpected error with valid session: %v", err) + } + // Check that packets are received within 1s. + err = clientConn.SetDeadline(time.Now().Add(time.Second)) + if err != nil { + t.Fatalf("failed to set deadline on client conn") + } + buf := make([]byte, 1024) + packetsRead := 0 + for { + n, err := clientConn.Read(buf) + if err != nil { + // The error should not be != timeout + if netErr, ok := err.(net.Error); ok && !netErr.Timeout() { + t.Errorf("Read() returned an error: %v", netErr) + } + break + } + + // Is it a valid latency measurement? + var latencyPacket model.LatencyPacket + fmt.Printf("packet: %s\n", buf[:n]) + err = json.Unmarshal(buf[:n], &latencyPacket) + if err != nil { + t.Fatalf("cannot unmarshal latency packet: %v", err) + } + packetsRead++ + } + + if packetsRead == 0 { + t.Errorf("did not receive any latency packets after kickoff") + } +} + +func Test_processS2CPacket(t *testing.T) { + serverConn, err := net.ListenUDP("udp", nil) + if err != nil { + t.Fatalf("cannot create test socket") + } + + tempDir := t.TempDir() + h := NewHandler(tempDir, 5*time.Second) + + clientConn, err := net.Dial("udp", serverConn.LocalAddr().String()) + if err != nil { + t.Fatalf("cannot connect to test socket") + } + + // Create a valid session with a fake sendTime. + pingTime := time.Now() + pongTime := pingTime.Add(100 * time.Millisecond) + session := h.sessions.Set("test", model.NewSession("test"), + ttlcache.DefaultTTL) + + // Set sendTime for Seq=0 to pingTime. + sendTimes := session.Value().SendTimes + session.Value().SendTimes = append(sendTimes, pingTime) + session.Value().RoundTrips = append(session.Value().RoundTrips, model.RoundTrip{}) + payload := []byte(`{"Type":"s2c","ID":"test","Seq":0}`) + err = h.processPacket(serverConn, clientConn.RemoteAddr(), payload, pongTime) + if err != nil { + t.Fatalf("unexpected error while processing pong packet: %v", err) + } + + // The measurement slice should contain one measurement. + if len(session.Value().RoundTrips) != 1 { + t.Errorf("wrong number of measurements (expected %d, got %d)", 1, + len(session.Value().RoundTrips)) + } + + // Check the computed RTT. + rtt := session.Value().LastRTT.Load() + expected := pongTime.Sub(pingTime).Microseconds() + if session.Value().LastRTT.Load() != pongTime.Sub(pingTime).Microseconds() { + t.Errorf("wrong computed RTT (expected %d, got %d)", expected, rtt) + } + + // Process a pong packet with an unknown sequence number. + payload = []byte(`{"Type":"s2c","ID":"test","Seq":1000}`) + err = h.processPacket(serverConn, clientConn.RemoteAddr(), payload, pongTime) + if err != errorInvalidSeqN { + t.Errorf("wrong error returned: %v", err) + } +} diff --git a/pkg/latency1/model/result.go b/pkg/latency1/model/result.go new file mode 100644 index 0000000..faceab8 --- /dev/null +++ b/pkg/latency1/model/result.go @@ -0,0 +1,167 @@ +package model + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/m-lab/go/prometheusx" +) + +// LatencyPacket is the payload of a latency measurement UDP packet. +type LatencyPacket struct { + // Type is the message type. Possible values are "s2c" and "c2s". + Type string + + // ID is this latency measurement's unique ID. + ID string + + // Seq is the progressive sequence number for this measurement. + Seq int + + // LastRTT is the previous RTT (if any) measured by the party sending this + // message. When there is no previous RTT, this will be zero. + LastRTT int `json:",omitempty"` +} + +// ArchivalData is the archival data format for latency1 measurements. +type ArchivalData struct { + // GitShortCommit is the Git commit (short form) of the running server code. + GitShortCommit string + // Version is the symbolic version (if any) of the running server code. + Version string + // ID is the unique identifier for this latency measurement. + ID string + + // Client is the client's ip:port pair. + Client string + // Server is the server's ip:port pair. + Server string + + // StartTime is the test's start time. + StartTime time.Time + + // EndTime is the test's end time. Since there is no explicit termination + // message in the protocol, this is set when the session expires. + EndTime time.Time + + // RoundTrips is a list of roundtrips. + RoundTrips []RoundTrip + + // PacketSent is the number of packets sent during this measurement. + PacketsSent int + // PacketsReceived is the number of packets received during this + // measurement. + PacketsReceived int +} + +// RoundTrip is a roundtrip. If the reply was lost, Lost will be true. +// If a reply was received, RTT will be populated with the round-trip time. +type RoundTrip struct { + // RTT is the round-trip time (microseconds). + RTT int + // Lost says if the packet was lost. + Lost bool `json:",omitempty"` +} + +// Session is the in-memory structure holding information about a UDP latency +// measurement session. +type Session struct { + // ID is the unique identifier for this latency measurement. + ID string + // StartTime is the test's start time. + StartTime time.Time + // EndTime is the test's end time. + EndTime time.Time + + // Client is the client's ip:port pair. + Client string + // Server is the server's ip:port pair. + Server string + + // Started is true if this session's send loop has been started already. + Started bool + // StartedMu is the mutex associated to Started. + StartedMu sync.Mutex + + // SendTimes is a slice of send times. The slice's index is the packet's + // sequence number. + SendTimes []time.Time + // SendTimesMu is a mutex to synchronize access to SendTimes. + SendTimesMu sync.Mutex + + // RoundTrips is a list of roundtrips. + RoundTrips []RoundTrip + + // LastRTT contains the last observed RTT. + LastRTT *atomic.Int64 +} + +// PacketsReceived returns the number of received packets for this session. +func (s *Session) PacketsReceived() int { + recv := 0 + for _, v := range s.RoundTrips { + if !v.Lost { + recv++ + } + } + return recv +} + +// Summary is the measurement's summary. +type Summary struct { + // ID is the unique identifier for this latency measurement. + ID string + // StartTime is the test's start time. + StartTime time.Time + // Results is a list of RTT results. + Results []RoundTrip + + // PacketSent is the number of packets sent during this measurement. + PacketsSent int + // PacketsReceived is the number of packets received during this + // measurement. + PacketsReceived int +} + +// NewSession returns an empty Session with all the fields initialized. +func NewSession(id string) *Session { + return &Session{ + ID: id, + StartTime: time.Now(), + + Started: false, + + RoundTrips: make([]RoundTrip, 0), + + LastRTT: &atomic.Int64{}, + + SendTimes: []time.Time{}, + } +} + +// Archive converts this Session to ArchivalData. +func (s *Session) Archive() *ArchivalData { + return &ArchivalData{ + ID: s.ID, + GitShortCommit: prometheusx.GitShortCommit, + Version: "TODO", + Client: s.Client, + Server: s.Server, + StartTime: s.StartTime, + RoundTrips: s.RoundTrips, + PacketsSent: len(s.SendTimes), + PacketsReceived: s.PacketsReceived(), + } +} + +// Summarize converts this Session to a Summary. +func (s *Session) Summarize() *Summary { + return &Summary{ + ID: s.ID, + StartTime: s.StartTime, + PacketsSent: len(s.SendTimes), + PacketsReceived: s.PacketsReceived(), + Results: s.RoundTrips, + } +} From 71a5ca1b14c82adbb0f474f26737a8aa8780662c Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Fri, 23 Jun 2023 11:05:56 -0400 Subject: [PATCH 02/37] Rename throughput1 package --- pkg/{ndt8 => throughput1}/model/measurement.go | 0 pkg/{ndt8 => throughput1}/model/namevalue.go | 0 pkg/{ndt8 => throughput1}/model/result.go | 0 pkg/{ndt8 => throughput1}/protocol.go | 0 pkg/{ndt8 => throughput1}/protocol_test.go | 0 pkg/{ndt8 => throughput1}/spec/spec.go | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename pkg/{ndt8 => throughput1}/model/measurement.go (100%) rename pkg/{ndt8 => throughput1}/model/namevalue.go (100%) rename pkg/{ndt8 => throughput1}/model/result.go (100%) rename pkg/{ndt8 => throughput1}/protocol.go (100%) rename pkg/{ndt8 => throughput1}/protocol_test.go (100%) rename pkg/{ndt8 => throughput1}/spec/spec.go (100%) diff --git a/pkg/ndt8/model/measurement.go b/pkg/throughput1/model/measurement.go similarity index 100% rename from pkg/ndt8/model/measurement.go rename to pkg/throughput1/model/measurement.go diff --git a/pkg/ndt8/model/namevalue.go b/pkg/throughput1/model/namevalue.go similarity index 100% rename from pkg/ndt8/model/namevalue.go rename to pkg/throughput1/model/namevalue.go diff --git a/pkg/ndt8/model/result.go b/pkg/throughput1/model/result.go similarity index 100% rename from pkg/ndt8/model/result.go rename to pkg/throughput1/model/result.go diff --git a/pkg/ndt8/protocol.go b/pkg/throughput1/protocol.go similarity index 100% rename from pkg/ndt8/protocol.go rename to pkg/throughput1/protocol.go diff --git a/pkg/ndt8/protocol_test.go b/pkg/throughput1/protocol_test.go similarity index 100% rename from pkg/ndt8/protocol_test.go rename to pkg/throughput1/protocol_test.go diff --git a/pkg/ndt8/spec/spec.go b/pkg/throughput1/spec/spec.go similarity index 100% rename from pkg/ndt8/spec/spec.go rename to pkg/throughput1/spec/spec.go From 207d8b68db6c0b83a88c204935f6de9bf2f7e41c Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Fri, 23 Jun 2023 11:13:24 -0400 Subject: [PATCH 03/37] Rename throughput1 throughout --- Dockerfile | 2 +- cmd/generate-schema/main.go | 14 ++++++------- cmd/msak-server/server.go | 36 ++++++++++++++++---------------- internal/handler/handler.go | 22 +++++++++---------- internal/handler/handler_test.go | 10 ++++----- internal/measurer/measurer.go | 16 +++++++------- pkg/throughput1/model/result.go | 6 +++--- pkg/throughput1/protocol.go | 16 +++++++------- pkg/throughput1/protocol_test.go | 14 ++++++------- pkg/throughput1/spec/spec.go | 10 ++++----- 10 files changed, 73 insertions(+), 73 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7392b5d..875c663 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ COPY --from=build /msak/msak-server /msak/ COPY --from=build /msak/generate-schema /msak/ # Generate msak's JSON schema. -RUN /msak/generate-schema -ndt8=/msak/ndt8.json +RUN /msak/generate-schema -throughput1=/msak/throughput1.json # Verify that the msak-server binary can be run. RUN ./msak-server -h diff --git a/cmd/generate-schema/main.go b/cmd/generate-schema/main.go index fe1f52b..52cf14d 100644 --- a/cmd/generate-schema/main.go +++ b/cmd/generate-schema/main.go @@ -6,27 +6,27 @@ import ( "github.com/m-lab/go/cloud/bqx" "github.com/m-lab/go/rtx" - "github.com/m-lab/msak/pkg/ndt8/model" + "github.com/m-lab/msak/pkg/throughput1/model" "cloud.google.com/go/bigquery" ) var ( - ndt8Schema string + throughput1Schema string ) func init() { - flag.StringVar(&ndt8Schema, "ndt8", "/var/spool/datatypes/ndt8.json", "filename to write ndt8 schema") + flag.StringVar(&throughput1Schema, "throughput1", "/var/spool/datatypes/throughput1.json", "filename to write throughput1 schema") } func main() { flag.Parse() // Generate and save ndt7 schema for autoloading. - ndt8Result := model.NDT8Result{} - sch, err := bigquery.InferSchema(ndt8Result) - rtx.Must(err, "failed to generate ndt8 schema") + throughput1Result := model.Throughput1Result{} + sch, err := bigquery.InferSchema(throughput1Result) + rtx.Must(err, "failed to generate throughput1 schema") sch = bqx.RemoveRequired(sch) b, err := sch.ToJSONFields() rtx.Must(err, "failed to marshal schema") - ioutil.WriteFile(ndt8Schema, b, 0o644) + ioutil.WriteFile(throughput1Schema, b, 0o644) } diff --git a/cmd/msak-server/server.go b/cmd/msak-server/server.go index 9126ab6..686faed 100644 --- a/cmd/msak-server/server.go +++ b/cmd/msak-server/server.go @@ -16,7 +16,7 @@ import ( "github.com/m-lab/go/rtx" "github.com/m-lab/msak/internal/handler" "github.com/m-lab/msak/internal/netx" - "github.com/m-lab/msak/pkg/ndt8/spec" + "github.com/m-lab/msak/pkg/throughput1/spec" ) var ( @@ -71,54 +71,54 @@ func main() { rtx.Must(err, "Failed to load verifier") } // Enforce tokens on uploads and downloads. - ndt8TxPaths := controller.Paths{ + throughput1TxPaths := controller.Paths{ spec.DownloadPath: true, spec.UploadPath: true, } - ndt8TokenPaths := controller.Paths{ + throughput1TokenPaths := controller.Paths{ spec.DownloadPath: true, spec.UploadPath: true, } acm, _ := controller.Setup(ctx, v, tokenVerify, tokenMachine, - ndt8TxPaths, ndt8TokenPaths) + throughput1TxPaths, throughput1TokenPaths) - ndt8Mux := http.NewServeMux() - ndt8Handler := handler.New(*flagDataDir) - ndt8Mux.Handle(spec.DownloadPath, http.HandlerFunc(ndt8Handler.Download)) - ndt8Mux.Handle(spec.UploadPath, http.HandlerFunc(ndt8Handler.Upload)) - ndt8ServerCleartext := httpServer( + throughput1Mux := http.NewServeMux() + throughput1Handler := handler.New(*flagDataDir) + throughput1Mux.Handle(spec.DownloadPath, http.HandlerFunc(throughput1Handler.Download)) + throughput1Mux.Handle(spec.UploadPath, http.HandlerFunc(throughput1Handler.Upload)) + throughput1ServerCleartext := httpServer( *flagEndpointCleartext, - acm.Then(ndt8Mux)) + acm.Then(throughput1Mux)) log.Info("About to listen for ws tests", "endpoint", *flagEndpointCleartext) - tcpl, err := net.Listen("tcp", ndt8ServerCleartext.Addr) + tcpl, err := net.Listen("tcp", throughput1ServerCleartext.Addr) rtx.Must(err, "failed to create listener") l := netx.NewListener(tcpl.(*net.TCPListener)) defer l.Close() go func() { - err := ndt8ServerCleartext.Serve(l) + err := throughput1ServerCleartext.Serve(l) rtx.Must(err, "Could not start cleartext server") - defer ndt8ServerCleartext.Close() + defer throughput1ServerCleartext.Close() }() // Only start TLS-based services if certs and keys are provided if *flagCertFile != "" && *flagKeyFile != "" { - ndt8Server := httpServer( + throughput1Server := httpServer( *flagEndpoint, - acm.Then(ndt8Mux)) + acm.Then(throughput1Mux)) log.Info("About to listen for wss tests", "endpoint", *flagEndpoint) - tcpl, err := net.Listen("tcp", ndt8Server.Addr) + tcpl, err := net.Listen("tcp", throughput1Server.Addr) rtx.Must(err, "failed to create listener") l := netx.NewListener(tcpl.(*net.TCPListener)) defer l.Close() go func() { - err := ndt8Server.ServeTLS(l, *flagCertFile, *flagKeyFile) + err := throughput1Server.ServeTLS(l, *flagCertFile, *flagKeyFile) rtx.Must(err, "Could not start cleartext server") - defer ndt8Server.Close() + defer throughput1Server.Close() }() } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9f60af4..720a926 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -14,13 +14,13 @@ import ( "github.com/m-lab/go/prometheusx" "github.com/m-lab/msak/internal/netx" "github.com/m-lab/msak/internal/persistence" - "github.com/m-lab/msak/pkg/ndt8" - "github.com/m-lab/msak/pkg/ndt8/model" + "github.com/m-lab/msak/pkg/throughput1" + "github.com/m-lab/msak/pkg/throughput1/model" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) -// knownOptions are the known ndt8 options. +// knownOptions are the known throughput1 options. var knownOptions = map[string]struct{}{ "streams": {}, "duration": {}, @@ -41,7 +41,7 @@ var ( ClientConnections = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: "msak", - Subsystem: "ndt8", + Subsystem: "throughput1", Name: "client_connections_total", }, []string{"direction", "status"}, @@ -142,10 +142,10 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res } // Everything looks good, try upgrading the connection to WebSocket. - // Once upgraded, the underlying TCP connection is hijacked and the ndt8 + // Once upgraded, the underlying TCP connection is hijacked and the throughput1 // protocol code will take care of closing it. Note that for this reason // we cannot call writeBadRequest after attempting an Upgrade. - wsConn, err := ndt8.Upgrade(rw, req) + wsConn, err := throughput1.Upgrade(rw, req) if err != nil { ClientConnections.WithLabelValues(string(kind), "websocket-upgrade-failed").Inc() @@ -182,7 +182,7 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res wsConn.Close() return } - archivalData := model.NDT8Result{ + archivalData := model.Throughput1Result{ MeasurementID: mid, UUID: uuid, StartTime: time.Now(), @@ -203,7 +203,7 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res timeout, cancel := context.WithTimeout(req.Context(), duration) defer cancel() - proto := ndt8.New(wsConn) + proto := throughput1.New(wsConn) var senderCh, receiverCh <-chan model.WireMeasurement var errCh <-chan error if kind == model.DirectionDownload { @@ -243,12 +243,12 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res } } -func (h *Handler) writeResult(uuid string, kind model.TestDirection, result *model.NDT8Result) { +func (h *Handler) writeResult(uuid string, kind model.TestDirection, result *model.Throughput1Result) { _, err := persistence.WriteDataFile( - h.archivalDataDir, "ndt8", string(kind), uuid, + h.archivalDataDir, "throughput1", string(kind), uuid, result) if err != nil { - log.Error("failed to write ndt8 result", "uuid", uuid, "error", err) + log.Error("failed to write throughtpu1 result", "uuid", uuid, "error", err) return } } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 8765174..a62dbe3 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -15,9 +15,9 @@ import ( "github.com/m-lab/go/rtx" "github.com/m-lab/msak/internal/handler" "github.com/m-lab/msak/internal/netx" - "github.com/m-lab/msak/pkg/ndt8" - "github.com/m-lab/msak/pkg/ndt8/model" - "github.com/m-lab/msak/pkg/ndt8/spec" + "github.com/m-lab/msak/pkg/throughput1" + "github.com/m-lab/msak/pkg/throughput1/model" + "github.com/m-lab/msak/pkg/throughput1/spec" ) func TestNew(t *testing.T) { @@ -76,7 +76,7 @@ func TestHandler_Upload(t *testing.T) { if conn == nil { t.Fatalf("websocket dial returned nil") } - proto := ndt8.New(conn) + proto := throughput1.New(conn) timeout, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() senderCh, receiverCh, errCh := proto.SenderLoop(timeout) @@ -123,7 +123,7 @@ func TestHandler_Download(t *testing.T) { t.Fatalf("websocket dial returned nil") } - proto := ndt8.New(conn) + proto := throughput1.New(conn) timeout, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() senderCh, receiverCh, errCh := proto.ReceiverLoop(timeout) diff --git a/internal/measurer/measurer.go b/internal/measurer/measurer.go index a4caf71..2516495 100644 --- a/internal/measurer/measurer.go +++ b/internal/measurer/measurer.go @@ -1,6 +1,6 @@ // The measurer package provides functions to periodically read kernel metrics // for a given network connection and return them over a channel wrapped in an -// ndt8 Measurement object. +// throughput1 Measurement object. package measurer import ( @@ -12,11 +12,11 @@ import ( "github.com/m-lab/go/memoryless" "github.com/m-lab/go/rtx" "github.com/m-lab/msak/internal/netx" - "github.com/m-lab/msak/pkg/ndt8/model" - "github.com/m-lab/msak/pkg/ndt8/spec" + "github.com/m-lab/msak/pkg/throughput1/model" + "github.com/m-lab/msak/pkg/throughput1/spec" ) -type ndt8Measurer struct { +type throughput1Measurer struct { connInfo netx.ConnInfo startTime time.Time bytesReadAtStart int64 @@ -33,7 +33,7 @@ type ndt8Measurer struct { // If passed a connection that is not a netx.Conn, this function will panic. func Start(ctx context.Context, conn net.Conn) <-chan model.Measurement { // Implementation note: this channel must be buffered to account for slow - // readers. The "typical" reader is an ndt8 send or receive loop, which + // readers. The "typical" reader is an throughput1 send or receive loop, which // might be busy with data r/w. The buffer size corresponds to at least 10 // seconds: // @@ -42,7 +42,7 @@ func Start(ctx context.Context, conn net.Conn) <-chan model.Measurement { connInfo := netx.ToConnInfo(conn) read, written := connInfo.ByteCounters() - m := &ndt8Measurer{ + m := &throughput1Measurer{ connInfo: connInfo, dstChan: dst, startTime: time.Now(), @@ -58,7 +58,7 @@ func Start(ctx context.Context, conn net.Conn) <-chan model.Measurement { return dst } -func (m *ndt8Measurer) loop(ctx context.Context) { +func (m *throughput1Measurer) loop(ctx context.Context) { log.Debug("Measurer started", "context", ctx) defer log.Debug("Measurer stopped", "context", ctx) t, err := memoryless.NewTicker(ctx, memoryless.Config{ @@ -81,7 +81,7 @@ func (m *ndt8Measurer) loop(ctx context.Context) { } } -func (m *ndt8Measurer) measure(ctx context.Context) { +func (m *throughput1Measurer) measure(ctx context.Context) { // On non-Linux systems, collecting kernel metrics WILL fail. In that case, // we still want to return a (empty) Measurement. bbrInfo, tcpInfo, err := m.connInfo.Info() diff --git a/pkg/throughput1/model/result.go b/pkg/throughput1/model/result.go index 6bf8c6f..652a1a3 100644 --- a/pkg/throughput1/model/result.go +++ b/pkg/throughput1/model/result.go @@ -4,9 +4,9 @@ import ( "time" ) -// NDT8Result is the struct that is serialized as JSON to disk as the archival -// record of an ndt8 test. -type NDT8Result struct { +// Throughput1Result is the struct that is serialized as JSON to disk as the archival +// record of an throughput1 test. +type Throughput1Result struct { // GitShortCommit is the Git commit (short form) of the running server code. GitShortCommit string // Version is the symbolic version (if any) of the running server code. diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index 95fe992..85bae72 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -1,4 +1,4 @@ -package ndt8 +package throughput1 import ( "context" @@ -15,8 +15,8 @@ import ( "github.com/gorilla/websocket" "github.com/m-lab/msak/internal/measurer" "github.com/m-lab/msak/internal/netx" - "github.com/m-lab/msak/pkg/ndt8/model" - "github.com/m-lab/msak/pkg/ndt8/spec" + "github.com/m-lab/msak/pkg/throughput1/model" + "github.com/m-lab/msak/pkg/throughput1/spec" ) type senderFunc func(ctx context.Context, @@ -27,7 +27,7 @@ type Measurer interface { Start(context.Context, net.Conn) <-chan model.Measurement } -// DefaultMeasurer is the default ndt8 measurer that wraps the measurer +// DefaultMeasurer is the default throughput1 measurer that wraps the measurer // package's Start function. type DefaultMeasurer struct{} @@ -36,7 +36,7 @@ func (*DefaultMeasurer) Start(ctx context.Context, return measurer.Start(ctx, c) } -// Protocol is the implementation of the ndt8 protocol. +// Protocol is the implementation of the throughput1 protocol. type Protocol struct { conn *websocket.Conn connInfo netx.ConnInfo @@ -61,7 +61,7 @@ func New(conn *websocket.Conn) *Protocol { // Upgrade takes a HTTP request and upgrades the connection to WebSocket. // Returns a websocket Conn if the upgrade succeeded, and an error otherwise. func Upgrade(w http.ResponseWriter, r *http.Request) (*websocket.Conn, error) { - // We expect WebSocket's subprotocol to be ndt8's. The same subprotocol is + // We expect WebSocket's subprotocol to be throughput1's. The same subprotocol is // added as a header on the response. if r.Header.Get("Sec-WebSocket-Protocol") != spec.SecWebSocketProtocol { w.WriteHeader(http.StatusBadRequest) @@ -91,7 +91,7 @@ func (p *Protocol) makePreparedMessage(size int) (*websocket.PreparedMessage, er return websocket.NewPreparedMessage(websocket.BinaryMessage, data) } -// SenderLoop starts the send loop of the ndt8 protocol. The context's lifetime +// SenderLoop starts the send loop of the throughput1 protocol. The context's lifetime // determines how long to run for. It returns one channel for sender-side // measurements, one channel for receiver-side measurements and one channel for // errors. While the measurements channels could be ignored, the errors channel @@ -101,7 +101,7 @@ func (p *Protocol) SenderLoop(ctx context.Context) (<-chan model.WireMeasurement return p.senderReceiverLoop(ctx, p.sender) } -// ReceiverLoop starts the receiver loop of the ndt8 protocol. The context's +// ReceiverLoop starts the receiver loop of the throughput1 protocol. The context's // lifetime determines how long to run for. It returns one channel for // sender-side measurements, one channel for receiver-side measurements and one // channel for errors. While the measurements channels could be ignored, the diff --git a/pkg/throughput1/protocol_test.go b/pkg/throughput1/protocol_test.go index fe2366e..45bcaa7 100644 --- a/pkg/throughput1/protocol_test.go +++ b/pkg/throughput1/protocol_test.go @@ -1,4 +1,4 @@ -package ndt8_test +package throughput1_test import ( "bytes" @@ -14,8 +14,8 @@ import ( "github.com/gorilla/websocket" "github.com/m-lab/go/rtx" "github.com/m-lab/msak/internal/netx" - "github.com/m-lab/msak/pkg/ndt8" - "github.com/m-lab/msak/pkg/ndt8/spec" + "github.com/m-lab/msak/pkg/throughput1" + "github.com/m-lab/msak/pkg/throughput1/spec" ) func TestProtocol_Upgrade(t *testing.T) { @@ -26,7 +26,7 @@ func TestProtocol_Upgrade(t *testing.T) { r.Header.Add("Upgrade", "websocket") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, err := ndt8.Upgrade(w, r) + _, err := throughput1.Upgrade(w, r) if err != nil { return } @@ -64,9 +64,9 @@ func TestProtocol_Upgrade(t *testing.T) { } func downloadHandler(rw http.ResponseWriter, req *http.Request) { - wsConn, err := ndt8.Upgrade(rw, req) + wsConn, err := throughput1.Upgrade(rw, req) rtx.Must(err, "failed to upgrade to WS") - proto := ndt8.New(wsConn) + proto := throughput1.New(wsConn) ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second) defer cancel() tx, rx, errCh := proto.SenderLoop(ctx) @@ -113,7 +113,7 @@ func TestProtocol_Download(t *testing.T) { conn, _, err := d.Dial(u.String(), headers) rtx.Must(err, "cannot dial server") - proto := ndt8.New(conn) + proto := throughput1.New(conn) senderCh, receiverCh, errCh := proto.ReceiverLoop(context.Background()) start := time.Now() for { diff --git a/pkg/throughput1/spec/spec.go b/pkg/throughput1/spec/spec.go index c7ec202..778f9ac 100644 --- a/pkg/throughput1/spec/spec.go +++ b/pkg/throughput1/spec/spec.go @@ -1,11 +1,11 @@ -// Package spec contains constants for the ndt8 protocol. +// Package spec contains constants for the throughput1 protocol. package spec import "time" const ( // MinMessagesize is the initial size of a Websocket binary message during - // an ndt8 test. + // an throughput1 test. MinMessageSize = 1 << 10 // MaxScaledMessageSize is the maximum value of a scaled binary WebSocket @@ -28,15 +28,15 @@ const ( ScalingFraction = 16 // DownloadPath selects the download subtest. - DownloadPath = "/ndt/v8/download" + DownloadPath = "/msak/v1/download" // UploadPath selects the upload subtest. - UploadPath = "/ndt/v8/upload" + UploadPath = "/mask/v1/upload" // MaxRuntime is the maximum runtime of a subtest. MaxRuntime = 15 * time.Second // SecWebSocketProtocol is the value of the Sec-WebSocket-Protocol header. - SecWebSocketProtocol = "net.measurementlab.ndt.v8" + SecWebSocketProtocol = "net.measurementlab.msak.v1" ) // SubtestKind indicates the subtest kind From 4b2280c72cf96fc00708f59fc0125bedfec0c882 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Fri, 23 Jun 2023 11:19:54 -0400 Subject: [PATCH 04/37] Fix typo --- internal/handler/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 720a926..b64e0a3 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -248,7 +248,7 @@ func (h *Handler) writeResult(uuid string, kind model.TestDirection, result *mod h.archivalDataDir, "throughput1", string(kind), uuid, result) if err != nil { - log.Error("failed to write throughtpu1 result", "uuid", uuid, "error", err) + log.Error("failed to write throughput1 result", "uuid", uuid, "error", err) return } } From dbfd2fca875801112bcc86a3305d350e9c750173 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Fri, 23 Jun 2023 11:23:55 -0400 Subject: [PATCH 05/37] Update resource paths to reference throughput --- pkg/throughput1/spec/spec.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/throughput1/spec/spec.go b/pkg/throughput1/spec/spec.go index 778f9ac..cb966b8 100644 --- a/pkg/throughput1/spec/spec.go +++ b/pkg/throughput1/spec/spec.go @@ -28,15 +28,15 @@ const ( ScalingFraction = 16 // DownloadPath selects the download subtest. - DownloadPath = "/msak/v1/download" + DownloadPath = "/throughput/v1/download" // UploadPath selects the upload subtest. - UploadPath = "/mask/v1/upload" + UploadPath = "/throughput/v1/upload" // MaxRuntime is the maximum runtime of a subtest. MaxRuntime = 15 * time.Second // SecWebSocketProtocol is the value of the Sec-WebSocket-Protocol header. - SecWebSocketProtocol = "net.measurementlab.msak.v1" + SecWebSocketProtocol = "net.measurementlab.throughput.v1" ) // SubtestKind indicates the subtest kind From 2e76e45c0cb8ba36224762a41d1dda25f703b5c4 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Mon, 26 Jun 2023 17:41:22 +0200 Subject: [PATCH 06/37] Rename Results to RoundTrips (#16) --- pkg/latency1/model/result.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/latency1/model/result.go b/pkg/latency1/model/result.go index faceab8..4ed0b60 100644 --- a/pkg/latency1/model/result.go +++ b/pkg/latency1/model/result.go @@ -114,8 +114,8 @@ type Summary struct { ID string // StartTime is the test's start time. StartTime time.Time - // Results is a list of RTT results. - Results []RoundTrip + // RoundTrips is a list of roundtrips. + RoundTrips []RoundTrip // PacketSent is the number of packets sent during this measurement. PacketsSent int @@ -162,6 +162,6 @@ func (s *Session) Summarize() *Summary { StartTime: s.StartTime, PacketsSent: len(s.SendTimes), PacketsReceived: s.PacketsReceived(), - Results: s.RoundTrips, + RoundTrips: s.RoundTrips, } } From 563d58bacaba7cd6eb32631c3c96bf0d9d155375 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Tue, 27 Jun 2023 11:33:16 +0200 Subject: [PATCH 07/37] Add utility functions to netx for saving/loading a uuid to/from a ctx. --- internal/netx/conn.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/internal/netx/conn.go b/internal/netx/conn.go index ac4c9e1..2dfb1e0 100644 --- a/internal/netx/conn.go +++ b/internal/netx/conn.go @@ -1,6 +1,7 @@ package netx import ( + "context" "crypto/tls" "fmt" "net" @@ -17,6 +18,10 @@ import ( "github.com/m-lab/uuid" ) +type contextKey string + +const uuidCtxKey = "netx-uuid" + // ConnInfo provides operations on a net.Conn's underlying file descriptor. type ConnInfo interface { ByteCounters() (uint64, uint64) @@ -120,3 +125,20 @@ func (c *Conn) UUID() (string, error) { } return uuid, nil } + +// SaveUUID saves this connection's UUID in a context.Context using a globally +// unique key. LoadUUID should be used to retrieve the uuid from the context. +func (c *Conn) SaveUUID(ctx context.Context) context.Context { + uuid, _ := c.UUID() + return context.WithValue(ctx, contextKey(uuidCtxKey), uuid) +} + +// LoadUUID reads a connection UUID from a context.Context using a globally +// unique key. Returns an empty string if the UUID is not found in the context. +func LoadUUID(ctx context.Context) string { + uuid, ok := ctx.Value(contextKey(uuidCtxKey)).(string) + if !ok { + return "" + } + return uuid +} From 7f15604b2a644671db2a26c9f8fa79f1f4b99189 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Tue, 27 Jun 2023 11:57:12 +0200 Subject: [PATCH 08/37] Add SaveUUID to interface and add unit tests. --- internal/netx/conn.go | 1 + internal/netx/listener_linux_test.go | 35 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/internal/netx/conn.go b/internal/netx/conn.go index 2dfb1e0..5274a6a 100644 --- a/internal/netx/conn.go +++ b/internal/netx/conn.go @@ -30,6 +30,7 @@ type ConnInfo interface { UUID() (string, error) GetCC() (string, error) SetCC(string) error + SaveUUID(context.Context) context.Context } // ToConnInfo is a helper function to convert a net.Conn into a netx.ConnInfo. diff --git a/internal/netx/listener_linux_test.go b/internal/netx/listener_linux_test.go index ce57df2..078da02 100644 --- a/internal/netx/listener_linux_test.go +++ b/internal/netx/listener_linux_test.go @@ -1,6 +1,7 @@ package netx_test import ( + "context" "io/ioutil" "net" "net/http" @@ -205,3 +206,37 @@ func TestToConnInfoPanic(t *testing.T) { netx.ToConnInfo(&net.UDPConn{}) } + +func TestSaveAndLoadCtx(t *testing.T) { + tcpl, err := net.ListenTCP("tcp", &net.TCPAddr{}) + rtx.Must(err, "failed to create listener") + l := netx.NewListener(tcpl) + defer l.Close() + dialAsync(t, tcpl.Addr().String()) + got, err := l.Accept() + if err != nil { + t.Fatalf("Listener.Accept() unexpected error = %v", err) + } + defer got.Close() + + var c netx.ConnInfo + var ok bool + if c, ok = got.(netx.ConnInfo); !ok { + t.Fatalf("Listener.Accept() wrong Conn type = %T, want netx.Conn", got) + } + + // Attempt to load a UUID from the context when there isn't any. + uuid := netx.LoadUUID(context.Background()) + if uuid != "" { + t.Errorf("LoadUUID: expected empty string, got %s", uuid) + } + + // Check that the UUID is saved to the context. + expected, _ := c.UUID() + ctx := c.SaveUUID(context.Background()) + actual := netx.LoadUUID(ctx) + if actual != expected { + t.Errorf("LoadUUID returned wrong value (expected %s, got %s)", + expected, actual) + } +} From 3bc4cf5f7ce0d23a921e7d39287a4fc3c9ecb109 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Tue, 27 Jun 2023 13:49:01 +0200 Subject: [PATCH 09/37] UUID() cannot return an error anymore. --- internal/handler/handler.go | 9 +-------- internal/netx/conn.go | 9 ++++----- internal/netx/listener_linux_test.go | 5 +---- pkg/throughput1/protocol.go | 5 +---- 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 520d529..1fbd5fc 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -174,14 +174,7 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res } } - uuid, err := conn.UUID() - if err != nil { - // UUID() has a fallback that won't ever fail. This should not happen. - log.Error("Failed to read UUID", "ctx", - fmt.Sprintf("%p", req.Context()), "error", err) - wsConn.Close() - return - } + uuid := conn.UUID() archivalData := model.Throughput1Result{ MeasurementID: mid, UUID: uuid, diff --git a/internal/netx/conn.go b/internal/netx/conn.go index 5274a6a..cacba13 100644 --- a/internal/netx/conn.go +++ b/internal/netx/conn.go @@ -27,7 +27,7 @@ type ConnInfo interface { ByteCounters() (uint64, uint64) Info() (inetdiag.BBRInfo, tcp.LinuxTCPInfo, error) AcceptTime() time.Time - UUID() (string, error) + UUID() string GetCC() (string, error) SetCC(string) error SaveUUID(context.Context) context.Context @@ -115,7 +115,7 @@ func (c *Conn) AcceptTime() time.Time { // UUID returns an M-Lab UUID. On platforms not supporting SO_COOKIE, it // returns a google/uuid as a fallback. If the fallback fails, it panics. -func (c *Conn) UUID() (string, error) { +func (c *Conn) UUID() string { uuid, err := uuid.FromFile(c.fp) if err != nil { // fallback: use google/uuid if the platform does not support SO_COOKIE. @@ -124,14 +124,13 @@ func (c *Conn) UUID() (string, error) { rtx.Must(err, "unable to fallback to uuid") uuid = gid.String() } - return uuid, nil + return uuid } // SaveUUID saves this connection's UUID in a context.Context using a globally // unique key. LoadUUID should be used to retrieve the uuid from the context. func (c *Conn) SaveUUID(ctx context.Context) context.Context { - uuid, _ := c.UUID() - return context.WithValue(ctx, contextKey(uuidCtxKey), uuid) + return context.WithValue(ctx, contextKey(uuidCtxKey), c.UUID()) } // LoadUUID reads a connection UUID from a context.Context using a globally diff --git a/internal/netx/listener_linux_test.go b/internal/netx/listener_linux_test.go index 078da02..1366776 100644 --- a/internal/netx/listener_linux_test.go +++ b/internal/netx/listener_linux_test.go @@ -109,9 +109,6 @@ func TestConn_GetInfoAndUUID(t *testing.T) { if c, ok = got.(netx.ConnInfo); !ok { t.Fatalf("Listener.Accept() wrong Conn type = %T, want netx.Conn", got) } - if _, err := c.UUID(); err != nil { - t.Errorf("GetUUID failed: %v", err) - } if _, _, err = c.Info(); err != nil { t.Fatalf("GetInfo failed: %v", err) } @@ -232,7 +229,7 @@ func TestSaveAndLoadCtx(t *testing.T) { } // Check that the UUID is saved to the context. - expected, _ := c.UUID() + expected := c.UUID() ctx := c.SaveUUID(context.Background()) actual := netx.LoadUUID(ctx) if actual != expected { diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index 85bae72..5ead855 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -286,10 +286,7 @@ func (p *Protocol) createWireMeasurement(ctx context.Context) model.WireMeasurem log.Printf("failed to read cc (ctx %p): %v\n", ctx, err) } - uuid, err := p.connInfo.UUID() - if err != nil { - log.Printf("failed to get UUID (ctx %p): %v\n", ctx, err) - } + uuid := p.connInfo.UUID() wm.CC = cc wm.UUID = uuid return wm From 864458cbaa595b880053f5fd5c20f749f74a79d2 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 28 Jun 2023 12:49:29 +0200 Subject: [PATCH 10/37] Update workflow to use macos-12 and ubuntu 22.04. (#20) --- .github/workflows/test.yml | 2 +- go.mod | 3 ++- go.sum | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58488f3..f1c941b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: go: [ "1.20.2" ] - os: [ "ubuntu-20.04", "windows-2019", "macos-10.15" ] + os: [ "ubuntu-22.04", "windows-2019", "macos-12" ] steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 diff --git a/go.mod b/go.mod index cae1031..ea6ada2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/m-lab/msak -go 1.19 +go 1.20 require ( cloud.google.com/go/bigquery v1.51.1 @@ -10,6 +10,7 @@ require ( github.com/jellydator/ttlcache/v3 v3.0.1 github.com/m-lab/access v0.0.11 github.com/m-lab/go v0.1.58 + github.com/m-lab/locate v0.11.0 github.com/m-lab/ndt-server v0.20.17 github.com/m-lab/tcp-info v1.5.3 github.com/m-lab/uuid v1.0.1 diff --git a/go.sum b/go.sum index 2f59bfe..2040e1e 100644 --- a/go.sum +++ b/go.sum @@ -150,6 +150,7 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= @@ -232,6 +233,8 @@ github.com/m-lab/access v0.0.11 h1:i2aoal7zgdzXAA7pGL5mXpM8yybURDJGZLwBMmA4Le8= github.com/m-lab/access v0.0.11/go.mod h1:ky+hXvIDE1VgEdWhMRJLjYonRrcvfiEJ1BEZtK6+zFQ= github.com/m-lab/go v0.1.58 h1:Byow2/BmhoKy6+0hHMx5WxZiD7lF7fQcnEMtaHNxkC0= github.com/m-lab/go v0.1.58/go.mod h1:O1D/EoVarJ8lZt9foANcqcKtwxHatBzUxXFFyC87aQQ= +github.com/m-lab/locate v0.11.0 h1:9lU4nMSu1tlLk/q5G9hgPABuk5R/CG/jrQOv5RPnrFM= +github.com/m-lab/locate v0.11.0/go.mod h1:yDjn89hHiOkFTxPaTIEvFDoT8ctZZBqwTTpU6TQpHYc= github.com/m-lab/ndt-server v0.20.17 h1:gTgHBbBgrpcyaZVGik84LIXYxHItDpvGZLnfOOs+zKA= github.com/m-lab/ndt-server v0.20.17/go.mod h1:NQyIilZvNVU3+EPYKmZWuZ2mHOPVwqD7c1gEPtkb+MA= github.com/m-lab/tcp-info v1.5.3 h1:4IspTPcNc8D8LNRvuFnID8gDiz+hxPAtYvpKZaiGGe8= From 98d91e29d573bd04764aca600c40290f71fcbe99 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 28 Jun 2023 13:34:28 +0200 Subject: [PATCH 11/37] Add UUID to latency1 model (#18) * Configure server to save the UUID in the request's context. * Add UUID to latency1 model. * Add UUID to session on creation. * Add UUID to request in tests. * Use the connection UUID as main identifier. * Disable HTTP keepalives * Update log.Debug with the uuid * Only test that summary ID is not empty. --- cmd/msak-server/server.go | 13 +++++++++++-- internal/latency1/latency1.go | 20 ++++++++++++++------ internal/latency1/latency1_test.go | 16 ++++++++++++---- pkg/latency1/model/result.go | 18 ++++++++++++------ 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/cmd/msak-server/server.go b/cmd/msak-server/server.go index 686faed..edc700d 100644 --- a/cmd/msak-server/server.go +++ b/cmd/msak-server/server.go @@ -39,10 +39,14 @@ func init() { flag.StringVar(&tokenMachine, "token.machine", "", "Use given machine name to verify token claims") } -// httpServer creates a new *http.Server with explicit Read and Write timeouts. +// httpServer creates a new *http.Server with explicit Read and Write +// timeouts, the provided address and handler, and an empty TLS configuration. +// +// This server can only be used with a net.Listener that returns netx.ConnInfo +// after accepting a new connection. func httpServer(addr string, handler http.Handler) *http.Server { tlsconf := &tls.Config{} - return &http.Server{ + s := &http.Server{ Addr: addr, Handler: handler, TLSConfig: tlsconf, @@ -52,7 +56,12 @@ func httpServer(addr string, handler http.Handler) *http.Server { // servers. ReadTimeout: time.Minute, WriteTimeout: time.Minute, + ConnContext: func(ctx context.Context, c net.Conn) context.Context { + return netx.ToConnInfo(c).SaveUUID(ctx) + }, } + s.SetKeepAlivesEnabled(false) + return s } func main() { diff --git a/internal/latency1/latency1.go b/internal/latency1/latency1.go index 010f287..eb21e98 100644 --- a/internal/latency1/latency1.go +++ b/internal/latency1/latency1.go @@ -14,6 +14,7 @@ import ( "github.com/m-lab/go/memoryless" "github.com/m-lab/go/rtx" "github.com/m-lab/msak/internal/handler" + "github.com/m-lab/msak/internal/netx" "github.com/m-lab/msak/internal/persistence" "github.com/m-lab/msak/pkg/latency1/model" ) @@ -44,7 +45,7 @@ func NewHandler(dir string, cacheTTL time.Duration) *Handler { cache.OnEviction(func(ctx context.Context, er ttlcache.EvictionReason, i *ttlcache.Item[string, *model.Session]) { - log.Debug("Session expired", "id", i.Value().ID, "reason", er) + log.Debug("Session expired", "id", i.Key(), "reason", er) // Save data to disk when the session expires. archive := i.Value().Archive() @@ -77,13 +78,20 @@ func (h *Handler) Authorize(rw http.ResponseWriter, req *http.Request) { return } + // Retrieve the connection's UUID from context. + uuid := netx.LoadUUID(req.Context()) + if uuid == "" { + // This cannot happen unless the HTTP server instance is misconfigured. + log.Fatal("received request without UUID", "addr", req.RemoteAddr) + } + // Create a new session for this mid. - session := model.NewSession(mid) + session := model.NewSession(uuid) h.sessionsMu.Lock() h.sessions.Set(mid, session, ttlcache.DefaultTTL) h.sessionsMu.Unlock() - log.Debug("session created", "id", mid) + log.Debug("session created", "id", mid, "uuid", uuid) // Create a valid kickoff packet for this session and send it in the // response body. @@ -157,7 +165,7 @@ func (h *Handler) sendLoop(ctx context.Context, conn net.PacketConn, memoryless.Run(timeout, func() { b, marshalErr := json.Marshal(&model.LatencyPacket{ - ID: session.ID, + ID: session.UUID, Type: "s2c", Seq: seq, LastRTT: int(session.LastRTT.Load()), @@ -198,7 +206,7 @@ func (h *Handler) sendLoop(ctx context.Context, conn net.PacketConn, seq++ - log.Debug("packet sent", "len", n, "id", session.ID, "seq", seq) + log.Debug("packet sent", "len", n, "uuid", session.UUID, "seq", seq) }, memoryless.Config{ // Using randomized intervals allows to detect cyclic network @@ -249,7 +257,7 @@ func (h *Handler) processPacket(conn net.PacketConn, remoteAddr net.Addr, session.RoundTrips[m.Seq].RTT = int(rtt) session.RoundTrips[m.Seq].Lost = false - log.Debug("received pong, updating result", "mid", session.ID, + log.Debug("received pong, updating result", "uuid", session.UUID, "result", session.RoundTrips[m.Seq]) // TODO: prometheus metric return nil diff --git a/internal/latency1/latency1_test.go b/internal/latency1/latency1_test.go index f3e5943..cf80e13 100644 --- a/internal/latency1/latency1_test.go +++ b/internal/latency1/latency1_test.go @@ -1,6 +1,7 @@ package latency1 import ( + "context" "encoding/json" "fmt" "io" @@ -12,6 +13,7 @@ import ( "time" "github.com/jellydator/ttlcache/v3" + "github.com/m-lab/msak/internal/netx" "github.com/m-lab/msak/pkg/latency1/model" ) @@ -67,7 +69,10 @@ func TestHandler_Authorize(t *testing.T) { h := NewHandler(tempDir, 5*time.Second) rw := httptest.NewRecorder() - req, err := http.NewRequest(http.MethodGet, "/latency/v1/authorize", nil) + conn := netx.Conn{} + ctx := conn.SaveUUID(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + "/latency/v1/authorize", nil) if err != nil { t.Fatalf("cannot create request: %v", err) } @@ -94,7 +99,10 @@ func TestHandler_Result(t *testing.T) { tempDir := t.TempDir() h := NewHandler(tempDir, 5*time.Second) rw := httptest.NewRecorder() - req, err := http.NewRequest(http.MethodGet, "/latency/v1/authorize", nil) + conn := netx.Conn{} + ctx := conn.SaveUUID(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + "/latency/v1/authorize", nil) if err != nil { t.Fatalf("cannot create request: %v", err) } @@ -127,8 +135,8 @@ func TestHandler_Result(t *testing.T) { if err != nil { t.Errorf("cannot unmarshal response body: %v", err) } - if summary.ID != "test" { - t.Errorf("invalid ID in summary") + if summary.ID == "" { + t.Errorf("empty ID in summary") } // Do not provide any mid. diff --git a/pkg/latency1/model/result.go b/pkg/latency1/model/result.go index 4ed0b60..4c05208 100644 --- a/pkg/latency1/model/result.go +++ b/pkg/latency1/model/result.go @@ -33,6 +33,10 @@ type ArchivalData struct { // ID is the unique identifier for this latency measurement. ID string + // UUID is the unique identifier of the TCP connection that started this + // latency measurement. + UUID string + // Client is the client's ip:port pair. Client string // Server is the server's ip:port pair. @@ -67,8 +71,10 @@ type RoundTrip struct { // Session is the in-memory structure holding information about a UDP latency // measurement session. type Session struct { - // ID is the unique identifier for this latency measurement. - ID string + // UUID is the unique identifier of the TCP connection that started + // this latency measurement. + UUID string + // StartTime is the test's start time. StartTime time.Time // EndTime is the test's end time. @@ -125,9 +131,9 @@ type Summary struct { } // NewSession returns an empty Session with all the fields initialized. -func NewSession(id string) *Session { +func NewSession(uuid string) *Session { return &Session{ - ID: id, + UUID: uuid, StartTime: time.Now(), Started: false, @@ -143,7 +149,7 @@ func NewSession(id string) *Session { // Archive converts this Session to ArchivalData. func (s *Session) Archive() *ArchivalData { return &ArchivalData{ - ID: s.ID, + ID: s.UUID, GitShortCommit: prometheusx.GitShortCommit, Version: "TODO", Client: s.Client, @@ -158,7 +164,7 @@ func (s *Session) Archive() *ArchivalData { // Summarize converts this Session to a Summary. func (s *Session) Summarize() *Summary { return &Summary{ - ID: s.ID, + ID: s.UUID, StartTime: s.StartTime, PacketsSent: len(s.SendTimes), PacketsReceived: s.PacketsReceived(), From 47bd0de81617dcd515bb01409e9c511f35197e6d Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 28 Jun 2023 13:35:11 +0200 Subject: [PATCH 12/37] Add latency1 handlers and UDP server to msak-server (#19) * Add utility functions to netx for saving/loading a uuid to/from a ctx. * Add SaveUUID to interface and add unit tests. * UUID() cannot return an error anymore. * Configure server to save the UUID in the request's context. * Add UUID to latency1 model. * Add UUID to session on creation. * Add UUID to request in tests. * Use the connection UUID as main identifier. * Disable HTTP keepalives * Update log.Debug with the uuid * Only test that summary ID is not empty. * Add spec package for latency1 * Add handlers for latency1 measurement and start UDP server. * s/udpServer/udpListener/ --- cmd/msak-server/server.go | 36 ++++++++++++++++++++++++++++-------- pkg/latency1/spec/spec.go | 13 +++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 pkg/latency1/spec/spec.go diff --git a/cmd/msak-server/server.go b/cmd/msak-server/server.go index edc700d..a5bb150 100644 --- a/cmd/msak-server/server.go +++ b/cmd/msak-server/server.go @@ -15,7 +15,9 @@ import ( "github.com/m-lab/go/prometheusx" "github.com/m-lab/go/rtx" "github.com/m-lab/msak/internal/handler" + "github.com/m-lab/msak/internal/latency1" "github.com/m-lab/msak/internal/netx" + latency1spec "github.com/m-lab/msak/pkg/latency1/spec" "github.com/m-lab/msak/pkg/throughput1/spec" ) @@ -25,9 +27,12 @@ var ( flagEndpoint = flag.String("wss_addr", ":4443", "Listen address/port for TLS connections") flagEndpointCleartext = flag.String("ws_addr", ":8080", "Listen address/port for cleartext connections") flagDataDir = flag.String("datadir", "./data", "Directory to store data in") - tokenVerifyKey = flagx.FileBytesArray{} - tokenVerify bool - tokenMachine string + flagLatencyEndpoint = flag.String("latency_addr", ":1053", "Listen address/port for UDP latency tests") + flagLatencyTTL = flag.Duration("latency_ttl", + latency1spec.DefaultSessionCacheTTL, "Session cache's TTL") + tokenVerifyKey = flagx.FileBytesArray{} + tokenVerify bool + tokenMachine string // Context for the whole program. ctx, cancel = context.WithCancel(context.Background()) @@ -91,13 +96,19 @@ func main() { acm, _ := controller.Setup(ctx, v, tokenVerify, tokenMachine, throughput1TxPaths, throughput1TokenPaths) - throughput1Mux := http.NewServeMux() + mux := http.NewServeMux() + latency1Handler := latency1.NewHandler(*flagDataDir, *flagLatencyTTL) throughput1Handler := handler.New(*flagDataDir) - throughput1Mux.Handle(spec.DownloadPath, http.HandlerFunc(throughput1Handler.Download)) - throughput1Mux.Handle(spec.UploadPath, http.HandlerFunc(throughput1Handler.Upload)) + + mux.Handle(spec.DownloadPath, http.HandlerFunc(throughput1Handler.Download)) + mux.Handle(spec.UploadPath, http.HandlerFunc(throughput1Handler.Upload)) + mux.Handle(latency1spec.AuthorizeV1, http.HandlerFunc( + latency1Handler.Authorize)) + mux.Handle(latency1spec.ResultV1, http.HandlerFunc( + latency1Handler.Result)) throughput1ServerCleartext := httpServer( *flagEndpointCleartext, - acm.Then(throughput1Mux)) + acm.Then(mux)) log.Info("About to listen for ws tests", "endpoint", *flagEndpointCleartext) @@ -116,7 +127,7 @@ func main() { if *flagCertFile != "" && *flagKeyFile != "" { throughput1Server := httpServer( *flagEndpoint, - acm.Then(throughput1Mux)) + acm.Then(mux)) log.Info("About to listen for wss tests", "endpoint", *flagEndpoint) tcpl, err := net.Listen("tcp", throughput1Server.Addr) @@ -131,6 +142,15 @@ func main() { }() } + // Start a UDP server for latency measurements. + addr, err := net.ResolveUDPAddr("udp", *flagLatencyEndpoint) + rtx.Must(err, "failed to resolve latency endpoint address") + udpListener, err := net.ListenUDP("udp", addr) + rtx.Must(err, "cannot start latency UDP server") + defer udpListener.Close() + + go latency1Handler.ProcessPacketLoop(udpListener) + <-ctx.Done() cancel() } diff --git a/pkg/latency1/spec/spec.go b/pkg/latency1/spec/spec.go new file mode 100644 index 0000000..602928c --- /dev/null +++ b/pkg/latency1/spec/spec.go @@ -0,0 +1,13 @@ +package spec + +import "time" + +const ( + // AuthorizeV1 is the v1 /authorize endpoint. + AuthorizeV1 = "/latency/v1/authorize" + // ResultV1 is the v1 /result endpoint. + ResultV1 = "/latency/v1/result" + + // DefaultSessionCacheTTL is the default session cache TTL. + DefaultSessionCacheTTL = 1 * time.Minute +) From 1daff597b0efde46eda4c80ccca3b192a014fdd7 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Fri, 21 Jul 2023 14:14:27 +0200 Subject: [PATCH 13/37] Send session ID in packets instead of UUID (#21) --- internal/latency1/latency1.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/latency1/latency1.go b/internal/latency1/latency1.go index eb21e98..f9d13bf 100644 --- a/internal/latency1/latency1.go +++ b/internal/latency1/latency1.go @@ -156,7 +156,7 @@ func (h *Handler) Result(rw http.ResponseWriter, req *http.Request) { // sendLoop sends UDP pings with progressive sequence numbers until the context // expires or is canceled. func (h *Handler) sendLoop(ctx context.Context, conn net.PacketConn, - remoteAddr net.Addr, session *model.Session, duration time.Duration) error { + remoteAddr net.Addr, id string, session *model.Session, duration time.Duration) error { seq := 0 var err error @@ -165,7 +165,7 @@ func (h *Handler) sendLoop(ctx context.Context, conn net.PacketConn, memoryless.Run(timeout, func() { b, marshalErr := json.Marshal(&model.LatencyPacket{ - ID: session.UUID, + ID: id, Type: "s2c", Seq: seq, LastRTT: int(session.LastRTT.Load()), @@ -272,7 +272,7 @@ func (h *Handler) processPacket(conn net.PacketConn, remoteAddr net.Addr, session.Started = true session.Client = remoteAddr.String() session.Server = conn.LocalAddr().String() - go h.sendLoop(context.Background(), conn, remoteAddr, session, + go h.sendLoop(context.Background(), conn, remoteAddr, m.ID, session, sendDuration) } } From 24a1228377e51adc0c6514fb2de1ed98844d6ad0 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Tue, 25 Jul 2023 12:47:25 +0200 Subject: [PATCH 14/37] Add latency1 paths to access controller (#22) * Add latency1 paths to txcontroller. * Update names and comment --- cmd/msak-server/server.go | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/cmd/msak-server/server.go b/cmd/msak-server/server.go index a5bb150..aa4476a 100644 --- a/cmd/msak-server/server.go +++ b/cmd/msak-server/server.go @@ -84,17 +84,21 @@ func main() { if (tokenVerify) && err != nil { rtx.Must(err, "Failed to load verifier") } - // Enforce tokens on uploads and downloads. - throughput1TxPaths := controller.Paths{ - spec.DownloadPath: true, - spec.UploadPath: true, + // Enforce tokens and txcontroller on every endpoint. + txControllerPaths := controller.Paths{ + spec.DownloadPath: true, + spec.UploadPath: true, + latency1spec.AuthorizeV1: true, + latency1spec.ResultV1: true, } - throughput1TokenPaths := controller.Paths{ - spec.DownloadPath: true, - spec.UploadPath: true, + tokenPaths := controller.Paths{ + spec.DownloadPath: true, + spec.UploadPath: true, + latency1spec.AuthorizeV1: true, + latency1spec.ResultV1: true, } acm, _ := controller.Setup(ctx, v, tokenVerify, tokenMachine, - throughput1TxPaths, throughput1TokenPaths) + txControllerPaths, tokenPaths) mux := http.NewServeMux() latency1Handler := latency1.NewHandler(*flagDataDir, *flagLatencyTTL) @@ -106,39 +110,39 @@ func main() { latency1Handler.Authorize)) mux.Handle(latency1spec.ResultV1, http.HandlerFunc( latency1Handler.Result)) - throughput1ServerCleartext := httpServer( + serverCleartext := httpServer( *flagEndpointCleartext, acm.Then(mux)) log.Info("About to listen for ws tests", "endpoint", *flagEndpointCleartext) - tcpl, err := net.Listen("tcp", throughput1ServerCleartext.Addr) + tcpl, err := net.Listen("tcp", serverCleartext.Addr) rtx.Must(err, "failed to create listener") l := netx.NewListener(tcpl.(*net.TCPListener)) defer l.Close() go func() { - err := throughput1ServerCleartext.Serve(l) + err := serverCleartext.Serve(l) rtx.Must(err, "Could not start cleartext server") - defer throughput1ServerCleartext.Close() + defer serverCleartext.Close() }() // Only start TLS-based services if certs and keys are provided if *flagCertFile != "" && *flagKeyFile != "" { - throughput1Server := httpServer( + server := httpServer( *flagEndpoint, acm.Then(mux)) log.Info("About to listen for wss tests", "endpoint", *flagEndpoint) - tcpl, err := net.Listen("tcp", throughput1Server.Addr) + tcpl, err := net.Listen("tcp", server.Addr) rtx.Must(err, "failed to create listener") l := netx.NewListener(tcpl.(*net.TCPListener)) defer l.Close() go func() { - err := throughput1Server.ServeTLS(l, *flagCertFile, *flagKeyFile) + err := server.ServeTLS(l, *flagCertFile, *flagKeyFile) rtx.Must(err, "Could not start cleartext server") - defer throughput1Server.Close() + defer server.Close() }() } From 97674886bbf8d8fd73e243d346e53848ac7c99d6 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Tue, 8 Aug 2023 15:47:49 +0200 Subject: [PATCH 15/37] Separate network-level and application-level measurements (#23) * Separate network-level and application-level measurements. This commit adds application-level measurements and namespaces to better separate the two kind of measurements in the Measurement object. It is a breaking change and will need a schema update. * Make application-level byte counters atomic. * sync.Once and atomic.Int64 do not need to be pointers. --- internal/measurer/measurer.go | 10 +++-- internal/measurer/measurer_test.go | 2 +- pkg/throughput1/model/measurement.go | 19 +++++---- pkg/throughput1/protocol.go | 63 ++++++++++++++++++++++++---- pkg/throughput1/protocol_test.go | 6 ++- 5 files changed, 78 insertions(+), 22 deletions(-) diff --git a/internal/measurer/measurer.go b/internal/measurer/measurer.go index 2516495..246933a 100644 --- a/internal/measurer/measurer.go +++ b/internal/measurer/measurer.go @@ -96,10 +96,12 @@ func (m *throughput1Measurer) measure(ctx context.Context) { case <-ctx.Done(): // NOTHING case m.dstChan <- model.Measurement{ - ElapsedTime: time.Since(m.startTime).Microseconds(), - BytesSent: int64(totalWritten) - m.bytesWrittenAtStart, - BytesReceived: int64(totalRead) - m.bytesReadAtStart, - BBRInfo: &bbrInfo, + ElapsedTime: time.Since(m.startTime).Microseconds(), + Network: model.ByteCounters{ + BytesSent: int64(totalWritten) - m.bytesWrittenAtStart, + BytesReceived: int64(totalRead) - m.bytesReadAtStart, + }, + BBRInfo: &bbrInfo, TCPInfo: &model.TCPInfo{ LinuxTCPInfo: tcpInfo, ElapsedTime: time.Since(m.connInfo.AcceptTime()).Microseconds(), diff --git a/internal/measurer/measurer_test.go b/internal/measurer/measurer_test.go index 9872c5e..6205f36 100644 --- a/internal/measurer/measurer_test.go +++ b/internal/measurer/measurer_test.go @@ -35,7 +35,7 @@ func TestNdt8Measurer_Start(t *testing.T) { select { case m := <-mchan: fmt.Println("received measurement") - if m.BytesSent != 4 { + if m.Network.BytesSent != 4 { t.Errorf("invalid byte counter value") } case <-time.After(1 * time.Second): diff --git a/pkg/throughput1/model/measurement.go b/pkg/throughput1/model/measurement.go index 40d5624..7db6603 100644 --- a/pkg/throughput1/model/measurement.go +++ b/pkg/throughput1/model/measurement.go @@ -24,13 +24,10 @@ type WireMeasurement struct { // The Measurement struct contains measurement results. This structure is // meant to be serialised as JSON and sent as a textual message. type Measurement struct { - // BytesSent is the number of bytes sent at the application level by the - // party sending this Measurement. - BytesSent int64 `json:",omitempty"` - - // BytesReceived is the number of bytes received at the application level - // by the party sending this Measurement. - BytesReceived int64 `json:",omitempty"` + // Application contains the application-level BytesSent/Received pair. + Application ByteCounters + // Network contains the network-level BytesSent/Received pair. + Network ByteCounters // ElapsedTime is the time elapsed since the start of the measurement // according to the party sending this Measurement. @@ -47,6 +44,14 @@ type Measurement struct { TCPInfo *TCPInfo `json:",omitempty"` } +type ByteCounters struct { + // BytesSent is the number of bytes sent. + BytesSent int64 `json:",omitempty"` + + // BytesReceived is the number of bytes received. + BytesReceived int64 `json:",omitempty"` +} + // TCPInfo is an extension to Linux's TCPInfo struct that includes the time // elapsed since the connection was accepted. type TCPInfo struct { diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index 5ead855..1800819 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -10,6 +10,7 @@ import ( "net" "net/http" "sync" + "sync/atomic" "time" "github.com/gorilla/websocket" @@ -42,7 +43,10 @@ type Protocol struct { connInfo netx.ConnInfo rnd *rand.Rand measurer Measurer - once *sync.Once + once sync.Once + + applicationBytesReceived atomic.Int64 + applicationBytesSent atomic.Int64 } // New returns a new Protocol with the specified connection and every other @@ -54,7 +58,6 @@ func New(conn *websocket.Conn) *Protocol { // Seed randomness source with the current time. rnd: rand.New(rand.NewSource(time.Now().UnixMilli())), measurer: &DefaultMeasurer{}, - once: &sync.Once{}, } } @@ -138,7 +141,8 @@ func (p *Protocol) senderReceiverLoop(ctx context.Context, } // receiver reads from the connection until NextReader fails. It returns -// the measurements received over the provided channel. +// the measurements received over the provided channel and updates the sent and +// received byte counters as needed. func (p *Protocol) receiver(ctx context.Context, results chan<- model.WireMeasurement, errCh chan<- error) { for { @@ -147,12 +151,22 @@ func (p *Protocol) receiver(ctx context.Context, errCh <- err return } + if kind == websocket.BinaryMessage { + // Binary messages are discarded after reading their size. + size, err := io.Copy(io.Discard, reader) + if err != nil { + errCh <- err + return + } + p.applicationBytesReceived.Add(size) + } if kind == websocket.TextMessage { data, err := io.ReadAll(reader) if err != nil { errCh <- err return } + p.applicationBytesReceived.Add(int64(len(data))) var m model.WireMeasurement if err := json.Unmarshal(data, &m); err != nil { errCh <- err @@ -177,12 +191,27 @@ func (p *Protocol) sendCounterflow(ctx context.Context, wm = p.createWireMeasurement(ctx) }) wm.Measurement = m - err := p.conn.WriteJSON(wm) + wm.Application = model.ByteCounters{ + BytesSent: p.applicationBytesSent.Load(), + BytesReceived: p.applicationBytesReceived.Load(), + } + // Encode as JSON separately so we can read the message size before + // sending. + jsonwm, err := json.Marshal(wm) + if err != nil { + log.Printf("failed to encode measurement (ctx: %p, err: %v)", + ctx, err) + errCh <- err + return + } + err = p.conn.WriteMessage(websocket.TextMessage, jsonwm) if err != nil { log.Printf("failed to write measurement JSON (ctx: %p, err: %v)", ctx, err) errCh <- err return } + p.applicationBytesSent.Add(int64(len(jsonwm))) + // This send is non-blocking in case there is no one to read the // Measurement message and the channel's buffer is full. select { @@ -195,7 +224,6 @@ func (p *Protocol) sendCounterflow(ctx context.Context, func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measurement, results chan<- model.WireMeasurement, errCh chan<- error) { - ci := netx.ToConnInfo(p.conn.UnderlyingConn()) size := spec.MinMessageSize message, err := p.makePreparedMessage(size) if err != nil { @@ -219,12 +247,27 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme wm = p.createWireMeasurement(ctx) }) wm.Measurement = m - err = p.conn.WriteJSON(wm) + wm.Application = model.ByteCounters{ + BytesReceived: p.applicationBytesReceived.Load(), + BytesSent: p.applicationBytesSent.Load(), + } + // Encode as JSON separately so we can read the message size before + // sending. + jsonwm, err := json.Marshal(wm) + if err != nil { + log.Printf("failed to encode measurement (ctx: %p, err: %v)", + ctx, err) + errCh <- err + return + } + err = p.conn.WriteMessage(websocket.TextMessage, jsonwm) if err != nil { log.Printf("failed to write measurement JSON (ctx: %p, err: %v)", ctx, err) errCh <- err return } + p.applicationBytesSent.Add(int64(len(jsonwm))) + // This send is non-blocking in case there is no one to read the // Measurement message and the channel's buffer is full. select { @@ -238,14 +281,14 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme errCh <- err return } + p.applicationBytesSent.Add(int64(size)) // Determine whether it's time to scale the message size. if size >= spec.MaxScaledMessageSize { continue } - _, w := ci.ByteCounters() - if uint64(size) > w/spec.ScalingFraction { + if size > int(p.applicationBytesSent.Load())/spec.ScalingFraction { continue } @@ -264,11 +307,15 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme func (p *Protocol) close(ctx context.Context) { msg := websocket.FormatCloseMessage( websocket.CloseNormalClosure, "Done sending") + err := p.conn.WriteControl(websocket.CloseMessage, msg, time.Now().Add(time.Second)) if err != nil { log.Printf("WriteControl failed (ctx: %p, err: %v)", ctx, err) return } + // The closing message is part of the measurement and added to bytesSent. + p.applicationBytesSent.Add(int64(len(msg))) + log.Printf("Close message sent (ctx: %p)", ctx) } diff --git a/pkg/throughput1/protocol_test.go b/pkg/throughput1/protocol_test.go index 45bcaa7..c3fbbbe 100644 --- a/pkg/throughput1/protocol_test.go +++ b/pkg/throughput1/protocol_test.go @@ -121,8 +121,10 @@ func TestProtocol_Download(t *testing.T) { case <-context.Background().Done(): return case m := <-senderCh: - fmt.Printf("senderCh BytesReceived: %d, BytesSent: %d\n", m.BytesReceived, m.BytesSent) - fmt.Printf("senderCh Goodput: %f Mb/s\n", float64(m.BytesReceived)/float64(time.Since(start).Microseconds())*8) + fmt.Printf("senderCh Network.BytesReceived: %d, Network.BytesSent: %d\n", + m.Network.BytesReceived, m.Network.BytesSent) + fmt.Printf("senderCh Network throughput: %f Mb/s\n", + float64(m.Network.BytesReceived)/float64(time.Since(start).Microseconds())*8) case <-receiverCh: case err := <-errCh: From f1935956dae6ce9f4080552180d1047cd7b917d6 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 9 Aug 2023 00:26:56 +0200 Subject: [PATCH 16/37] Add latency1 schema to generate-schema tool (#25) * Add latency1 schema to generate-schema tool. * Update dockerfile to also write latency1 schema --- Dockerfile | 4 ++-- cmd/generate-schema/main.go | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 875c663..c9cddc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,8 @@ WORKDIR /msak COPY --from=build /msak/msak-server /msak/ COPY --from=build /msak/generate-schema /msak/ -# Generate msak's JSON schema. -RUN /msak/generate-schema -throughput1=/msak/throughput1.json +# Generate msak's JSON schemas. +RUN /msak/generate-schema -throughput1=/msak/throughput1.json -latency1=/msak/latency1.json # Verify that the msak-server binary can be run. RUN ./msak-server -h diff --git a/cmd/generate-schema/main.go b/cmd/generate-schema/main.go index 52cf14d..5469cf5 100644 --- a/cmd/generate-schema/main.go +++ b/cmd/generate-schema/main.go @@ -2,10 +2,11 @@ package main import ( "flag" - "io/ioutil" + "os" "github.com/m-lab/go/cloud/bqx" "github.com/m-lab/go/rtx" + latency1model "github.com/m-lab/msak/pkg/latency1/model" "github.com/m-lab/msak/pkg/throughput1/model" "cloud.google.com/go/bigquery" @@ -13,20 +14,33 @@ import ( var ( throughput1Schema string + latency1Schema string ) func init() { flag.StringVar(&throughput1Schema, "throughput1", "/var/spool/datatypes/throughput1.json", "filename to write throughput1 schema") + flag.StringVar(&latency1Schema, "latency1", "/var/spool/datatypes/latency1.json", "filename to write latency1 schema") } func main() { flag.Parse() - // Generate and save ndt7 schema for autoloading. + // Generate and save schemas for autoloading. + // throughput1 schema. throughput1Result := model.Throughput1Result{} sch, err := bigquery.InferSchema(throughput1Result) rtx.Must(err, "failed to generate throughput1 schema") sch = bqx.RemoveRequired(sch) b, err := sch.ToJSONFields() - rtx.Must(err, "failed to marshal schema") - ioutil.WriteFile(throughput1Schema, b, 0o644) + rtx.Must(err, "failed to marshal throughput1 schema") + err = os.WriteFile(throughput1Schema, b, 0o644) + rtx.Must(err, "failed to write throughput1 schema") + // latency1 schema. + latency1Result := latency1model.ArchivalData{} + sch, err = bigquery.InferSchema(latency1Result) + rtx.Must(err, "failed to generate latency1 schema") + sch = bqx.RemoveRequired(sch) + b, err = sch.ToJSONFields() + rtx.Must(err, "failed to marshal latency1 schema") + err = os.WriteFile(latency1Schema, b, 0o644) + rtx.Must(err, "failed to write latency1 schema") } From 0694eb14aef868bd55891cd632f74d5ec5054b0f Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 9 Aug 2023 00:37:15 +0200 Subject: [PATCH 17/37] Set version via ldflags at compile time (#24) * Set version via ldflags at compile time. * Fix versionflags * Set default value of version.Version to "unspecified". --- build.sh | 8 +++++++- internal/handler/handler.go | 3 ++- pkg/latency1/model/result.go | 3 ++- pkg/version/version.go | 5 +++++ 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 pkg/version/version.go diff --git a/build.sh b/build.sh index 98d1297..389f31b 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,12 @@ #!/bin/sh +# Script to build msak with the correct flags. +set -ex + +VERSION=$(git describe --tags) +versionflags="-X github.com/m-lab/msak/pkg/version.Version=$VERSION" + COMMIT=$(git log -1 --format=%h) -versionflags="-X github.com/m-lab/go/prometheusx.GitShortCommit=${COMMIT}" +versionflags="${versionflags} -X github.com/m-lab/go/prometheusx.GitShortCommit=${COMMIT}" go build -v \ -tags netgo \ diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1fbd5fc..65e10ae 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -16,6 +16,7 @@ import ( "github.com/m-lab/msak/internal/persistence" "github.com/m-lab/msak/pkg/throughput1" "github.com/m-lab/msak/pkg/throughput1/model" + "github.com/m-lab/msak/pkg/version" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -183,7 +184,7 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res Client: wsConn.UnderlyingConn().RemoteAddr().String(), Direction: string(kind), GitShortCommit: prometheusx.GitShortCommit, - Version: "v0.0.1", + Version: version.Version, ClientMetadata: metadata, ClientOptions: clientOptions, } diff --git a/pkg/latency1/model/result.go b/pkg/latency1/model/result.go index 4c05208..c4ff02e 100644 --- a/pkg/latency1/model/result.go +++ b/pkg/latency1/model/result.go @@ -6,6 +6,7 @@ import ( "time" "github.com/m-lab/go/prometheusx" + "github.com/m-lab/msak/pkg/version" ) // LatencyPacket is the payload of a latency measurement UDP packet. @@ -151,7 +152,7 @@ func (s *Session) Archive() *ArchivalData { return &ArchivalData{ ID: s.UUID, GitShortCommit: prometheusx.GitShortCommit, - Version: "TODO", + Version: version.Version, Client: s.Client, Server: s.Server, StartTime: s.StartTime, diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 0000000..24f901d --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,5 @@ +package version + +// Version is the version of msak. This is meant to be overridden at compile +// time using `-ldflags "-X var=value` +var Version = "unspecified" From e25fe060092648a02527fd6d1e6c7ccc66f9bc49 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 28 Sep 2023 00:47:22 +0200 Subject: [PATCH 18/37] Replace *net.TCPConn with a TCPLikeConn abstraction. --- internal/handler/handler_test.go | 2 +- internal/netx/conn.go | 12 ++++++++++-- internal/netx/conn_linux.go | 3 +-- pkg/throughput1/protocol_test.go | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index a62dbe3..60a9ea6 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -42,7 +42,7 @@ func setupTestWSDialer(u *url.URL) *websocket.Dialer { if err != nil { return nil, err } - return netx.FromTCPConn(conn.(*net.TCPConn)) + return netx.FromTCPLikeConn(conn.(*net.TCPConn)) }, } } diff --git a/internal/netx/conn.go b/internal/netx/conn.go index cacba13..fb46fed 100644 --- a/internal/netx/conn.go +++ b/internal/netx/conn.go @@ -33,6 +33,13 @@ type ConnInfo interface { SaveUUID(context.Context) context.Context } +// TCPLikeConn is a net.Conn with a File() method. This is useful for creating a +// netx.Conn based on a custom TCPConn-like type - e.g. for testing. +type TCPLikeConn interface { + net.Conn + File() (*os.File, error) +} + // ToConnInfo is a helper function to convert a net.Conn into a netx.ConnInfo. // It panics if netConn does not contain a type supporting ConnInfo. func ToConnInfo(netConn net.Conn) ConnInfo { @@ -57,8 +64,9 @@ type Conn struct { bytesWritten atomic.Uint64 } -func FromTCPConn(tcpConn *net.TCPConn) (*Conn, error) { - return fromTCPConn(tcpConn) +// FromTCPLikeConn creates a netx.Conn from a TCPLikeConn. +func FromTCPLikeConn(tcpConn TCPLikeConn) (*Conn, error) { + return fromTCPLikeConn(tcpConn) } // Read reads from the underlying net.Conn and updates the read bytes counter. diff --git a/internal/netx/conn_linux.go b/internal/netx/conn_linux.go index 8650adc..babc8bb 100644 --- a/internal/netx/conn_linux.go +++ b/internal/netx/conn_linux.go @@ -1,11 +1,10 @@ package netx import ( - "net" "time" ) -func fromTCPConn(tcpConn *net.TCPConn) (*Conn, error) { +func fromTCPLikeConn(tcpConn TCPLikeConn) (*Conn, error) { // On Linux system, this can only fail when the file duplication fails. fp, err := tcpConn.File() if err != nil { diff --git a/pkg/throughput1/protocol_test.go b/pkg/throughput1/protocol_test.go index c3fbbbe..6180fca 100644 --- a/pkg/throughput1/protocol_test.go +++ b/pkg/throughput1/protocol_test.go @@ -106,7 +106,7 @@ func TestProtocol_Download(t *testing.T) { if err != nil { return nil, err } - return netx.FromTCPConn(conn.(*net.TCPConn)) + return netx.FromTCPLikeConn(conn.(*net.TCPConn)) }, } From 6b4f1fc15143d419c7526acb055fe2be76cbb08b Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 28 Sep 2023 10:31:54 +0200 Subject: [PATCH 19/37] Rename to fromTCPLikeConn in _stub.go files, too. --- internal/netx/conn_stub.go | 3 +-- internal/netx/listener_stub.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/netx/conn_stub.go b/internal/netx/conn_stub.go index 5fd4e4f..7d19e20 100644 --- a/internal/netx/conn_stub.go +++ b/internal/netx/conn_stub.go @@ -4,11 +4,10 @@ package netx import ( - "net" "time" ) -func fromTCPConn(tcpConn *net.TCPConn) (*Conn, error) { +func fromTCPLikeConn(tcpConn TCPLikeConn) (*Conn, error) { // On non-Linux systems, TCPInfo/BBRInfo aren't supported, the file pointer // is not needed. return &Conn{ diff --git a/internal/netx/listener_stub.go b/internal/netx/listener_stub.go index e7b8286..690b781 100644 --- a/internal/netx/listener_stub.go +++ b/internal/netx/listener_stub.go @@ -11,5 +11,5 @@ func (ln *Listener) accept() (net.Conn, error) { return nil, err } - return fromTCPConn(tc) + return fromTCPLikeConn(tc) } From 21baee2f9351ddb84318414b951169c9e5513ecd Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Fri, 29 Sep 2023 23:42:27 +0200 Subject: [PATCH 20/37] Add throughput1 client library (#26) * Add handler * Save archival data from deferred func * Add main, fix handler * Add script to generate local test certs * Add build script and Dockerfile. * Update go.mod/go.sum * Add basic client for testing * Swap logger * Swap logger in measurer, too * Remove files that don't belong to this PR * Remove more files that don't belong here * go mod tidy * Change metadata length limits * Add access_token to the knownOptions map. * Add partial tests and prom metrics * Duration should be milliseconds * Increase test coverage * Update go.mod * Update go.mod go.sum * Add client code * Merge branch 'main' into sandbox-roberto-client * Rename ndt8 -> throughput1 * Separate network-level and application-level measurements. This commit adds application-level measurements and namespaces to better separate the two kind of measurements in the Measurement object. It is a breaking change and will need a schema update. * Fix client * Merge branch 'main' into sandbox-roberto-client * Merge branch 'add-application-byte-counters' into sandbox-roberto-client * Merge branch 'temp-test' into sandbox-roberto-client * Make output more detailed, add server-side messages * Make application-level byte counters atomic. * Merge branch 'add-application-byte-counters' into sandbox-roberto-client * Merge branch 'main' into sandbox-roberto-client * Add debug flag and simplify code * Add -debug flag, docstrings, and call Upload() in main * Remove unused fields * Complete rename from NDT8 to Throughput1 * Use the same version for libraryVersion and clientVersion (and set it to version.Version, overridden by build flags) * Rename client to msak-client-go * Add docstrings and use the right timeout in connect() call. * Refactor code to extract runStream(). * Add docstrings. * Fix race condition. * Refactor code, only output every 100ms * Remove extra \n from OnDebug messages * Address code review comments - Make client name const - Use client package consts for flags' default values - Use log.Fatal * Update defaults, add comments * Include * Add DefaultScheme const and more docstrings * Replace fmt.Print with Emitter.OnDebug * Panic on invalid subtests * Remove ServiceURL * Move client config to a dedicated struct * Rename client.ClientConfig to client.Config * More refactoring and more comments/docstrings. * Add TLSClientConfig to defaultDialer. * Add docstrings in Emitter interface. * Merge branch 'main' into sandbox-roberto-client * FromTCPConn -> FromTCPLikeConn * Panic on empty clientName/Version in client.New() * Add comments in nextURLFromLocate and remove extra printf --- cmd/msak-client/client.go | 56 ++++++ pkg/client/client.go | 362 ++++++++++++++++++++++++++++++++++++++ pkg/client/client_test.go | 117 ++++++++++++ pkg/client/config.go | 38 ++++ pkg/client/emitter.go | 73 ++++++++ 5 files changed, 646 insertions(+) create mode 100644 cmd/msak-client/client.go create mode 100644 pkg/client/client.go create mode 100644 pkg/client/client_test.go create mode 100644 pkg/client/config.go create mode 100644 pkg/client/emitter.go diff --git a/cmd/msak-client/client.go b/cmd/msak-client/client.go new file mode 100644 index 0000000..422bdc2 --- /dev/null +++ b/cmd/msak-client/client.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "flag" + "log" + + "github.com/google/uuid" + "github.com/m-lab/msak/pkg/client" + "github.com/m-lab/msak/pkg/version" +) + +const clientName = "msak-client-go" + +var clientVersion = version.Version + +var ( + flagServer = flag.String("server", "", "Server address") + flagStreams = flag.Int("streams", client.DefaultStreams, "Number of streams") + flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") + flagDelay = flag.Duration("delay", 0, "Delay between each stream") + flagDuration = flag.Duration("duration", client.DefaultLength, "Length of the last stream") + flagScheme = flag.String("scheme", client.DefaultScheme, "Websocket scheme (wss or ws)") + flagMID = flag.String("mid", uuid.NewString(), "Measurement ID to use") + flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") + flagDebug = flag.Bool("debug", false, "Enable debug logging") +) + +func main() { + flag.Parse() + + // For a given number of streams, there will be streams-1 delays. This makes + // sure that all the streams can at least start with the current configuration. + if float64(*flagStreams-1)*flagDelay.Seconds() >= flagDuration.Seconds() { + log.Fatal("Invalid configuration: please check streams, delay and duration and make sure they make sense.") + } + + config := client.Config{ + Server: *flagServer, + Scheme: *flagScheme, + NumStreams: *flagStreams, + CongestionControl: *flagCC, + Delay: *flagDelay, + Length: *flagDuration, + MeasurementID: *flagMID, + Emitter: client.HumanReadable{ + Debug: *flagDebug, + }, + NoVerify: *flagNoVerify, + } + + cl := client.New(clientName, clientVersion, config) + + cl.Download(context.Background()) + cl.Upload(context.Background()) +} diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..b29df56 --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,362 @@ +package client + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "log" + "net" + "net/http" + "net/url" + "runtime" + "sync" + "time" + + "github.com/gorilla/websocket" + "github.com/m-lab/locate/api/locate" + v2 "github.com/m-lab/locate/api/v2" + "github.com/m-lab/msak/internal/netx" + "github.com/m-lab/msak/pkg/throughput1" + "github.com/m-lab/msak/pkg/throughput1/model" + "github.com/m-lab/msak/pkg/throughput1/spec" + "github.com/m-lab/msak/pkg/version" +) + +const ( + // DefaultWebSocketHandshakeTimeout is the default timeout used by the client + // for the WebSocket handshake. + DefaultWebSocketHandshakeTimeout = 5 * time.Second + + // DefaultStreams is the default number of streams for a new client. + DefaultStreams = 3 + + // DefaultLength is the default test duration for a new client. + DefaultLength = 5 * time.Second + + // DefaultScheme is the default WebSocket scheme for a new Client. + DefaultScheme = "wss" + + libraryName = "msak-client" +) + +var ( + // ErrNoTargets is returned if all Locate targets have been tried. + ErrNoTargets = errors.New("no targets available") + + libraryVersion = version.Version +) + +// defaultDialer is the default websocket.Dialer used by the client. +// Its NetDial function wraps the net.Conn with a netx.Conn. +var defaultDialer = &websocket.Dialer{ + HandshakeTimeout: DefaultWebSocketHandshakeTimeout, + NetDial: func(network, addr string) (net.Conn, error) { + conn, err := net.Dial(network, addr) + if err != nil { + return nil, err + } + return netx.FromTCPLikeConn(conn.(*net.TCPConn)) + }, + TLSClientConfig: &tls.Config{}, +} + +// Locator is an interface used to get a list of available servers to test against. +type Locator interface { + Nearest(ctx context.Context, service string) ([]v2.Target, error) +} + +// Throughput1Client is a client for the throughput1 protocol. +type Throughput1Client struct { + // ClientName is the name of the client sent to the server as part of the user-agent. + ClientName string + // ClientVersion is the version of the client sent to the server as part of the user-agent. + ClientVersion string + + config Config + + dialer *websocket.Dialer + locator Locator + + // targets and tIndex cache the results from the Locate API. + targets []v2.Target + tIndex map[string]int + + // recvByteCounters is a map of stream IDs to number of bytes, used to compute the goodput. + // A new byte count is appended every time the client sees a receiver-side Measurement. + recvByteCounters map[int][]int64 + recvByteCountersMutex sync.Mutex +} + +// Result contains the aggregate metrics collected during the test. +type Result struct { + // Goodput is the average number of application-level bits per second that + // have been transferred so far across all the streams. + Goodput float64 + // Throughput is the average number of network-level bits per second that + // have been transferred so far across all the streams. + Throughput float64 + // Elapsed is the total time elapsed since the test started. + Elapsed time.Duration + // MinRTT is the minimum of MinRTT values observed across all the streams. + MinRTT uint32 +} + +// makeUserAgent creates the user agent string. +func makeUserAgent(clientName, clientVersion string) string { + return clientName + "/" + clientVersion + " " + libraryName + "/" + libraryVersion +} + +// New returns a new Throughput1Client with the provided client name, version and config. +// It panics if clientName or clientVersion are empty. +func New(clientName, clientVersion string, config Config) *Throughput1Client { + if clientName == "" || clientVersion == "" { + panic("client name and version must be non-empty") + } + defaultDialer.TLSClientConfig.InsecureSkipVerify = config.NoVerify + return &Throughput1Client{ + ClientName: clientName, + ClientVersion: clientVersion, + + config: config, + dialer: defaultDialer, + + locator: locate.NewClient(makeUserAgent(clientName, clientVersion)), + + tIndex: map[string]int{}, + recvByteCounters: map[int][]int64{}, + } +} + +func (c *Throughput1Client) connect(ctx context.Context, serviceURL *url.URL) (*websocket.Conn, error) { + q := serviceURL.Query() + q.Set("streams", fmt.Sprint(c.config.NumStreams)) + q.Set("cc", c.config.CongestionControl) + q.Set("duration", fmt.Sprintf("%d", c.config.Length.Milliseconds())) + q.Set("client_arch", runtime.GOARCH) + q.Set("client_library_name", libraryName) + q.Set("client_library_version", libraryVersion) + q.Set("client_os", runtime.GOOS) + q.Set("client_name", c.ClientName) + q.Set("client_version", c.ClientVersion) + serviceURL.RawQuery = q.Encode() + headers := http.Header{} + headers.Add("Sec-WebSocket-Protocol", spec.SecWebSocketProtocol) + headers.Add("User-Agent", makeUserAgent(c.ClientName, c.ClientVersion)) + conn, _, err := c.dialer.DialContext(ctx, serviceURL.String(), headers) + return conn, err +} + +// nextURLFromLocate returns the next URL to try from the Locate API. +// If it's the first time we're calling this function, it contacts the Locate +// API. Subsequently, it returns the next URL from the cache. +// If there are no more URLs to try, it returns an error. +func (c *Throughput1Client) nextURLFromLocate(ctx context.Context, p string) (string, error) { + if len(c.targets) == 0 { + targets, err := c.locator.Nearest(ctx, "msak/throughput1") + if err != nil { + return "", err + } + // cache targets on success. + c.targets = targets + } + // Returns the next URL from the cache. + // The index to access the next URL (tIndex[k]) is per-path rather than global. + k := c.config.Scheme + "://" + p + if c.tIndex[k] < len(c.targets) { + r := c.targets[c.tIndex[k]].URLs[k] + c.tIndex[k]++ + return r, nil + } + return "", ErrNoTargets +} + +func (c *Throughput1Client) start(ctx context.Context, subtest spec.SubtestKind) error { + // Find the URL to use for this measurement. + var mURL *url.URL + // If the server has been provided, use it and use default paths based on + // the subtest kind (download/upload). + if c.config.Server != "" { + c.config.Emitter.OnDebug(fmt.Sprintf("using server provided via flags %s", c.config.Server)) + path := getPathForSubtest(subtest) + mURL = &url.URL{ + Scheme: c.config.Scheme, + Host: c.config.Server, + Path: path, + } + q := mURL.Query() + q.Set("mid", c.config.MeasurementID) + mURL.RawQuery = q.Encode() + } + + // If no server has been provided, use the Locate API. + if mURL == nil { + c.config.Emitter.OnDebug("using locate") + urlStr, err := c.nextURLFromLocate(ctx, getPathForSubtest(subtest)) + if err != nil { + return err + } + mURL, err = url.Parse(urlStr) + if err != nil { + return err + } + log.Print("URL: ", mURL.String()) + } + + wg := &sync.WaitGroup{} + globalTimeout, cancel := context.WithTimeout(ctx, c.config.Length) + defer cancel() + + // Reset the counters. + c.recvByteCounters = map[int][]int64{} + globalStartTime := time.Now() + + go func() { + t := time.NewTicker(100 * time.Millisecond) + // Print goodput every 100ms. Stop when the context is cancelled. + for { + select { + case <-globalTimeout.Done(): + return + case <-t.C: + c.emitResult(globalStartTime) + } + } + }() + + // Main client loop. Spawns one goroutine per stream. + for i := 0; i < c.config.NumStreams; i++ { + streamID := i + wg.Add(1) + + go func() { + defer wg.Done() + + // Run a single stream. + err := c.runStream(globalTimeout, streamID, mURL, subtest, globalStartTime) + if err != nil { + c.config.Emitter.OnError(err) + } + }() + + time.Sleep(c.config.Delay) + } + + wg.Wait() + + return nil +} + +func (c *Throughput1Client) runStream(ctx context.Context, streamID int, mURL *url.URL, + subtest spec.SubtestKind, globalStartTime time.Time) error { + + measurements := make(chan model.WireMeasurement) + + c.config.Emitter.OnStart(mURL.Host, subtest) + conn, err := c.connect(ctx, mURL) + if err != nil { + c.config.Emitter.OnError(err) + close(measurements) + return err + } + c.config.Emitter.OnConnect(mURL.String()) + + proto := throughput1.New(conn) + + var clientCh, serverCh <-chan model.WireMeasurement + var errCh <-chan error + switch subtest { + case spec.SubtestDownload: + clientCh, serverCh, errCh = proto.ReceiverLoop(ctx) + case spec.SubtestUpload: + clientCh, serverCh, errCh = proto.SenderLoop(ctx) + } + + for { + select { + case <-ctx.Done(): + c.config.Emitter.OnComplete(streamID, mURL.Host) + return nil + case m := <-clientCh: + // If subtest is download, store the client-side measurement. + if subtest != spec.SubtestDownload { + continue + } + c.config.Emitter.OnMeasurement(streamID, m) + c.config.Emitter.OnDebug(fmt.Sprintf("Stream #%d - application r/w: %d/%d, network r/w: %d/%d", + streamID, m.Application.BytesReceived, m.Application.BytesSent, + m.Network.BytesReceived, m.Network.BytesSent)) + c.storeMeasurement(streamID, m) + case m := <-serverCh: + // If subtest is upload, store the server-side measurement. + if subtest != spec.SubtestUpload { + continue + } + c.config.Emitter.OnMeasurement(streamID, m) + c.config.Emitter.OnDebug(fmt.Sprintf("#%d - application r/w: %d/%d, network r/w: %d/%d", + streamID, m.Application.BytesReceived, m.Application.BytesSent, + m.Network.BytesReceived, m.Network.BytesSent)) + c.storeMeasurement(streamID, m) + case err := <-errCh: + return err + } + } +} + +func (c *Throughput1Client) storeMeasurement(streamID int, m model.WireMeasurement) { + // Append the value of the Application.BytesReceived counter to the corresponding recvByteCounters map entry. + c.recvByteCountersMutex.Lock() + c.recvByteCounters[streamID] = append(c.recvByteCounters[streamID], m.Application.BytesReceived) + c.recvByteCountersMutex.Unlock() +} + +// applicationBytes returns the aggregate application-level bytes transferred by all the streams. +func (c *Throughput1Client) applicationBytes() int64 { + var sum int64 + c.recvByteCountersMutex.Lock() + for _, bytes := range c.recvByteCounters { + sum += bytes[len(bytes)-1] + } + c.recvByteCountersMutex.Unlock() + return sum +} + +// emitResult emits the result of the current measurement via the configured Emitter. +func (c *Throughput1Client) emitResult(start time.Time) { + applicationBytes := c.applicationBytes() + elapsed := time.Since(start) + goodput := float64(applicationBytes) / float64(elapsed.Seconds()) * 8 // bps + result := Result{ + Elapsed: elapsed, + Goodput: goodput, + Throughput: 0, // TODO + } + c.config.Emitter.OnResult(result) +} + +// Download runs a download test using the settings configured for this client. +func (c *Throughput1Client) Download(ctx context.Context) { + err := c.start(ctx, spec.SubtestDownload) + if err != nil { + log.Println(err) + } +} + +// Upload runs an upload test using the settings configured for this client. +func (c *Throughput1Client) Upload(ctx context.Context) { + err := c.start(ctx, spec.SubtestUpload) + if err != nil { + log.Println(err) + } +} + +func getPathForSubtest(subtest spec.SubtestKind) string { + switch subtest { + case spec.SubtestDownload: + return spec.DownloadPath + case spec.SubtestUpload: + return spec.UploadPath + default: + panic(fmt.Sprintf("invalid subtest: %s", subtest)) + } +} diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go new file mode 100644 index 0000000..fe1e536 --- /dev/null +++ b/pkg/client/client_test.go @@ -0,0 +1,117 @@ +package client + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "runtime" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/m-lab/go/testingx" + "github.com/m-lab/msak/pkg/throughput1/spec" +) + +func TestNew(t *testing.T) { + t.Run("new clients have the expected name and version", func(t *testing.T) { + c := New("test", "v1.0.0", Config{}) + if c.ClientName != "test" || c.ClientVersion != "v1.0.0" { + t.Errorf("client.New() returned client with wrong name/version") + } + }) + t.Run("new clients must have non-empty name and version", func(t *testing.T) { + // Test that New panics when passed empty strings. + defer func() { + if r := recover(); r == nil { + t.Errorf("client.New() did not panic when passed empty strings") + } + }() + + New("", "", Config{}) + }) +} + +func Test_makeUserAgent(t *testing.T) { + t.Run("generate requested user agent", func(t *testing.T) { + got := makeUserAgent("clientname", "clientversion") + expected := fmt.Sprintf("%s/%s %s/%s", "clientname", "clientversion", + libraryName, libraryVersion) + if got != expected { + t.Errorf("makeUserAgent() = %s, want %s", got, expected) + } + }) +} + +func setupTestServer(handler http.Handler) *httptest.Server { + return httptest.NewServer(handler) +} + +func TestNDT8Client_connect(t *testing.T) { + + c := New("test", "version", Config{ + NumStreams: 3, + CongestionControl: "cubic", + Length: 5 * time.Second, + }) + + t.Run("connect sends qs parameters and headers", func(t *testing.T) { + upgrader := websocket.Upgrader{} + + // Set up a test server with a handler that verifies querystring parameters + // and headers. + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wsConn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer wsConn.Close() + + // Check querystring parameters. + expected := map[string]string{ + "streams": "3", + "cc": "cubic", + "duration": fmt.Sprintf("%d", c.config.Length.Milliseconds()), + "client_arch": runtime.GOARCH, + "client_library_name": libraryName, + "client_library_version": libraryVersion, + "client_os": runtime.GOOS, + "client_name": c.ClientName, + "client_version": c.ClientVersion, + } + for k, v := range expected { + if got := r.URL.Query().Get(k); got != v { + t.Errorf("expected qs parameter %s = %s, got %s", k, v, got) + } + } + + // Check headers + expected = map[string]string{ + "Sec-WebSocket-Protocol": spec.SecWebSocketProtocol, + "User-Agent": makeUserAgent(c.ClientName, c.ClientVersion), + } + for k, v := range expected { + if got := r.Header.Get(k); got != v { + t.Errorf("expected header %s = %s, got %s", k, v, got) + } + } + }) + + s := setupTestServer(handler) + defer s.Close() + + urlStr := "ws" + strings.TrimPrefix(s.URL, "http") + + u, err := url.Parse(urlStr) + testingx.Must(t, err, "cannot parse server URL") + + _, err = c.connect(context.Background(), u) + if err != nil { + t.Errorf("NDT8Client.connect() error: %v", err) + return + } + }) +} diff --git a/pkg/client/config.go b/pkg/client/config.go new file mode 100644 index 0000000..5a89ac2 --- /dev/null +++ b/pkg/client/config.go @@ -0,0 +1,38 @@ +package client + +import ( + "time" +) + +// Config is the configuration for a Client. +type Config struct { + // Server is the server to connect to. If empty, the server is obtained by + // querying the configured Locator. + Server string + + // Scheme is the WebSocket scheme used to connect to the server (ws or wss). + Scheme string + + // NumStreams is the number of streams that will be spawned by this client to run a + // download or an upload test. + NumStreams int + + // Length is the duration of the test. + Length time.Duration + + // Delay is the delay between each stream. + Delay time.Duration + + // CongestionControl is the congestion control algorithm to request from the server. + CongestionControl string + + // MeasurementID is the manually configured Measurement ID ("mid") to pass to the server. + MeasurementID string + + // Emitter is the interface used to emit the results of the test. It can be overridden + // to provide a custom output. + Emitter Emitter + + // NoVerify disables the TLS certificate verification. + NoVerify bool +} diff --git a/pkg/client/emitter.go b/pkg/client/emitter.go new file mode 100644 index 0000000..633b073 --- /dev/null +++ b/pkg/client/emitter.go @@ -0,0 +1,73 @@ +package client + +import ( + "fmt" + + "github.com/m-lab/msak/pkg/throughput1/model" + "github.com/m-lab/msak/pkg/throughput1/spec" +) + +// Emitter is an interface for emitting results. +type Emitter interface { + // OnStart is called when a stream starts. + OnStart(server string, kind spec.SubtestKind) + // OnConnect is called when the WebSocket connection is established. + OnConnect(server string) + // OnMeasurement is called on received Measurement objects. + OnMeasurement(id int, m model.WireMeasurement) + // OnResult is called when the aggregate result is ready. + OnResult(Result) + // OnError is called on errors. + OnError(err error) + // OnComplete is called after a stream completes. + OnComplete(streamID int, server string) + // OnDebug is called to print debug information. + OnDebug(msg string) +} + +// HumanReadable prints human-readable output to stdout. +// It can be configured to include debug output, too. +type HumanReadable struct { + Debug bool +} + +// OnResult prints the aggregate result. +func (HumanReadable) OnResult(r Result) { + fmt.Printf("Elapsed: %.2fs, Goodput: %f Mb/s, MinRTT: %d\n", r.Elapsed.Seconds(), + r.Goodput/1024/1024, r.MinRTT) +} + +// OnStart is called when the stream starts and prints the subtest and server hostname. +func (HumanReadable) OnStart(server string, kind spec.SubtestKind) { + fmt.Printf("Starting %s stream (server: %s)\n", kind, server) +} + +// OnConnect is called when the connection to the server is established. +func (HumanReadable) OnConnect(server string) { + fmt.Printf("Connected to %s\n", server) +} + +// OnMeasurement is called on received Measurement objects. +func (HumanReadable) OnMeasurement(id int, m model.WireMeasurement) { + // NOTHING - don't print individual measurement objects in this Emitter. +} + +// OnError is called on errors. +func (HumanReadable) OnError(err error) { + fmt.Println(err) +} + +// OnComplete is called after a stream completes. +func (HumanReadable) OnComplete(streamID int, server string) { + fmt.Printf("Stream %d complete (server %s)\n", streamID, server) +} + +// OnDebug is called to print debug information. +func (e HumanReadable) OnDebug(msg string) { + if e.Debug { + fmt.Printf("DEBUG: %s\n", msg) + } +} + +// Checks that HumanReadable implements Emitter. +var _ Emitter = &HumanReadable{} From 3efa63b61cddd9b6b24c9c3279a65a8c4a4000c8 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 6 Dec 2023 16:59:09 +0100 Subject: [PATCH 21/37] Set the global start time when the first connection is established (#28) * Set the global start time when the first websocket is connected. * Refactor code for testability + fix test timeout. The test timeout now starts when the first connection is established. --- pkg/client/client.go | 75 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index b29df56..22585fb 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -86,6 +86,10 @@ type Throughput1Client struct { // A new byte count is appended every time the client sees a receiver-side Measurement. recvByteCounters map[int][]int64 recvByteCountersMutex sync.Mutex + + // sharedStartTime is the time at which the test started, shared across all streams. + // It is set when the first streams connects to the server and used to compute the elapsed time. + sharedStartTime time.Time } // Result contains the aggregate metrics collected during the test. @@ -204,24 +208,28 @@ func (c *Throughput1Client) start(ctx context.Context, subtest spec.SubtestKind) } wg := &sync.WaitGroup{} - globalTimeout, cancel := context.WithTimeout(ctx, c.config.Length) - defer cancel() // Reset the counters. c.recvByteCounters = map[int][]int64{} - globalStartTime := time.Now() + + startTimeCh := make(chan time.Time, 1) + + testCtx, cancelTest := context.WithCancel(ctx) + defer cancelTest() go func() { - t := time.NewTicker(100 * time.Millisecond) - // Print goodput every 100ms. Stop when the context is cancelled. - for { - select { - case <-globalTimeout.Done(): - return - case <-t.C: - c.emitResult(globalStartTime) - } + // Wait for the start signal to come from any of the streams. + // Returns early if the context is cancelled. + started := c.waitStart(testCtx, startTimeCh) + if !started { + return } + + // Once at least one of the streams has started, start a timer to cancel + // the context after the configured test duration. + time.AfterFunc(c.config.Length, cancelTest) + + c.emitLoop(testCtx) }() // Main client loop. Spawns one goroutine per stream. @@ -233,7 +241,7 @@ func (c *Throughput1Client) start(ctx context.Context, subtest spec.SubtestKind) defer wg.Done() // Run a single stream. - err := c.runStream(globalTimeout, streamID, mURL, subtest, globalStartTime) + err := c.runStream(testCtx, streamID, mURL, subtest, startTimeCh) if err != nil { c.config.Emitter.OnError(err) } @@ -247,8 +255,33 @@ func (c *Throughput1Client) start(ctx context.Context, subtest spec.SubtestKind) return nil } +func (c *Throughput1Client) waitStart(ctx context.Context, startTimeCh chan time.Time) bool { + select { + case startTime := <-startTimeCh: + c.sharedStartTime = startTime + case <-ctx.Done(): + return false + } + + return true +} + +// emitLoop emits the results every 100ms once a . It stops when the context is cancelled. +func (c *Throughput1Client) emitLoop(ctx context.Context) { + t := time.NewTicker(100 * time.Millisecond) + + for { + select { + case <-ctx.Done(): + return + case <-t.C: + c.emitResult(c.sharedStartTime) + } + } +} + func (c *Throughput1Client) runStream(ctx context.Context, streamID int, mURL *url.URL, - subtest spec.SubtestKind, globalStartTime time.Time) error { + subtest spec.SubtestKind, startTimeCh chan time.Time) error { measurements := make(chan model.WireMeasurement) @@ -259,6 +292,16 @@ func (c *Throughput1Client) runStream(ctx context.Context, streamID int, mURL *u close(measurements) return err } + defer conn.Close() + + // Send the start time to the channel. This is a non-blocking send since the + // receiver only reads one value + select { + case startTimeCh <- time.Now(): + default: + // NOTHING + } + c.config.Emitter.OnConnect(mURL.String()) proto := throughput1.New(conn) @@ -315,7 +358,9 @@ func (c *Throughput1Client) applicationBytes() int64 { var sum int64 c.recvByteCountersMutex.Lock() for _, bytes := range c.recvByteCounters { - sum += bytes[len(bytes)-1] + if len(bytes) > 0 { + sum += bytes[len(bytes)-1] + } } c.recvByteCountersMutex.Unlock() return sum From 271633e3e58947c2676649a935b375c373b47d82 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 3 Jan 2024 16:19:51 +0100 Subject: [PATCH 22/37] Add support for bytes limit (#29) * Add support for byte limits. The "bytes" querystring parameter allows the client to specify the maximum number of bytes the server will send/receive before terminating the connection. * s/byte/bytes/g * Comments * Apply limits over sent bytes Co-Authored-By: Cristina Leon --- cmd/msak-client/client.go | 22 ++++++++------- internal/handler/handler.go | 16 +++++++++++ internal/handler/handler_test.go | 5 ++++ pkg/client/client.go | 1 + pkg/client/config.go | 4 +++ pkg/throughput1/protocol.go | 41 ++++++++++++++++++++++----- pkg/throughput1/protocol_test.go | 48 ++++++++++++++++++++++++++++++++ pkg/throughput1/spec/spec.go | 5 ++++ 8 files changed, 125 insertions(+), 17 deletions(-) diff --git a/cmd/msak-client/client.go b/cmd/msak-client/client.go index 422bdc2..cc5f4e4 100644 --- a/cmd/msak-client/client.go +++ b/cmd/msak-client/client.go @@ -15,15 +15,16 @@ const clientName = "msak-client-go" var clientVersion = version.Version var ( - flagServer = flag.String("server", "", "Server address") - flagStreams = flag.Int("streams", client.DefaultStreams, "Number of streams") - flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") - flagDelay = flag.Duration("delay", 0, "Delay between each stream") - flagDuration = flag.Duration("duration", client.DefaultLength, "Length of the last stream") - flagScheme = flag.String("scheme", client.DefaultScheme, "Websocket scheme (wss or ws)") - flagMID = flag.String("mid", uuid.NewString(), "Measurement ID to use") - flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") - flagDebug = flag.Bool("debug", false, "Enable debug logging") + flagServer = flag.String("server", "", "Server address") + flagStreams = flag.Int("streams", client.DefaultStreams, "Number of streams") + flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") + flagDelay = flag.Duration("delay", 0, "Delay between each stream") + flagDuration = flag.Duration("duration", client.DefaultLength, "Length of the last stream") + flagScheme = flag.String("scheme", client.DefaultScheme, "Websocket scheme (wss or ws)") + flagMID = flag.String("mid", uuid.NewString(), "Measurement ID to use") + flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") + flagDebug = flag.Bool("debug", false, "Enable debug logging") + flagByteLimit = flag.Int("bytes", 0, "Byte limit to request to the server") ) func main() { @@ -46,7 +47,8 @@ func main() { Emitter: client.HumanReadable{ Debug: *flagDebug, }, - NoVerify: *flagNoVerify, + NoVerify: *flagNoVerify, + ByteLimit: *flagByteLimit, } cl := client.New(clientName, clientVersion, config) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 65e10ae..b90bde3 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -16,6 +16,7 @@ import ( "github.com/m-lab/msak/internal/persistence" "github.com/m-lab/msak/pkg/throughput1" "github.com/m-lab/msak/pkg/throughput1/model" + "github.com/m-lab/msak/pkg/throughput1/spec" "github.com/m-lab/msak/pkg/version" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -130,6 +131,20 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res model.NameValue{Name: "delay", Value: requestDelay}) } + requestByteLimit := query.Get(spec.ByteLimitParameterName) + var byteLimit int + if requestByteLimit != "" { + if byteLimit, err = strconv.Atoi(requestByteLimit); err != nil { + ClientConnections.WithLabelValues(string(kind), "invalid-byte-limit").Inc() + log.Info("Received request with an invalid byte limit", "source", req.RemoteAddr, + "value", requestByteLimit) + writeBadRequest(rw) + return + } + clientOptions = append(clientOptions, + model.NameValue{Name: spec.ByteLimitParameterName, Value: requestByteLimit}) + } + // Read metadata (i.e. everything in the querystring that's not a known // option). metadata, err := getRequestMetadata(req) @@ -198,6 +213,7 @@ func (h *Handler) upgradeAndRunMeasurement(kind model.TestDirection, rw http.Res defer cancel() proto := throughput1.New(wsConn) + proto.SetByteLimit(byteLimit) var senderCh, receiverCh <-chan model.WireMeasurement var errCh <-chan error if kind == model.DirectionDownload { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 60a9ea6..f0ae4c0 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -215,6 +215,11 @@ func TestHandler_Validation(t *testing.T) { target: "/?mid=test&streams=2&duration=invalid", statusCode: http.StatusBadRequest, }, + { + name: "invalid byte limit", + target: "/?mid=test&streams=2&duration=1000&bytes=invalid", + statusCode: http.StatusBadRequest, + }, { name: "metadata key too long", target: "/?mid=test&streams=2&" + longKey, diff --git a/pkg/client/client.go b/pkg/client/client.go index 22585fb..edb8be5 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -136,6 +136,7 @@ func (c *Throughput1Client) connect(ctx context.Context, serviceURL *url.URL) (* q := serviceURL.Query() q.Set("streams", fmt.Sprint(c.config.NumStreams)) q.Set("cc", c.config.CongestionControl) + q.Set(spec.ByteLimitParameterName, fmt.Sprint(c.config.ByteLimit)) q.Set("duration", fmt.Sprintf("%d", c.config.Length.Milliseconds())) q.Set("client_arch", runtime.GOARCH) q.Set("client_library_name", libraryName) diff --git a/pkg/client/config.go b/pkg/client/config.go index 5a89ac2..e55416c 100644 --- a/pkg/client/config.go +++ b/pkg/client/config.go @@ -35,4 +35,8 @@ type Config struct { // NoVerify disables the TLS certificate verification. NoVerify bool + + // ByteLimit is the maximum number of bytes to download or upload. If set to 0, the + // limit is disabled. + ByteLimit int } diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index 1800819..43df87f 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -47,6 +47,8 @@ type Protocol struct { applicationBytesReceived atomic.Int64 applicationBytesSent atomic.Int64 + + byteLimit int } // New returns a new Protocol with the specified connection and every other @@ -61,6 +63,12 @@ func New(conn *websocket.Conn) *Protocol { } } +// SetByteLimit sets the number of bytes sent after which a test (either download or upload) will stop. +// Set the value to zero to disable the byte limit. +func (p *Protocol) SetByteLimit(value int) { + p.byteLimit = value +} + // Upgrade takes a HTTP request and upgrades the connection to WebSocket. // Returns a websocket Conn if the upgrade succeeded, and an error otherwise. func Upgrade(w http.ResponseWriter, r *http.Request) (*websocket.Conn, error) { @@ -180,6 +188,7 @@ func (p *Protocol) receiver(ctx context.Context, func (p *Protocol) sendCounterflow(ctx context.Context, measurerCh <-chan model.Measurement, results chan<- model.WireMeasurement, errCh chan<- error) { + byteLimit := int64(p.byteLimit) for { select { case <-ctx.Done(): @@ -218,13 +227,19 @@ func (p *Protocol) sendCounterflow(ctx context.Context, case results <- wm: default: } + + // End the test once enough bytes have been received. + if byteLimit > 0 && m.TCPInfo != nil && m.TCPInfo.BytesReceived >= byteLimit { + p.close(ctx) + return + } } } } func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measurement, results chan<- model.WireMeasurement, errCh chan<- error) { - size := spec.MinMessageSize + size := p.ScaleMessage(spec.MinMessageSize, 0) message, err := p.makePreparedMessage(size) if err != nil { log.Printf("makePreparedMessage failed (ctx: %p)", ctx) @@ -283,27 +298,39 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme } p.applicationBytesSent.Add(int64(size)) - // Determine whether it's time to scale the message size. - if size >= spec.MaxScaledMessageSize { - continue + bytesSent := int(p.applicationBytesSent.Load()) + if p.byteLimit > 0 && bytesSent >= p.byteLimit { + p.close(ctx) + return } - if size > int(p.applicationBytesSent.Load())/spec.ScalingFraction { + // Determine whether it's time to scale the message size. + if size >= spec.MaxScaledMessageSize || size > bytesSent/spec.ScalingFraction { + size = p.ScaleMessage(size, bytesSent) continue } - size *= 2 + size = p.ScaleMessage(size*2, bytesSent) message, err = p.makePreparedMessage(size) if err != nil { log.Printf("failed to make prepared message (ctx: %p, err: %v)", ctx, err) errCh <- err return } - } } } +// ScaleMessage sets the binary message size taking into consideration byte limits. +func (p *Protocol) ScaleMessage(msgSize int, bytesSent int) int { + // Check if the next payload size will push the total number of bytes over the limit. + excess := bytesSent + msgSize - p.byteLimit + if p.byteLimit > 0 && excess > 0 { + msgSize -= excess + } + return msgSize +} + func (p *Protocol) close(ctx context.Context) { msg := websocket.FormatCloseMessage( websocket.CloseNormalClosure, "Done sending") diff --git a/pkg/throughput1/protocol_test.go b/pkg/throughput1/protocol_test.go index 6180fca..37c0e5d 100644 --- a/pkg/throughput1/protocol_test.go +++ b/pkg/throughput1/protocol_test.go @@ -137,3 +137,51 @@ func TestProtocol_Download(t *testing.T) { } } } + +func TestProtocol_ScaleMessage(t *testing.T) { + tests := []struct { + name string + byteLimit int + msgSize int + bytesSent int + want int + }{ + { + name: "no-limit", + byteLimit: 0, + msgSize: 10, + bytesSent: 100, + want: 10, + }, + { + name: "under-limit", + byteLimit: 200, + msgSize: 10, + bytesSent: 100, + want: 10, + }, + { + name: "at-limit", + byteLimit: 110, + msgSize: 10, + bytesSent: 100, + want: 10, + }, + { + name: "over-limit", + byteLimit: 110, + msgSize: 20, + bytesSent: 100, + want: 10, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &throughput1.Protocol{} + p.SetByteLimit(tt.byteLimit) + if got := p.ScaleMessage(tt.msgSize, tt.bytesSent); got != tt.want { + t.Errorf("Protocol.ScaleMessage() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/throughput1/spec/spec.go b/pkg/throughput1/spec/spec.go index cb966b8..4a00424 100644 --- a/pkg/throughput1/spec/spec.go +++ b/pkg/throughput1/spec/spec.go @@ -37,6 +37,11 @@ const ( // SecWebSocketProtocol is the value of the Sec-WebSocket-Protocol header. SecWebSocketProtocol = "net.measurementlab.throughput.v1" + + // ByteLimitParameterName is the name of the parameter that clients can use + // to terminate throughput1 download tests once the test has transferred + // the specified number of bytes. + ByteLimitParameterName = "bytes" ) // SubtestKind indicates the subtest kind From 1f6358738961110f56632b86507136c305cd9736 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Wed, 3 Jan 2024 17:28:15 -0500 Subject: [PATCH 23/37] feat: separate start and measure operations --- internal/measurer/measurer.go | 37 +++++++++++++++++++++--------- internal/measurer/measurer_test.go | 3 ++- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/internal/measurer/measurer.go b/internal/measurer/measurer.go index 246933a..7823f84 100644 --- a/internal/measurer/measurer.go +++ b/internal/measurer/measurer.go @@ -16,13 +16,22 @@ import ( "github.com/m-lab/msak/pkg/throughput1/spec" ) -type throughput1Measurer struct { +// Throughput1Measurer tracks state for collecting connection measurements. +type Throughput1Measurer struct { connInfo netx.ConnInfo startTime time.Time bytesReadAtStart int64 bytesWrittenAtStart int64 dstChan chan model.Measurement + + // ReadChan is a readable channel for measurements created by the measurer. + ReadChan <-chan model.Measurement +} + +// New creates an empty Throughput1Measurer. The measurer must be started with Start. +func New() *Throughput1Measurer { + return &Throughput1Measurer{} } // Start starts a measurer goroutine that periodically reads the tcp_info and @@ -31,7 +40,7 @@ type throughput1Measurer struct { // // The context determines the measurer goroutine's lifetime. // If passed a connection that is not a netx.Conn, this function will panic. -func Start(ctx context.Context, conn net.Conn) <-chan model.Measurement { +func (m *Throughput1Measurer) Start(ctx context.Context, conn net.Conn) <-chan model.Measurement { // Implementation note: this channel must be buffered to account for slow // readers. The "typical" reader is an throughput1 send or receive loop, which // might be busy with data r/w. The buffer size corresponds to at least 10 @@ -42,9 +51,10 @@ func Start(ctx context.Context, conn net.Conn) <-chan model.Measurement { connInfo := netx.ToConnInfo(conn) read, written := connInfo.ByteCounters() - m := &throughput1Measurer{ + *m = Throughput1Measurer{ connInfo: connInfo, dstChan: dst, + ReadChan: dst, startTime: time.Now(), // Byte counters are offset by their initial value, so that the // BytesSent/BytesReceived fields represent "application-level bytes @@ -55,10 +65,10 @@ func Start(ctx context.Context, conn net.Conn) <-chan model.Measurement { bytesWrittenAtStart: int64(written), } go m.loop(ctx) - return dst + return m.ReadChan } -func (m *throughput1Measurer) loop(ctx context.Context) { +func (m *Throughput1Measurer) loop(ctx context.Context) { log.Debug("Measurer started", "context", ctx) defer log.Debug("Measurer stopped", "context", ctx) t, err := memoryless.NewTicker(ctx, memoryless.Config{ @@ -81,7 +91,16 @@ func (m *throughput1Measurer) loop(ctx context.Context) { } } -func (m *throughput1Measurer) measure(ctx context.Context) { +func (m *Throughput1Measurer) measure(ctx context.Context) { + select { + case <-ctx.Done(): + // NOTHING + case m.dstChan <- m.Measure(ctx): + } +} + +// Measure collects metrics about the life of the connection. +func (m *Throughput1Measurer) Measure(ctx context.Context) model.Measurement { // On non-Linux systems, collecting kernel metrics WILL fail. In that case, // we still want to return a (empty) Measurement. bbrInfo, tcpInfo, err := m.connInfo.Info() @@ -92,10 +111,7 @@ func (m *throughput1Measurer) measure(ctx context.Context) { // Read current bytes counters. totalRead, totalWritten := m.connInfo.ByteCounters() - select { - case <-ctx.Done(): - // NOTHING - case m.dstChan <- model.Measurement{ + return model.Measurement{ ElapsedTime: time.Since(m.startTime).Microseconds(), Network: model.ByteCounters{ BytesSent: int64(totalWritten) - m.bytesWrittenAtStart, @@ -106,6 +122,5 @@ func (m *throughput1Measurer) measure(ctx context.Context) { LinuxTCPInfo: tcpInfo, ElapsedTime: time.Since(m.connInfo.AcceptTime()).Microseconds(), }, - }: } } diff --git a/internal/measurer/measurer_test.go b/internal/measurer/measurer_test.go index 6205f36..17fe0f7 100644 --- a/internal/measurer/measurer_test.go +++ b/internal/measurer/measurer_test.go @@ -23,7 +23,8 @@ func TestNdt8Measurer_Start(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - mchan := measurer.Start(ctx, serverConn) + m := measurer.New() + mchan := m.Start(ctx, serverConn) go func() { _, err := serverConn.Write([]byte("test")) rtx.Must(err, "failed to write to pipe") From 4f722ff846f417b68d8c1d0cd3086d6d4d559ef9 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Wed, 3 Jan 2024 17:28:41 -0500 Subject: [PATCH 24/37] Use new measurer interface --- pkg/throughput1/protocol.go | 97 ++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index 43df87f..a7ba88e 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -24,17 +24,10 @@ type senderFunc func(ctx context.Context, measurerCh <-chan model.Measurement, results chan<- model.WireMeasurement, errCh chan<- error) +// Measurer is an interface for collecting connection metrics. type Measurer interface { Start(context.Context, net.Conn) <-chan model.Measurement -} - -// DefaultMeasurer is the default throughput1 measurer that wraps the measurer -// package's Start function. -type DefaultMeasurer struct{} - -func (*DefaultMeasurer) Start(ctx context.Context, - c net.Conn) <-chan model.Measurement { - return measurer.Start(ctx, c) + Measure(ctx context.Context) model.Measurement } // Protocol is the implementation of the throughput1 protocol. @@ -59,7 +52,7 @@ func New(conn *websocket.Conn) *Protocol { connInfo: netx.ToConnInfo(conn.UnderlyingConn()), // Seed randomness source with the current time. rnd: rand.New(rand.NewSource(time.Now().UnixMilli())), - measurer: &DefaultMeasurer{}, + measurer: measurer.New(), } } @@ -185,6 +178,32 @@ func (p *Protocol) receiver(ctx context.Context, } } +func (p *Protocol) sendWireMeasurement(ctx context.Context, m model.Measurement) (*model.WireMeasurement, error) { + wm := model.WireMeasurement{} + p.once.Do(func() { + wm = p.createWireMeasurement(ctx) + }) + wm.Measurement = m + wm.Application = model.ByteCounters{ + BytesSent: p.applicationBytesSent.Load(), + BytesReceived: p.applicationBytesReceived.Load(), + } + // Encode as JSON separately so we can read the message size before + // sending. + jsonwm, err := json.Marshal(wm) + if err != nil { + log.Printf("failed to encode measurement (ctx: %p, err: %v)", ctx, err) + return nil, err + } + err = p.conn.WriteMessage(websocket.TextMessage, jsonwm) + if err != nil { + log.Printf("failed to write measurement JSON (ctx: %p, err: %v)", ctx, err) + return nil, err + } + p.applicationBytesSent.Add(int64(len(jsonwm))) + return &wm, nil +} + func (p *Protocol) sendCounterflow(ctx context.Context, measurerCh <-chan model.Measurement, results chan<- model.WireMeasurement, errCh chan<- error) { @@ -192,44 +211,25 @@ func (p *Protocol) sendCounterflow(ctx context.Context, for { select { case <-ctx.Done(): + // TODO: do we need to send a final wiremessage here? p.close(ctx) return case m := <-measurerCh: - wm := model.WireMeasurement{} - p.once.Do(func() { - wm = p.createWireMeasurement(ctx) - }) - wm.Measurement = m - wm.Application = model.ByteCounters{ - BytesSent: p.applicationBytesSent.Load(), - BytesReceived: p.applicationBytesReceived.Load(), - } - // Encode as JSON separately so we can read the message size before - // sending. - jsonwm, err := json.Marshal(wm) + wm, err := p.sendWireMeasurement(ctx, m) if err != nil { - log.Printf("failed to encode measurement (ctx: %p, err: %v)", - ctx, err) errCh <- err return } - err = p.conn.WriteMessage(websocket.TextMessage, jsonwm) - if err != nil { - log.Printf("failed to write measurement JSON (ctx: %p, err: %v)", ctx, err) - errCh <- err - return - } - p.applicationBytesSent.Add(int64(len(jsonwm))) - // This send is non-blocking in case there is no one to read the // Measurement message and the channel's buffer is full. select { - case results <- wm: + case results <- *wm: default: } // End the test once enough bytes have been received. if byteLimit > 0 && m.TCPInfo != nil && m.TCPInfo.BytesReceived >= byteLimit { + // WireMessage was just sent above, so we do not need to send another. p.close(ctx) return } @@ -254,39 +254,21 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme for { select { case <-ctx.Done(): + // Attempt to send final write message before close. Ignore errors. + p.sendWireMeasurement(ctx, p.measurer.Measure(ctx)) p.close(ctx) return case m := <-measurerCh: - wm := model.WireMeasurement{} - p.once.Do(func() { - wm = p.createWireMeasurement(ctx) - }) - wm.Measurement = m - wm.Application = model.ByteCounters{ - BytesReceived: p.applicationBytesReceived.Load(), - BytesSent: p.applicationBytesSent.Load(), - } - // Encode as JSON separately so we can read the message size before - // sending. - jsonwm, err := json.Marshal(wm) - if err != nil { - log.Printf("failed to encode measurement (ctx: %p, err: %v)", - ctx, err) - errCh <- err - return - } - err = p.conn.WriteMessage(websocket.TextMessage, jsonwm) + wm, err := p.sendWireMeasurement(ctx, m) if err != nil { - log.Printf("failed to write measurement JSON (ctx: %p, err: %v)", ctx, err) errCh <- err return } - p.applicationBytesSent.Add(int64(len(jsonwm))) // This send is non-blocking in case there is no one to read the // Measurement message and the channel's buffer is full. select { - case results <- wm: + case results <- *wm: default: } default: @@ -300,6 +282,11 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme bytesSent := int(p.applicationBytesSent.Load()) if p.byteLimit > 0 && bytesSent >= p.byteLimit { + _, err := p.sendWireMeasurement(ctx, p.measurer.Measure(ctx)) + if err != nil { + errCh <- err + return + } p.close(ctx) return } From 9545fd5e7ee5229daa859c8faa152d8ab82ece95 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Wed, 3 Jan 2024 17:47:48 -0500 Subject: [PATCH 25/37] Send for upload also --- pkg/throughput1/protocol.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index a7ba88e..5d14056 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -211,7 +211,8 @@ func (p *Protocol) sendCounterflow(ctx context.Context, for { select { case <-ctx.Done(): - // TODO: do we need to send a final wiremessage here? + // Attempt to send final write message before close. Ignore errors. + p.sendWireMeasurement(ctx, p.measurer.Measure(ctx)) p.close(ctx) return case m := <-measurerCh: From bdf32c83e78786541e34b711f116fdfcd6d3d68a Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 8 Jan 2024 12:28:32 -0500 Subject: [PATCH 26/37] Add minimal download throughput1 client & update README (#31) * Add minimal client * Add design doc and build steps to README * Update README with example usage --- README.md | 105 ++++++++++++- cmd/minimal-download/main.go | 283 +++++++++++++++++++++++++++++++++++ pkg/client/emitter.go | 5 +- 3 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 cmd/minimal-download/main.go diff --git a/README.md b/README.md index 78a9519..f1994c4 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,107 @@ [![Coverage Status](https://coveralls.io/repos/github/m-lab/msak/badge.svg?branch=main)](https://coveralls.io/github/m-lab/msak?branch=main) [![Go Reference](https://pkg.go.dev/badge/github.com/m-lab/msak.svg)](https://pkg.go.dev/github.com/m-lab/msak) -Measurements Swiss Army Knife +The [MSAK design doc (view)][1] describes the throughput1 and latency1 protocols. + +[1]: https://docs.google.com/document/d/1OmKXGhQe2mT1gSXI2NT_SxvnKu5OHpBGIYpoWNJwmWA/edit + +* `msak-server` - is the MSAK server +* `msak-client` - is a full reference client for the throughput1 protocol + +Additional reference clients are also available: + +* `minimal-download` - is a minimal download-only, reference cleint for the throughput1 protocol +* `msak-latency` - is a reference client for the latency1 protocol + +## Server + +To build the server and run locally without TLS certificates: + +```sh +$ go install github.com/m-lab/msak/cmd/msak-server@latest +... +$ msak-server +2024/01/04 17:41:01 INFO About to listen for ws tests endpoint=:8080 +2024/01/04 17:41:01 INFO Accepting UDP packets... +``` + +## Clients + +To build the client and target the local server: + +```sh +$ go install github.com/m-lab/msak/cmd/msak-client@latest +... +$ msak-client -duration=2s -streams=1 -server localhost:8080 -scheme ws +Starting download stream (server: localhost:8080) +Connected to ws://localhost:8080/throughput/v1/download?bytes=0&... +Elapsed: 0.10s, Goodput: 0.000000 Mb/s, MinRTT: 0 +Elapsed: 0.20s, Goodput: 0.000000 Mb/s, MinRTT: 0 +Elapsed: 0.30s, Goodput: 0.000000 Mb/s, MinRTT: 0 +Elapsed: 0.40s, Goodput: 0.000000 Mb/s, MinRTT: 0 +Elapsed: 0.50s, Goodput: 18941.574715 Mb/s, MinRTT: 0 +Elapsed: 0.60s, Goodput: 15784.580735 Mb/s, MinRTT: 0 +Elapsed: 0.70s, Goodput: 13526.483102 Mb/s, MinRTT: 0 +Elapsed: 0.80s, Goodput: 11839.166779 Mb/s, MinRTT: 0 +Elapsed: 0.90s, Goodput: 21180.762969 Mb/s, MinRTT: 0 +Elapsed: 1.00s, Goodput: 21419.807987 Mb/s, MinRTT: 0 +Stream 0 complete (server localhost:8080) +Starting upload stream (server: localhost:8080) +Connected to ws://localhost:8080/throughput/v1/upload?bytes=0&... +Elapsed: 0.10s, Goodput: 0.000000 Mb/s, MinRTT: 0 +Elapsed: 0.20s, Goodput: 18007.157915 Mb/s, MinRTT: 0 +Elapsed: 0.30s, Goodput: 19836.665589 Mb/s, MinRTT: 0 +Elapsed: 0.40s, Goodput: 14878.343408 Mb/s, MinRTT: 0 +Elapsed: 0.50s, Goodput: 11902.100544 Mb/s, MinRTT: 0 +Elapsed: 0.60s, Goodput: 23205.208525 Mb/s, MinRTT: 0 +Elapsed: 0.70s, Goodput: 19895.442053 Mb/s, MinRTT: 0 +Elapsed: 0.80s, Goodput: 21784.415044 Mb/s, MinRTT: 0 +Elapsed: 0.90s, Goodput: 19363.955232 Mb/s, MinRTT: 0 +Elapsed: 1.00s, Goodput: 17431.176757 Mb/s, MinRTT: 0 +Stream 0 complete (server localhost:8080) +``` + +To build the minimal client and target a local or remote server: + +```sh +$ go install github.com/m-lab/msak/cmd/minimal-download@latest +... +# Local +$ minimal-download -duration 1s -server.url ws://localhost:8080/throughput/v1/download +Download server #1 - rate 34215.33 Mbps, rtt 0.04ms, elapsed 0.1009s, application r/w: 0/436207616, network r/w: 0/435163654 kernel* r/w: 538/431369776 +Download server #1 - rate 33915.22 Mbps, rtt 0.02ms, elapsed 0.2009s, application r/w: 0/856687767, network r/w: 0/855647819 kernel* r/w: 538/851814781 +Download server #1 - rate 34634.09 Mbps, rtt 0.04ms, elapsed 0.5741s, application r/w: 0/2489321624, network r/w: 0/2488297250 kernel* r/w: 538/2485238689 +Download server #1 - rate 34451.50 Mbps, rtt 0.04ms, elapsed 0.7029s, application r/w: 0/3031436447, network r/w: 0/3030417247 kernel* r/w: 538/3026848582 +Download server #1 - rate 34387.62 Mbps, rtt 0.03ms, elapsed 1.0008s, application r/w: 0/4304408743, network r/w: 0/4304450273 kernel* r/w: 538/4301737109 +Download client #1 - Avg 34353.74 Mbps, MinRTT 0.00ms, elapsed 1.0024s, application r/w: 0/4304409778 + + +# Remote with time limit. +$ minimal-download -duration 1s +Download server #1 - rate 239.68 Mbps, rtt 13.96ms, elapsed 0.1014s, application r/w: 0/6815744, network r/w: 0/6400516 kernel* r/w: 1304/3039466 +Download server #1 - rate 375.56 Mbps, rtt 15.13ms, elapsed 0.2024s, application r/w: 0/13632647, network r/w: 0/13112011 kernel* r/w: 1304/9503338 +Download server #1 - rate 429.03 Mbps, rtt 19.15ms, elapsed 0.3034s, application r/w: 0/19925135, network r/w: 0/19298323 kernel* r/w: 1304/16271290 +Download server #1 - rate 473.54 Mbps, rtt 15.88ms, elapsed 0.5237s, application r/w: 0/35654810, network r/w: 0/34737910 kernel* r/w: 1304/30997450 +Download server #1 - rate 487.79 Mbps, rtt 15.34ms, elapsed 0.6464s, application r/w: 0/42995877, network r/w: 0/42564857 kernel* r/w: 1304/39414674 +Download server #1 - rate 499.34 Mbps, rtt 17.36ms, elapsed 1.0154s, application r/w: 0/66065584, network r/w: 0/66158482 kernel* r/w: 1304/63380522 +Download client #1 - Avg 502.43 Mbps, MinRTT 4.11ms, elapsed 1.0520s, application r/w: 0/66066624 + +# Remote with bytes limit. +$ minimal-download -bytes=150000 +Download server #1 - rate 8.24 Mbps, rtt 12.17ms, elapsed 0.0128s, application r/w: 0/150000, network r/w: 0/164976 kernel* r/w: 1309/13146 +Download client #1 - Avg 30.51 Mbps, MinRTT 10.99ms, elapsed 0.0433s, application r/w: 0/164972 +``` + +> NOTE: the application, network, and kernel metrics may differ to the degree +that some data is buffered, or includes added headers, or is traversing the +physical network itself. For example, after a websocket write and before +TLS/WebSocket headers are added (application), after TLS/WebSocket headers are +added before being sent to the Linux kernel (network), or after the Linux kernel +sends over the physical network and before the remote client acknowledges the +bytes received (kernel). + +The maximum difference between the application and network sent sizes should be +equal to the `spec.MaxScaledMessageSize` + WebSocket/TLS headers size, which +should typically be below 1MB, and the maximum difference between the network +and kernel sent sizes should equal the Linux kernel buffers plus the network's +bandwidth delay product. Typically values range between 64k and 4MB or more. diff --git a/cmd/minimal-download/main.go b/cmd/minimal-download/main.go new file mode 100644 index 0000000..67e8ce3 --- /dev/null +++ b/cmd/minimal-download/main.go @@ -0,0 +1,283 @@ +// Package main implements a bare-bones minimal MSAK throughput1 client. +package main + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "runtime" + "time" + + "github.com/google/uuid" + "github.com/gorilla/websocket" +) + +const ( + clientName = "msak-minimal-download-go" + clientVersion = "v0.0.1" + locateURL = "https://locate.measurementlab.net/v2/nearest/" +) + +var ( + flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") + flagDuration = flag.Duration("duration", 5*time.Second, "Length of the last stream") + flagByteLimit = flag.Int("bytes", 0, "Byte limit to request to the server") + flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") + flagServerURL = flag.String("server.url", "", "URL to directly target") + flagMID = flag.String("mid", uuid.NewString(), "Measurement ID to use") + flagScheme = flag.String("scheme", "wss", "Websocket scheme (wss or ws)") + flagLocateURL = flag.String("locate.url", locateURL, "The base url for the Locate API") +) + +// WireMeasurement is a wrapper for Measurement structs that contains +// information about this TCP stream that does not need to be sent every time. +// Every field except for Measurement is only expected to be non-empty once. +// +// Find the authoritative structures in: +// * github.com/m-lab/msak/pkg/throughput1/model/measurement.go +type WireMeasurement struct { + // CC is the congestion control used by the sender of this WireMeasurement. + CC string `json:",omitempty"` + // UUID is the unique identifier for this TCP stream. + UUID string `json:",omitempty"` + // LocalAddr is the local TCP endpoint (ip:port). + LocalAddr string `json:",omitempty"` + // RemoteAddr is the server's TCP endpoint (ip:port). + RemoteAddr string `json:",omitempty"` + // Measurement is the Measurement struct wrapped by this WireMeasurement. + Measurement +} + +// The Measurement struct contains measurement results. This structure is +// meant to be serialised as JSON and sent as a textual message. +type Measurement struct { + // Application contains the application-level BytesSent/Received pair. + Application ByteCounters + // Network contains the network-level BytesSent/Received pair. + Network ByteCounters + // ElapsedTime is the time elapsed since the start of the measurement + // according to the party sending this Measurement. + ElapsedTime int64 `json:",omitempty"` + // BBRInfo is an optional struct containing BBR metrics. Only applicable + // when the congestion control algorithm used by the party sending this + // Measurement is BBR. WARNING: field types are approximate. + BBRInfo map[string]int64 `json:",omitempty"` + // TCPInfo is an optional struct containing some of the TCP_INFO kernel + // metrics for this TCP stream. Only applicable when the party sending this + // Measurement has access to it. WARNING: field types are approximate. + TCPInfo map[string]int64 `json:",omitempty"` +} + +type ByteCounters struct { + // BytesSent is the number of bytes sent. + BytesSent int64 `json:",omitempty"` + // BytesReceived is the number of bytes received. + BytesReceived int64 `json:",omitempty"` +} + +// NearestResult is returned by the Locate API in response to query requests. +type NearestResult struct { + // Results contains an array of Targets matching the client request. + Results []Target `json:"results,omitempty"` +} + +// Target is returned by the Locate API. +type Target struct { + // URLs contains measurement service resource names and the complete URL for + // running a measurement. + URLs map[string]string `json:"urls"` +} + +// localDialer allows insecure TLS for explicit servers. +var localDialer = &websocket.Dialer{ + HandshakeTimeout: 5 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: *flagNoVerify, + }, +} + +func init() { + // Disable all prefixing for logging. + log.SetFlags(0) +} + +// connect to the given msak server URL, returning a *websocket.Conn. +func connect(ctx context.Context, s *url.URL) (*websocket.Conn, error) { + q := s.Query() + q.Set("streams", fmt.Sprintf("%d", 1)) + q.Set("cc", *flagCC) + q.Set("bytes", fmt.Sprintf("%d", *flagByteLimit)) + q.Set("duration", fmt.Sprintf("%d", (*flagDuration).Milliseconds())) + q.Set("client_arch", runtime.GOARCH) + q.Set("client_library_name", clientName+"-adhoc") + q.Set("client_library_version", clientVersion+"-adhoc") + q.Set("client_os", runtime.GOOS) + q.Set("client_name", clientName) + q.Set("client_version", clientVersion) + s.RawQuery = q.Encode() + headers := http.Header{} + headers.Add("Sec-WebSocket-Protocol", "net.measurementlab.throughput.v1") + headers.Add("User-Agent", clientName+"/"+clientVersion) + conn, _, err := localDialer.DialContext(ctx, s.String(), headers) + return conn, err +} + +// formatMessage reports a WireMeasurement in a human readable format. +func formatMessage(prefix string, stream int, m WireMeasurement) { + log.Printf("%s #%d - rate %0.2f Mbps, rtt %5.2fms, elapsed %0.4fs, application r/w: %d/%d, network r/w: %d/%d kernel* r/w: %d/%d\n", + prefix, stream, + 8*float64(m.TCPInfo["BytesAcked"])/(float64(m.ElapsedTime)), // to mbps. + float64(m.TCPInfo["RTT"])/1000.0, // to ms. + float64(m.ElapsedTime)/1000000.0, // to sec. + m.Application.BytesReceived, m.Application.BytesSent, + m.Network.BytesReceived, m.Network.BytesSent, + m.TCPInfo["BytesReceived"], m.TCPInfo["BytesAcked"], + ) +} + +// locateGetServers contacts the Locate API for a set of healthy servers. +func locateGetServers(ctx context.Context, userAgent, locate string) ([]Target, error) { + u, err := url.Parse(*flagLocateURL) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, locate) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + if userAgent == "" { + // User agent is required. + return nil, errors.New("no user agent given") + } + req.Header.Set("User-Agent", userAgent) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + reply := &NearestResult{} + err = json.Unmarshal(b, reply) + if err != nil { + return nil, err + } + return reply.Results, err +} + +// getDownloadServer find a single server from given flags or Locate API. +func getDownloadServer(ctx context.Context) (*url.URL, error) { + // Use explicit server if provided. + if *flagServerURL != "" { + u, err := url.Parse(*flagServerURL) + if err != nil { + return nil, err + } + q := u.Query() + q.Set("mid", *flagMID) + u.RawQuery = q.Encode() + return u, nil + } + + // Use Locate API to request otherwise. + targets, err := locateGetServers(ctx, clientName+"/"+clientVersion, "msak/throughput1") + if err != nil { + return nil, err + } + // Just use the first result. + for i := range targets { + srvurl := targets[i].URLs[*flagScheme+":///throughput/v1/download"] + // Get server url. + return url.Parse(srvurl) + } + return nil, errors.New("no server") +} + +// getConn connects to a download server, returning the *websocket.Conn. +func getConn(ctx context.Context) (*websocket.Conn, error) { + srv, err := getDownloadServer(ctx) + if err != nil { + return nil, err + } + // Connect to server. + return connect(ctx, srv) +} + +func main() { + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), *flagDuration*2) + defer cancel() + + conn, err := getConn(ctx) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + // Max runtime. + deadline := time.Now().Add(*flagDuration * 2) + conn.SetWriteDeadline(deadline) + conn.SetReadDeadline(deadline) + + // receive from text & binary messages from conn until the context expires or conn closes. + var applicationBytesReceived int64 + var minRTT int64 + start := time.Now() +outer: + for { + select { + case <-ctx.Done(): + break outer + default: + kind, reader, err := conn.NextReader() + if err != nil { + if !websocket.IsCloseError(err, websocket.CloseNormalClosure) { + log.Println("error", err) + } + break outer + } + switch kind { + case websocket.BinaryMessage: + // Binary messages are discarded after reading their size. + size, err := io.Copy(io.Discard, reader) + if err != nil { + log.Println("error", err) + return + } + applicationBytesReceived += size + case websocket.TextMessage: + data, err := io.ReadAll(reader) + if err != nil { + log.Println("error", err) + return + } + applicationBytesReceived += int64(len(data)) + + var m WireMeasurement + if err := json.Unmarshal(data, &m); err != nil { + log.Println("error", err) + return + } + formatMessage("Download server", 1, m) + minRTT = m.TCPInfo["MinRTT"] + } + } + } + since := time.Since(start) + log.Printf("Download client #1 - Avg %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + 8*float64(applicationBytesReceived)/1e6/since.Seconds(), // as mbps. + float64(minRTT)/1000.0, // as ms. + since.Seconds(), 0, applicationBytesReceived) +} diff --git a/pkg/client/emitter.go b/pkg/client/emitter.go index 633b073..e0ea0d8 100644 --- a/pkg/client/emitter.go +++ b/pkg/client/emitter.go @@ -3,6 +3,7 @@ package client import ( "fmt" + "github.com/gorilla/websocket" "github.com/m-lab/msak/pkg/throughput1/model" "github.com/m-lab/msak/pkg/throughput1/spec" ) @@ -54,7 +55,9 @@ func (HumanReadable) OnMeasurement(id int, m model.WireMeasurement) { // OnError is called on errors. func (HumanReadable) OnError(err error) { - fmt.Println(err) + if !websocket.IsCloseError(err, websocket.CloseNormalClosure) { + fmt.Println(err) + } } // OnComplete is called after a stream completes. From 616671cee5c831e84e7168b3a499c85d1b06d252 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 8 Jan 2024 12:29:14 -0500 Subject: [PATCH 27/37] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index f1994c4..acb4b4b 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,6 @@ Download server #1 - rate 34451.50 Mbps, rtt 0.04ms, elapsed 0.7029s, applicati Download server #1 - rate 34387.62 Mbps, rtt 0.03ms, elapsed 1.0008s, application r/w: 0/4304408743, network r/w: 0/4304450273 kernel* r/w: 538/4301737109 Download client #1 - Avg 34353.74 Mbps, MinRTT 0.00ms, elapsed 1.0024s, application r/w: 0/4304409778 - # Remote with time limit. $ minimal-download -duration 1s Download server #1 - rate 239.68 Mbps, rtt 13.96ms, elapsed 0.1014s, application r/w: 0/6815744, network r/w: 0/6400516 kernel* r/w: 1304/3039466 From 168a40b16a1fe7a4628679ef2bf41f11141337cb Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 8 Jan 2024 14:42:58 -0500 Subject: [PATCH 28/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index acb4b4b..ae4105e 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ added before being sent to the Linux kernel (network), or after the Linux kernel sends over the physical network and before the remote client acknowledges the bytes received (kernel). -The maximum difference between the application and network sent sizes should be +> The maximum difference between the application and network sent sizes should be equal to the `spec.MaxScaledMessageSize` + WebSocket/TLS headers size, which should typically be below 1MB, and the maximum difference between the network and kernel sent sizes should equal the Linux kernel buffers plus the network's From aff16c7c036b46f1051a96b91b653d79d040e2a7 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 8 Jan 2024 16:11:36 -0500 Subject: [PATCH 29/37] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ae4105e..e71614a 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,9 @@ Download server #1 - rate 8.24 Mbps, rtt 12.17ms, elapsed 0.0128s, application r Download client #1 - Avg 30.51 Mbps, MinRTT 10.99ms, elapsed 0.0433s, application r/w: 0/164972 ``` -> NOTE: the application, network, and kernel metrics may differ to the degree +## Measurements + +The application, network, and kernel metrics may differ to the degree that some data is buffered, or includes added headers, or is traversing the physical network itself. For example, after a websocket write and before TLS/WebSocket headers are added (application), after TLS/WebSocket headers are @@ -103,7 +105,7 @@ added before being sent to the Linux kernel (network), or after the Linux kernel sends over the physical network and before the remote client acknowledges the bytes received (kernel). -> The maximum difference between the application and network sent sizes should be +The maximum difference between the application and network sent sizes should be equal to the `spec.MaxScaledMessageSize` + WebSocket/TLS headers size, which should typically be below 1MB, and the maximum difference between the network and kernel sent sizes should equal the Linux kernel buffers plus the network's From 207420cd945669c7b2144e070522af50106c2ac2 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Mon, 8 Jan 2024 16:21:20 -0500 Subject: [PATCH 30/37] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index e71614a..667c23e 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,11 @@ Download server #1 - rate 8.24 Mbps, rtt 12.17ms, elapsed 0.0128s, application r Download client #1 - Avg 30.51 Mbps, MinRTT 10.99ms, elapsed 0.0433s, application r/w: 0/164972 ``` +Every TCP connection has performance metrics accessible from the server end and +the client end. The `minimal-download` client reports metrics generated by the +server side and concludes with client side average performance. Client side +performance is comparable to what a user (or user application) would see. + ## Measurements The application, network, and kernel metrics may differ to the degree From bc4cce963f1a0188ef6e9e2f4bbdda2b8b5ad9b3 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Wed, 10 Jan 2024 10:12:27 -0500 Subject: [PATCH 31/37] fix(size): create new message if size changes (#33) * fix(size): create new message if size changes * Update readme * Prevent flaky test --- README.md | 14 +++++++------- internal/latency1/latency1_test.go | 3 +++ pkg/throughput1/protocol.go | 9 ++++++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 667c23e..9a3f58e 100644 --- a/README.md +++ b/README.md @@ -72,12 +72,12 @@ $ go install github.com/m-lab/msak/cmd/minimal-download@latest ... # Local $ minimal-download -duration 1s -server.url ws://localhost:8080/throughput/v1/download -Download server #1 - rate 34215.33 Mbps, rtt 0.04ms, elapsed 0.1009s, application r/w: 0/436207616, network r/w: 0/435163654 kernel* r/w: 538/431369776 -Download server #1 - rate 33915.22 Mbps, rtt 0.02ms, elapsed 0.2009s, application r/w: 0/856687767, network r/w: 0/855647819 kernel* r/w: 538/851814781 -Download server #1 - rate 34634.09 Mbps, rtt 0.04ms, elapsed 0.5741s, application r/w: 0/2489321624, network r/w: 0/2488297250 kernel* r/w: 538/2485238689 -Download server #1 - rate 34451.50 Mbps, rtt 0.04ms, elapsed 0.7029s, application r/w: 0/3031436447, network r/w: 0/3030417247 kernel* r/w: 538/3026848582 -Download server #1 - rate 34387.62 Mbps, rtt 0.03ms, elapsed 1.0008s, application r/w: 0/4304408743, network r/w: 0/4304450273 kernel* r/w: 538/4301737109 -Download client #1 - Avg 34353.74 Mbps, MinRTT 0.00ms, elapsed 1.0024s, application r/w: 0/4304409778 +minimal-download -duration 1s -server.url ws://localhost:8080/throughput/v1/download +Download server #1 - rate 35260.80 Mbps, rtt 0.03ms, elapsed 0.2076s, application r/w: 0/918552576, network r/w: 0/917513214 kernel* r/w: 538/914903145 +Download server #1 - rate 34191.58 Mbps, rtt 0.03ms, elapsed 0.5168s, application r/w: 0/2213545117, network r/w: 0/2212518109 kernel* r/w: 538/2208912708 +Download server #1 - rate 33703.03 Mbps, rtt 0.05ms, elapsed 0.9170s, application r/w: 0/3868199075, network r/w: 0/3867187851 kernel* r/w: 538/3863293895 +Download server #1 - rate 33591.42 Mbps, rtt 0.03ms, elapsed 1.0005s, application r/w: 0/4203744426, network r/w: 0/4203784992 kernel* r/w: 538/4200858760 +Download client #1 - Avg 33552.56 Mbps, MinRTT 0.00ms, elapsed 1.0023s, application r/w: 0/4203745461 # Remote with time limit. $ minimal-download -duration 1s @@ -92,7 +92,7 @@ Download client #1 - Avg 502.43 Mbps, MinRTT 4.11ms, elapsed 1.0520s, applicati # Remote with bytes limit. $ minimal-download -bytes=150000 Download server #1 - rate 8.24 Mbps, rtt 12.17ms, elapsed 0.0128s, application r/w: 0/150000, network r/w: 0/164976 kernel* r/w: 1309/13146 -Download client #1 - Avg 30.51 Mbps, MinRTT 10.99ms, elapsed 0.0433s, application r/w: 0/164972 +Download client #1 - Avg 30.51 Mbps, MinRTT 10.99ms, elapsed 0.0433s, application r/w: 0/151008 ``` Every TCP connection has performance metrics accessible from the server end and diff --git a/internal/latency1/latency1_test.go b/internal/latency1/latency1_test.go index cf80e13..ea072c9 100644 --- a/internal/latency1/latency1_test.go +++ b/internal/latency1/latency1_test.go @@ -157,6 +157,9 @@ func TestHandler_Result(t *testing.T) { rw.Result().StatusCode) } + // Delay return to allow handler go routines to settle. + // TODO: add complete shutdown of handler to prevent flaky tests. + time.Sleep(100 * time.Millisecond) } func TestHandler_processPacket(t *testing.T) { diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index 5d14056..c66d295 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -292,13 +292,20 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme return } + origSize := size // Determine whether it's time to scale the message size. if size >= spec.MaxScaledMessageSize || size > bytesSent/spec.ScalingFraction { size = p.ScaleMessage(size, bytesSent) + } else { + size = p.ScaleMessage(size*2, bytesSent) + } + + if size == origSize { + // We do not need to create a new message. continue } - size = p.ScaleMessage(size*2, bytesSent) + // Create a new message for the new size. message, err = p.makePreparedMessage(size) if err != nil { log.Printf("failed to make prepared message (ctx: %p, err: %v)", ctx, err) From b1ce96b16c75cc1bc6ad61ba33de9942fab367b9 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Wed, 10 Jan 2024 12:43:56 -0500 Subject: [PATCH 32/37] fix(archive): archive final messages sent to client (#34) * fix(archive): archive final messages sent to client --- pkg/throughput1/protocol.go | 38 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/pkg/throughput1/protocol.go b/pkg/throughput1/protocol.go index c66d295..611c254 100644 --- a/pkg/throughput1/protocol.go +++ b/pkg/throughput1/protocol.go @@ -212,21 +212,15 @@ func (p *Protocol) sendCounterflow(ctx context.Context, select { case <-ctx.Done(): // Attempt to send final write message before close. Ignore errors. - p.sendWireMeasurement(ctx, p.measurer.Measure(ctx)) + p.sendAndPublishWireMeasurement(ctx, p.measurer.Measure(ctx), results) p.close(ctx) return case m := <-measurerCh: - wm, err := p.sendWireMeasurement(ctx, m) + err := p.sendAndPublishWireMeasurement(ctx, m, results) if err != nil { errCh <- err return } - // This send is non-blocking in case there is no one to read the - // Measurement message and the channel's buffer is full. - select { - case results <- *wm: - default: - } // End the test once enough bytes have been received. if byteLimit > 0 && m.TCPInfo != nil && m.TCPInfo.BytesReceived >= byteLimit { @@ -238,6 +232,21 @@ func (p *Protocol) sendCounterflow(ctx context.Context, } } +func (p *Protocol) sendAndPublishWireMeasurement(ctx context.Context, m model.Measurement, results chan<- model.WireMeasurement) error { + wm, err := p.sendWireMeasurement(ctx, m) + if err != nil { + return err + } + + // This send is non-blocking in case there is no one to read the + // Measurement message and the channel's buffer is full. + select { + case results <- *wm: + default: + } + return nil +} + func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measurement, results chan<- model.WireMeasurement, errCh chan<- error) { size := p.ScaleMessage(spec.MinMessageSize, 0) @@ -256,22 +265,15 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme select { case <-ctx.Done(): // Attempt to send final write message before close. Ignore errors. - p.sendWireMeasurement(ctx, p.measurer.Measure(ctx)) + p.sendAndPublishWireMeasurement(ctx, p.measurer.Measure(ctx), results) p.close(ctx) return case m := <-measurerCh: - wm, err := p.sendWireMeasurement(ctx, m) + err := p.sendAndPublishWireMeasurement(ctx, m, results) if err != nil { errCh <- err return } - - // This send is non-blocking in case there is no one to read the - // Measurement message and the channel's buffer is full. - select { - case results <- *wm: - default: - } default: err = p.conn.WritePreparedMessage(message) if err != nil { @@ -283,7 +285,7 @@ func (p *Protocol) sender(ctx context.Context, measurerCh <-chan model.Measureme bytesSent := int(p.applicationBytesSent.Load()) if p.byteLimit > 0 && bytesSent >= p.byteLimit { - _, err := p.sendWireMeasurement(ctx, p.measurer.Measure(ctx)) + err := p.sendAndPublishWireMeasurement(ctx, p.measurer.Measure(ctx), results) if err != nil { errCh <- err return From ac3c733d3bbf2a8b7777aa522b1ee0c9506d6e3c Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Wed, 10 Jan 2024 21:48:18 +0100 Subject: [PATCH 33/37] Add client for the latency1 protocol (#35) * Add latency client. * Use flag.URL for -server. * Update help text. --- cmd/msak-latency/main.go | 184 ++++++++++++++++++++++++++++++++++++++ pkg/latency1/spec/spec.go | 3 + 2 files changed, 187 insertions(+) create mode 100644 cmd/msak-latency/main.go diff --git a/cmd/msak-latency/main.go b/cmd/msak-latency/main.go new file mode 100644 index 0000000..1591be8 --- /dev/null +++ b/cmd/msak-latency/main.go @@ -0,0 +1,184 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "time" + + "github.com/m-lab/go/flagx" + "github.com/m-lab/go/rtx" + "github.com/m-lab/locate/api/locate" + v2 "github.com/m-lab/locate/api/v2" + "github.com/m-lab/msak/pkg/latency1/model" + "github.com/m-lab/msak/pkg/latency1/spec" +) + +var ( + flagServer = flagx.URL{} + flagScheme = flag.String("scheme", "http", "Server scheme (http|https)") + flagMID = flag.String("mid", "", "MID to use") +) + +func init() { + flag.Var(&flagServer, "server", "Server address. If a scheme is provided, it overrides -scheme.") +} + +func getTargetsFromLocate() []v2.Target { + locateV2 := locate.NewClient("msak-latency") + targets, err := locateV2.Nearest(context.Background(), spec.ServiceName) + rtx.Must(err, "cannot get server list from locate") + return targets +} + +func tryConnect(authorizeURL *url.URL) ([]byte, error) { + resp, err := http.Get(authorizeURL.String()) + if err != nil { + return nil, err + } + return io.ReadAll(resp.Body) +} + +func stats(result model.Summary) (int, float64, int, float64) { + if len(result.RoundTrips) == 0 { + return 0, 0, 0, 0 + } + var min, max, sum int + min = result.RoundTrips[0].RTT + for _, v := range result.RoundTrips { + if v.RTT < min { + min = v.RTT + } + if v.RTT > max { + max = v.RTT + } + sum += v.RTT + } + return min, float64(sum) / float64(len(result.RoundTrips)), + max, 1 - float64(result.PacketsReceived)/float64(result.PacketsSent) +} + +func runMeasurement(authorizeURL, resultURL *url.URL, kickoff []byte) { + udpServer, err := net.ResolveUDPAddr("udp", authorizeURL.Hostname()+":1053") + rtx.Must(err, "ResolveUDPAddr failed") + + conn, err := net.DialUDP("udp", nil, udpServer) + rtx.Must(err, "DialUDP failed") + defer conn.Close() + + // Set a time limit of 6s for the test. + conn.SetDeadline(time.Now().Add(6 * time.Second)) + + _, err = conn.Write(kickoff) + rtx.Must(err, "failed to send kickoff message") + + recvBuf := make([]byte, 512) + for { + n, err := conn.Read(recvBuf) + if err != nil { + fmt.Printf("read error: %v\n", err) + break + } + _, err = conn.Write(recvBuf[:n]) + if err != nil { + fmt.Printf("write error: %v\n", err) + break + } + fmt.Printf(".") + } + fmt.Println() + + // Get results. + resp, err := http.Get(resultURL.String()) + if err != nil { + fmt.Printf("failed to read test results: %v\n", err) + return + } + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Printf("failed to read response body: %v\n", err) + return + } + + var result model.Summary + err = json.Unmarshal(body, &result) + if err != nil { + fmt.Printf("error parsing result as JSON: %v\n", err) + return + } + min, avg, max, loss := stats(result) + fmt.Printf("rtt min/avg/max: %.3f/%.3f/%.3f ms, loss: %.1f\n", + float64(min)/1000, avg/1000, float64(max)/1000, loss) +} + +func main() { + flag.Parse() + flagx.ArgsFromEnv(flag.CommandLine) + + var ( + authorizeURL *url.URL + resultURL *url.URL + + kickoffMsg []byte + ) + + if flagServer.URL != nil { + // If a server was provided, use it. + var scheme string + // Use the scheme included in the server URL, if present. + if flagServer.Scheme != "" { + scheme = flagServer.Scheme + } else { + scheme = *flagScheme + } + var err error + authorizeURL = &url.URL{ + Scheme: scheme, + Host: flagServer.Host, + Path: spec.AuthorizeV1, + RawQuery: "mid=" + *flagMID, + } + resultURL = &url.URL{ + Scheme: scheme, + Host: flagServer.Host, + Path: spec.ResultV1, + RawQuery: "mid=" + *flagMID, + } + fmt.Printf("Attempting to connect to: %s\n", authorizeURL) + kickoffMsg, err = tryConnect(authorizeURL) + rtx.Must(err, "connection failed") + } else { + targets := getTargetsFromLocate() + + for _, t := range targets { + var err error + authorizeURL, err = url.Parse(t.URLs[*flagScheme+"://"+spec.AuthorizeV1]) + rtx.Must(err, "Locate returned an invalid authorization URL") + + resultURL, err = url.Parse(t.URLs[*flagScheme+"://"+spec.ResultV1]) + rtx.Must(err, "Locate returned an invalid result URL") + + fmt.Printf("Attempting to connect to: %s\n", authorizeURL) + kickoffMsg, err = tryConnect(authorizeURL) + if err == nil { + break + } + fmt.Printf("failed to connect to %s\n", authorizeURL) + } + + if len(kickoffMsg) == 0 { + fmt.Printf("no server found") + os.Exit(1) + } + } + + // Now we have a server and a kickoff message, start the measurement. + runMeasurement(authorizeURL, resultURL, kickoffMsg) + +} diff --git a/pkg/latency1/spec/spec.go b/pkg/latency1/spec/spec.go index 602928c..dcac203 100644 --- a/pkg/latency1/spec/spec.go +++ b/pkg/latency1/spec/spec.go @@ -3,6 +3,9 @@ package spec import "time" const ( + // ServiceName is the service name for the Locate V2 API. + ServiceName = "msak/latency1" + // AuthorizeV1 is the v1 /authorize endpoint. AuthorizeV1 = "/latency/v1/authorize" // ResultV1 is the v1 /result endpoint. From e8e5455874b9b4df6155bb818ce4df249fd0c037 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 11 Jan 2024 23:39:37 +0100 Subject: [PATCH 34/37] Fix goodput calculation in msak-client (#36) * Fix goodput calculation * Emit output when a measurement comes in --- pkg/client/client.go | 29 +++++++++++------------------ pkg/client/emitter.go | 2 +- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index edb8be5..0cba60f 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -11,6 +11,7 @@ import ( "net/url" "runtime" "sync" + "sync/atomic" "time" "github.com/gorilla/websocket" @@ -90,6 +91,7 @@ type Throughput1Client struct { // sharedStartTime is the time at which the test started, shared across all streams. // It is set when the first streams connects to the server and used to compute the elapsed time. sharedStartTime time.Time + started atomic.Bool } // Result contains the aggregate metrics collected during the test. @@ -221,16 +223,15 @@ func (c *Throughput1Client) start(ctx context.Context, subtest spec.SubtestKind) go func() { // Wait for the start signal to come from any of the streams. // Returns early if the context is cancelled. - started := c.waitStart(testCtx, startTimeCh) - if !started { + + c.started.Store(c.waitStart(testCtx, startTimeCh)) + if !c.started.Load() { return } // Once at least one of the streams has started, start a timer to cancel // the context after the configured test duration. time.AfterFunc(c.config.Length, cancelTest) - - c.emitLoop(testCtx) }() // Main client loop. Spawns one goroutine per stream. @@ -267,20 +268,6 @@ func (c *Throughput1Client) waitStart(ctx context.Context, startTimeCh chan time return true } -// emitLoop emits the results every 100ms once a . It stops when the context is cancelled. -func (c *Throughput1Client) emitLoop(ctx context.Context) { - t := time.NewTicker(100 * time.Millisecond) - - for { - select { - case <-ctx.Done(): - return - case <-t.C: - c.emitResult(c.sharedStartTime) - } - } -} - func (c *Throughput1Client) runStream(ctx context.Context, streamID int, mURL *url.URL, subtest spec.SubtestKind, startTimeCh chan time.Time) error { @@ -331,6 +318,9 @@ func (c *Throughput1Client) runStream(ctx context.Context, streamID int, mURL *u streamID, m.Application.BytesReceived, m.Application.BytesSent, m.Network.BytesReceived, m.Network.BytesSent)) c.storeMeasurement(streamID, m) + if c.started.Load() { + c.emitResult(c.sharedStartTime) + } case m := <-serverCh: // If subtest is upload, store the server-side measurement. if subtest != spec.SubtestUpload { @@ -341,6 +331,9 @@ func (c *Throughput1Client) runStream(ctx context.Context, streamID int, mURL *u streamID, m.Application.BytesReceived, m.Application.BytesSent, m.Network.BytesReceived, m.Network.BytesSent)) c.storeMeasurement(streamID, m) + if c.started.Load() { + c.emitResult(c.sharedStartTime) + } case err := <-errCh: return err } diff --git a/pkg/client/emitter.go b/pkg/client/emitter.go index e0ea0d8..ee91e51 100644 --- a/pkg/client/emitter.go +++ b/pkg/client/emitter.go @@ -35,7 +35,7 @@ type HumanReadable struct { // OnResult prints the aggregate result. func (HumanReadable) OnResult(r Result) { fmt.Printf("Elapsed: %.2fs, Goodput: %f Mb/s, MinRTT: %d\n", r.Elapsed.Seconds(), - r.Goodput/1024/1024, r.MinRTT) + r.Goodput/1e6, r.MinRTT) } // OnStart is called when the stream starts and prints the subtest and server hostname. From fa019b53375ea1056ae4a085a3385a0ade17cdb7 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Thu, 11 Jan 2024 17:40:27 -0500 Subject: [PATCH 35/37] Add upload and download options (#37) --- cmd/msak-client/client.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/msak-client/client.go b/cmd/msak-client/client.go index cc5f4e4..2f65ebb 100644 --- a/cmd/msak-client/client.go +++ b/cmd/msak-client/client.go @@ -15,16 +15,18 @@ const clientName = "msak-client-go" var clientVersion = version.Version var ( - flagServer = flag.String("server", "", "Server address") - flagStreams = flag.Int("streams", client.DefaultStreams, "Number of streams") - flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") - flagDelay = flag.Duration("delay", 0, "Delay between each stream") - flagDuration = flag.Duration("duration", client.DefaultLength, "Length of the last stream") - flagScheme = flag.String("scheme", client.DefaultScheme, "Websocket scheme (wss or ws)") - flagMID = flag.String("mid", uuid.NewString(), "Measurement ID to use") - flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") - flagDebug = flag.Bool("debug", false, "Enable debug logging") + flagServer = flag.String("server", "", "Server address") + flagStreams = flag.Int("streams", client.DefaultStreams, "Number of streams") + flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") + flagDelay = flag.Duration("delay", 0, "Delay between each stream") + flagDuration = flag.Duration("duration", client.DefaultLength, "Length of the last stream") + flagScheme = flag.String("scheme", client.DefaultScheme, "Websocket scheme (wss or ws)") + flagMID = flag.String("mid", uuid.NewString(), "Measurement ID to use") + flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") + flagDebug = flag.Bool("debug", false, "Enable debug logging") flagByteLimit = flag.Int("bytes", 0, "Byte limit to request to the server") + flagUpload = flag.Bool("upload", true, "Whether to run upload test") + flagDownload = flag.Bool("download", true, "Whether to run download test") ) func main() { @@ -47,12 +49,16 @@ func main() { Emitter: client.HumanReadable{ Debug: *flagDebug, }, - NoVerify: *flagNoVerify, + NoVerify: *flagNoVerify, ByteLimit: *flagByteLimit, } cl := client.New(clientName, clientVersion, config) - cl.Download(context.Background()) - cl.Upload(context.Background()) + if *flagDownload { + cl.Download(context.Background()) + } + if *flagUpload { + cl.Upload(context.Background()) + } } From b0d0b6860ba524d412ca591ae5d84eff23d4ca5b Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Thu, 11 Jan 2024 19:23:06 -0500 Subject: [PATCH 36/37] Add minimal, multi stream support to minimal-download client (#38) * Add minimal multistream option * Add all conn logic to download This change consolidates websocket logic into the download() method so that connection start and shutdown can happen concurrently across multiple streams. As such, we checkpoint firstStart firstClose and lastStart and lastClose times as well as byte counts at significant events. With these variables, we can calculate various avg rates or a peak rates. --- cmd/minimal-download/main.go | 169 ++++++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 44 deletions(-) diff --git a/cmd/minimal-download/main.go b/cmd/minimal-download/main.go index 67e8ce3..5f7c6e0 100644 --- a/cmd/minimal-download/main.go +++ b/cmd/minimal-download/main.go @@ -14,6 +14,8 @@ import ( "net/url" "path" "runtime" + "sync" + "sync/atomic" "time" "github.com/google/uuid" @@ -27,14 +29,16 @@ const ( ) var ( - flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") - flagDuration = flag.Duration("duration", 5*time.Second, "Length of the last stream") - flagByteLimit = flag.Int("bytes", 0, "Byte limit to request to the server") - flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") - flagServerURL = flag.String("server.url", "", "URL to directly target") - flagMID = flag.String("mid", uuid.NewString(), "Measurement ID to use") - flagScheme = flag.String("scheme", "wss", "Websocket scheme (wss or ws)") - flagLocateURL = flag.String("locate.url", locateURL, "The base url for the Locate API") + flagCC = flag.String("cc", "bbr", "Congestion control algorithm to use") + flagDuration = flag.Duration("duration", 5*time.Second, "Length of the last stream") + flagMaxDuration = flag.Duration("max-duration", 15*time.Second, "Maximum length of all connections") + flagByteLimit = flag.Int("bytes", 0, "Byte limit to request to the server") + flagNoVerify = flag.Bool("no-verify", false, "Skip TLS certificate verification") + flagServerURL = flag.String("server.url", "", "URL to directly target") + flagMID = flag.String("server.mid", uuid.NewString(), "Measurement ID to use") + flagScheme = flag.String("locate.scheme", "wss", "Websocket scheme (wss or ws)") + flagLocateURL = flag.String("locate.url", locateURL, "The base url for the Locate API") + flagStreams = flag.Int("streams", 1, "The number of concurrent streams to create") ) // WireMeasurement is a wrapper for Measurement structs that contains @@ -110,9 +114,9 @@ func init() { } // connect to the given msak server URL, returning a *websocket.Conn. -func connect(ctx context.Context, s *url.URL) (*websocket.Conn, error) { +func prepareHeaders(ctx context.Context, s *url.URL) (string, http.Header) { q := s.Query() - q.Set("streams", fmt.Sprintf("%d", 1)) + q.Set("streams", fmt.Sprintf("%d", *flagStreams)) q.Set("cc", *flagCC) q.Set("bytes", fmt.Sprintf("%d", *flagByteLimit)) q.Set("duration", fmt.Sprintf("%d", (*flagDuration).Milliseconds())) @@ -126,8 +130,7 @@ func connect(ctx context.Context, s *url.URL) (*websocket.Conn, error) { headers := http.Header{} headers.Add("Sec-WebSocket-Protocol", "net.measurementlab.throughput.v1") headers.Add("User-Agent", clientName+"/"+clientVersion) - conn, _, err := localDialer.DialContext(ctx, s.String(), headers) - return conn, err + return s.String(), headers } // formatMessage reports a WireMeasurement in a human readable format. @@ -204,38 +207,65 @@ func getDownloadServer(ctx context.Context) (*url.URL, error) { return nil, errors.New("no server") } -// getConn connects to a download server, returning the *websocket.Conn. -func getConn(ctx context.Context) (*websocket.Conn, error) { - srv, err := getDownloadServer(ctx) - if err != nil { - return nil, err - } - // Connect to server. - return connect(ctx, srv) +type sharedResults struct { + bytesTotal atomic.Int64 // total bytes seen over the life of all connections. + bytesAtLastStart atomic.Int64 // total bytes seen when the last connection starts. + bytesAtFirstStop atomic.Int64 // total bytes seen when the first connection stops/closes. + minRTT atomic.Int64 // minimum of all MinRTT values from all connections. + mu sync.Mutex + started atomic.Bool // set true after first connection opens. + firstStartTime time.Time + lastStartTime time.Time + stopped atomic.Bool // set true after first connection closes (may be different than start conn). + firstStopTime time.Time + lastStopTime time.Time } -func main() { - flag.Parse() - - ctx, cancel := context.WithTimeout(context.Background(), *flagDuration*2) - defer cancel() - - conn, err := getConn(ctx) +func (s *sharedResults) download(ctx context.Context, u string, headers http.Header, wg *sync.WaitGroup, streamCount int, stream int) { + // Connect to server. + conn, _, err := localDialer.DialContext(ctx, u, headers) if err != nil { - log.Fatal(err) + log.Println("skipping one stream; fialed to connect:", err) + return + } + defer func(conn *websocket.Conn) { + // Close on return. + conn.Close() + // On return, record first and last stop times. + s.mu.Lock() // protect stopTime. + now := time.Now() + if !s.stopped.Load() { + // Stop after first connect close. + s.stopped.Store(true) + s.firstStopTime = now + s.bytesAtFirstStop.Store(s.bytesTotal.Load()) + } + // This will update for every closed stream, but the last stream to close will be the correct "lastStopTime". + s.lastStopTime = now + s.mu.Unlock() + wg.Done() + }(conn) + + // Record first and last start times. + s.mu.Lock() + now := time.Now() + if !s.started.Load() { + s.started.Store(true) + // record start time as first open connection. + s.firstStartTime = now } - defer conn.Close() + // This will update for every stream, but the last stream to update will be the correct "lastStartTime". + s.lastStartTime = now + s.bytesAtLastStart.Store(s.bytesTotal.Load()) + s.mu.Unlock() - // Max runtime. - deadline := time.Now().Add(*flagDuration * 2) + // Set absolute deadline for connections. + deadline := time.Now().Add(*flagMaxDuration) conn.SetWriteDeadline(deadline) conn.SetReadDeadline(deadline) - // receive from text & binary messages from conn until the context expires or conn closes. - var applicationBytesReceived int64 - var minRTT int64 - start := time.Now() outer: + // Receive text & binary messages from conn until the context expires or conn closes. for { select { case <-ctx.Done(): @@ -256,28 +286,79 @@ outer: log.Println("error", err) return } - applicationBytesReceived += size + s.bytesTotal.Add(size) case websocket.TextMessage: data, err := io.ReadAll(reader) if err != nil { log.Println("error", err) return } - applicationBytesReceived += int64(len(data)) + s.bytesTotal.Add(int64(len(data))) var m WireMeasurement if err := json.Unmarshal(data, &m); err != nil { log.Println("error", err) return } - formatMessage("Download server", 1, m) - minRTT = m.TCPInfo["MinRTT"] + if m.TCPInfo["MinRTT"] < s.minRTT.Load() || s.minRTT.Load() == 0 { + // NOTE: this will be the minimum of MinRTT across all streams. + s.minRTT.Store(m.TCPInfo["MinRTT"]) + } + + switch { + case streamCount == 1: + // Use server metrics for single stream tests. + formatMessage("Download server", 1, m) + case streamCount > 1 && stream == 0: + // Only do this for one stream. + elapsed := time.Since(s.firstStartTime) + log.Printf("Download client #1 - Avg %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + 8*float64(s.bytesTotal.Load())/1e6/elapsed.Seconds(), // as mbps. + float64(s.minRTT.Load())/1000.0, // as ms. + elapsed.Seconds(), 0, s.bytesTotal.Load()) + } } } } - since := time.Since(start) - log.Printf("Download client #1 - Avg %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", - 8*float64(applicationBytesReceived)/1e6/since.Seconds(), // as mbps. - float64(minRTT)/1000.0, // as ms. - since.Seconds(), 0, applicationBytesReceived) +} + +func main() { + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), *flagMaxDuration) + defer cancel() + + srv, err := getDownloadServer(ctx) + if err != nil { + log.Fatal(err) + } + // Get common URL and headers. + u, headers := prepareHeaders(ctx, srv) + log.Printf("Connecting: %s://%s%s?...", srv.Scheme, srv.Host, srv.Path) + + s := &sharedResults{} + wg := &sync.WaitGroup{} + for i := 0; i < *flagStreams; i++ { + wg.Add(1) + go s.download(ctx, u, headers, wg, *flagStreams, i) + } + wg.Wait() + + log.Println("------") + elapsedAvg := s.firstStopTime.Sub(s.firstStartTime) + bytesAvg := s.bytesAtFirstStop.Load() // like msak-client, bytes during first-start to first-stop. + log.Printf("Download client #1 - Avg %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + 8*float64(bytesAvg)/1e6/elapsedAvg.Seconds(), // as mbps. + float64(s.minRTT.Load())/1000.0, // as ms. + elapsedAvg.Seconds(), 0, bytesAvg) + + // TODO: we assume connections all overlap during peak periods. + elapsedPeak := s.firstStopTime.Sub(s.lastStartTime) + bytesPeak := s.bytesAtFirstStop.Load() - s.bytesAtLastStart.Load() // bytes during of peak period. + if *flagStreams > 1 && bytesPeak > 0 && elapsedPeak > 0 { + log.Printf("Download client #1 - Peak %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + 8*float64(bytesPeak)/1e6/elapsedPeak.Seconds(), // as mbps. + float64(s.minRTT.Load())/1000.0, // as ms. + elapsedPeak.Seconds(), 0, bytesPeak) + } } From 4010abf489d21525d74bc477607af683bf534cc2 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Fri, 12 Jan 2024 10:21:12 -0500 Subject: [PATCH 37/37] Update formatting (#39) --- cmd/minimal-download/main.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/minimal-download/main.go b/cmd/minimal-download/main.go index 5f7c6e0..605fd93 100644 --- a/cmd/minimal-download/main.go +++ b/cmd/minimal-download/main.go @@ -135,7 +135,7 @@ func prepareHeaders(ctx context.Context, s *url.URL) (string, http.Header) { // formatMessage reports a WireMeasurement in a human readable format. func formatMessage(prefix string, stream int, m WireMeasurement) { - log.Printf("%s #%d - rate %0.2f Mbps, rtt %5.2fms, elapsed %0.4fs, application r/w: %d/%d, network r/w: %d/%d kernel* r/w: %d/%d\n", + log.Printf("%s #%d rate: %0.2f Mbps, rtt %5.2fms, elapsed %0.4fs, application r/w: %d/%d, network r/w: %d/%d kernel* r/w: %d/%d\n", prefix, stream, 8*float64(m.TCPInfo["BytesAcked"])/(float64(m.ElapsedTime)), // to mbps. float64(m.TCPInfo["RTT"])/1000.0, // to ms. @@ -312,7 +312,7 @@ outer: case streamCount > 1 && stream == 0: // Only do this for one stream. elapsed := time.Since(s.firstStartTime) - log.Printf("Download client #1 - Avg %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + log.Printf("Download stream #1 rate: %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", 8*float64(s.bytesTotal.Load())/1e6/elapsed.Seconds(), // as mbps. float64(s.minRTT.Load())/1000.0, // as ms. elapsed.Seconds(), 0, s.bytesTotal.Load()) @@ -345,9 +345,16 @@ func main() { wg.Wait() log.Println("------") + elapsedTotal := s.lastStopTime.Sub(s.firstStartTime) + bytesTotal := s.bytesTotal.Load() + log.Printf("Download total average: %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + 8*float64(bytesTotal)/1e6/elapsedTotal.Seconds(), // as mbps. + float64(s.minRTT.Load())/1000.0, // as ms. + elapsedTotal.Seconds(), 0, bytesTotal) + elapsedAvg := s.firstStopTime.Sub(s.firstStartTime) bytesAvg := s.bytesAtFirstStop.Load() // like msak-client, bytes during first-start to first-stop. - log.Printf("Download client #1 - Avg %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + log.Printf("Download first average: %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", 8*float64(bytesAvg)/1e6/elapsedAvg.Seconds(), // as mbps. float64(s.minRTT.Load())/1000.0, // as ms. elapsedAvg.Seconds(), 0, bytesAvg) @@ -356,7 +363,7 @@ func main() { elapsedPeak := s.firstStopTime.Sub(s.lastStartTime) bytesPeak := s.bytesAtFirstStop.Load() - s.bytesAtLastStart.Load() // bytes during of peak period. if *flagStreams > 1 && bytesPeak > 0 && elapsedPeak > 0 { - log.Printf("Download client #1 - Peak %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", + log.Printf("Download center average: %0.2f Mbps, MinRTT %5.2fms, elapsed %0.4fs, application r/w: %d/%d\n", 8*float64(bytesPeak)/1e6/elapsedPeak.Seconds(), // as mbps. float64(s.minRTT.Load())/1000.0, // as ms. elapsedPeak.Seconds(), 0, bytesPeak)