diff --git a/clog/builder.go b/clog/builder.go index d462ae8..5fdca23 100644 --- a/clog/builder.go +++ b/clog/builder.go @@ -6,6 +6,7 @@ import ( "reflect" "github.com/alcionai/clues" + "github.com/alcionai/clues/internal/node" "github.com/alcionai/clues/internal/stringify" otellog "go.opentelemetry.io/otel/log" "go.uber.org/zap" @@ -86,7 +87,7 @@ func (b builder) log(l logLevel, msg string) { for k, v := range cv { zsl = zsl.With(k, v) - attr := clues.NewAttribute(k, v) + attr := node.NewAttribute(k, v) record.AddAttributes(attr.KV()) } @@ -94,7 +95,7 @@ func (b builder) log(l logLevel, msg string) { for k, v := range b.with { zsl.With(k, v) - attr := clues.NewAttribute(stringify.Fmt(k)[0], v) + attr := node.NewAttribute(stringify.Fmt(k)[0], v) record.AddAttributes(attr.KV()) } diff --git a/clog/builder_test.go b/clog/builder_test.go index 561cdcb..e23e989 100644 --- a/clog/builder_test.go +++ b/clog/builder_test.go @@ -126,8 +126,8 @@ func runErrorLogs( func TestGetValue(t *testing.T) { var ( - p1 int = 1 - ps string = "ptr" + p1 = 1 + ps = "ptr" pn any ) diff --git a/clog/settings.go b/clog/settings.go index 687c591..faf73d7 100644 --- a/clog/settings.go +++ b/clog/settings.go @@ -14,8 +14,6 @@ import ( // consts // --------------------------------------------------- -const clogLogFileEnv = "CLOG_LOG_FILE" - type logLevel string const ( diff --git a/clues.go b/clues.go index 065e668..7178cee 100644 --- a/clues.go +++ b/clues.go @@ -3,6 +3,7 @@ package clues import ( "context" + "github.com/alcionai/clues/internal/node" "github.com/alcionai/clues/internal/stringify" ) @@ -10,34 +11,34 @@ import ( // persistent client initialization // --------------------------------------------------------------------------- -// Initialize will spin up any persistent clients that are held by clues, -// such as OTEL communication. Clues will use these optimistically in the -// background to provide additional telemetry hook-ins. +// InitializeOTEL will spin up the OTEL clients that are held by clues, +// Clues will eagerly use these clients in the background to provide +// additional telemetry hook-ins. // -// Clues will operate as expected in the event of an error, or if initialization -// is not called. This is a purely optional step. -func Initialize( +// Clues will operate as expected in the event of an error, or if OTEL is not +// initialized. This is a purely optional step. +func InitializeOTEL( ctx context.Context, serviceName string, config OTELConfig, ) (context.Context, error) { - nc := nodeFromCtx(ctx) + nc := node.FromCtx(ctx) - err := nc.init(ctx, serviceName, config) + err := nc.InitOTEL(ctx, serviceName, config.toInternalConfig()) if err != nil { return ctx, err } - return setNodeInCtx(ctx, nc), nil + return node.EmbedInCtx(ctx, nc), nil } // Close will flush all buffered data waiting to be read. If Initialize was not // called, this call is a no-op. Should be called in a defer after initializing. func Close(ctx context.Context) error { - nc := nodeFromCtx(ctx) + nc := node.FromCtx(ctx) - if nc.otel != nil { - err := nc.otel.close(ctx) + if nc.OTEL != nil { + err := nc.OTEL.Close(ctx) if err != nil { return Wrap(err, "closing otel client") } @@ -46,14 +47,25 @@ func Close(ctx context.Context) error { return nil } +// --------------------------------------------------------------------------- +// data access +// --------------------------------------------------------------------------- + +// In retrieves the clues structured data from the context. +// TODO: turn return an interface instead of a node, have nodes +// and errors both comply with that wrapper. +func In(ctx context.Context) *node.Node { + return node.FromCtx(ctx) +} + // --------------------------------------------------------------------------- // key-value metadata // --------------------------------------------------------------------------- // Add adds all key-value pairs to the clues. func Add(ctx context.Context, kvs ...any) context.Context { - nc := nodeFromCtx(ctx) - return setNodeInCtx(ctx, nc.addValues(stringify.Normalize(kvs...))) + nc := node.FromCtx(ctx) + return node.EmbedInCtx(ctx, nc.AddValues(stringify.Normalize(kvs...))) } // AddMap adds a shallow clone of the map to a namespaced set of clues. @@ -61,14 +73,14 @@ func AddMap[K comparable, V any]( ctx context.Context, m map[K]V, ) context.Context { - nc := nodeFromCtx(ctx) + nc := node.FromCtx(ctx) kvs := make([]any, 0, len(m)*2) for k, v := range m { kvs = append(kvs, k, v) } - return setNodeInCtx(ctx, nc.addValues(stringify.Normalize(kvs...))) + return node.EmbedInCtx(ctx, nc.AddValues(stringify.Normalize(kvs...))) } // --------------------------------------------------------------------------- @@ -82,12 +94,12 @@ func AddMap[K comparable, V any]( // reference is returned mostly as a quality-of-life step // so that callers don't need to declare the map outside of // this call. -func InjectTrace[C traceMapCarrierBase]( +func InjectTrace[C node.TraceMapCarrierBase]( ctx context.Context, mapCarrier C, ) C { - nodeFromCtx(ctx). - injectTrace(ctx, asTraceMapCarrier(mapCarrier)) + node.FromCtx(ctx). + InjectTrace(ctx, node.AsTraceMapCarrier(mapCarrier)) return mapCarrier } @@ -95,12 +107,12 @@ func InjectTrace[C traceMapCarrierBase]( // ReceiveTrace extracts the current trace details from the // headers and adds them to the context. If otel is not // initialized, no-ops. -func ReceiveTrace[C traceMapCarrierBase]( +func ReceiveTrace[C node.TraceMapCarrierBase]( ctx context.Context, mapCarrier C, ) context.Context { - return nodeFromCtx(ctx). - receiveTrace(ctx, asTraceMapCarrier(mapCarrier)) + return node.FromCtx(ctx). + ReceiveTrace(ctx, node.AsTraceMapCarrier(mapCarrier)) } // AddSpan stacks a clues node onto this context and uses the provided @@ -114,27 +126,28 @@ func AddSpan( name string, kvs ...any, ) context.Context { - nc := nodeFromCtx(ctx) + nc := node.FromCtx(ctx) - var node *dataNode + var spanned *node.Node if len(kvs) > 0 { - ctx, node = nc.addSpan(ctx, name) - node.id = name - node = node.addValues(stringify.Normalize(kvs...)) + ctx, spanned = nc.AddSpan(ctx, name) + spanned.ID = name + spanned = spanned.AddValues(stringify.Normalize(kvs...)) } else { - ctx, node = nc.addSpan(ctx, name) - node = node.trace(name) + ctx, spanned = nc.AddSpan(ctx, name) + spanned = spanned.AppendToTree(name) } - return setNodeInCtx(ctx, node) + return node.EmbedInCtx(ctx, spanned) } // CloseSpan closes the current span in the clues node. Should only be called // following a `clues.AddSpan()` call. func CloseSpan(ctx context.Context) context.Context { - nc := nodeFromCtx(ctx).closeSpan(ctx) - return setNodeInCtx(ctx, nc) + return node.EmbedInCtx( + ctx, + node.FromCtx(ctx).CloseSpan(ctx)) } // --------------------------------------------------------------------------- @@ -167,10 +180,10 @@ func AddComment( msg string, vs ...any, ) context.Context { - nc := nodeFromCtx(ctx) - nn := nc.addComment(1, msg, vs...) + nc := node.FromCtx(ctx) + nn := nc.AddComment(1, msg, vs...) - return setNodeInCtx(ctx, nn) + return node.EmbedInCtx(ctx, nn) } // --------------------------------------------------------------------------- @@ -194,27 +207,27 @@ func AddAgent( ctx context.Context, name string, ) context.Context { - nc := nodeFromCtx(ctx) - nn := nc.addAgent(name) + nc := node.FromCtx(ctx) + nn := nc.AddAgent(name) - return setNodeInCtx(ctx, nn) + return node.EmbedInCtx(ctx, nn) } // Relay adds all key-value pairs to the provided agent. The agent will -// record those values to the dataNode in which it was created. All relayed +// record those values to the node in which it was created. All relayed // values are namespaced to the owning agent. func Relay( ctx context.Context, agent string, vs ...any, ) { - nc := nodeFromCtx(ctx) - ag, ok := nc.agents[agent] + nc := node.FromCtx(ctx) + ag, ok := nc.Agents[agent] if !ok { return } // set values, not add. We don't want agents to own a full clues tree. - ag.data.setValues(stringify.Normalize(vs...)) + ag.Data.SetValues(stringify.Normalize(vs...)) } diff --git a/clues_test.go b/clues_test.go index 0bf0b87..1fc1516 100644 --- a/clues_test.go +++ b/clues_test.go @@ -269,9 +269,9 @@ func TestAddSpan(t *testing.T) { ctx := context.Background() if init { - ictx, err := clues.Initialize(ctx, test.name, clues.OTELConfig{ - GRPCEndpoint: "localhost:4317", - }) + ocfg := clues.OTELConfig{GRPCEndpoint: "localhost:4317"} + + ictx, err := clues.InitializeOTEL(ctx, test.name, ocfg) if err != nil { t.Error("initializing clues", err) return @@ -326,7 +326,7 @@ func TestImmutableCtx(t *testing.T) { } pre = clues.In(testCtx) - if _, ok := preMap["k"]; ok { + if _, ok := pre.Map()["k"]; ok { t.Errorf("previous map within ctx should not have been mutated by addition") } diff --git a/datanode.go b/datanode.go deleted file mode 100644 index 2338e01..0000000 --- a/datanode.go +++ /dev/null @@ -1,680 +0,0 @@ -package clues - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "path" - "runtime" - "strings" - - "github.com/alcionai/clues/internal/stringify" - "github.com/google/uuid" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - otellog "go.opentelemetry.io/otel/log" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/trace" - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" -) - -// --------------------------------------------------------------------------- -// data nodes -// --------------------------------------------------------------------------- - -type Adder interface { - Add(key string, n int64) -} - -// dataNodes contain the data tracked by both clues in contexts and in errors. -// -// These nodes compose a tree, such that nodes can walk their ancestry path from -// leaf (the current node) to root (the highest ancestor), but not from root to -// child. This allows clues to establish sets of common ancestor data with unique -// branches for individual descendants, making the addition of new data inherently -// theadsafe. -// -// For collisions during aggregation, distance from the root denotes priority, -// with the root having the lowest priority. IE: if a child overwrites a key -// declared by an ancestor, the child's entry takes priority. -type dataNode struct { - parent *dataNode - - // otel contains the client instance for the in memory otel runtime. It is only - // present if the end user calls the clues initialization step. - otel *otelClient - - // span is the current otel Span. - // Spans are kept separately from the otelClient because we want the client to - // maintain a consistent reference to otel initialization, while the span can - // get replaced at arbitrary points. - span trace.Span - - // ids are optional and are used primarily as tracing markers. - // if empty, the trace for that node will get skipped when building the - // full trace along the node's ancestry path in the tree. - id string - - // values are they arbitrary key:value pairs that appear in clues when callers - // use the Add(ctx, k, v) or err.With(k, v) adders. Each key-value pair added - // to the node is used to produce the final set of Values() in the dataNode, - // with lower nodes in the tree taking priority over higher nodes for any - // collision resolution. - values map[string]any - - // each node can hold a single commment. The history of comments produced - // by the ancestry path through the tree will get concatenated from oldest - // ancestor to the current node to produce the comment history. - comment comment - - // agents act as proxy dataNodes that can relay specific, intentional data - // additions. They're namespaced so that additions to the agents don't accidentally - // clobber other values in the dataNode. This also allows agents to protect - // variations of data from each other, in case users need to compare differences - // on the same keys. That's not the goal for agents, exactly, but it is capable. - agents map[string]*agent -} - -// spawnDescendant generates a new dataNode that is a descendant of the current -// node. A descendant maintains a pointer to its parent, and carries any genetic -// necessities (ie, copies of fields) that must be present for continued functionality. -func (dn *dataNode) spawnDescendant() *dataNode { - agents := maps.Clone(dn.agents) - - if agents == nil && dn.agents != nil { - agents = map[string]*agent{} - } - - return &dataNode{ - parent: dn, - otel: dn.otel, - span: dn.span, - agents: agents, - } -} - -// --------------------------------------------------------------------------- -// setters -// --------------------------------------------------------------------------- - -// addValues adds all entries in the map to the dataNode's values. -// automatically propagates values onto the current span. -func (dn *dataNode) addValues(m map[string]any) *dataNode { - if m == nil { - m = map[string]any{} - } - - spawn := dn.spawnDescendant() - spawn.setValues(m) - spawn.addSpanAttributes(m) - - return spawn -} - -// setValues is a helper called by addValues. -func (dn *dataNode) setValues(m map[string]any) { - if len(m) == 0 { - return - } - - if len(dn.values) == 0 { - dn.values = map[string]any{} - } - - maps.Copy(dn.values, m) -} - -// trace adds a new leaf containing a trace ID and no other values. -func (dn *dataNode) trace(name string) *dataNode { - if name == "" { - name = makeNodeID() - } - - spawn := dn.spawnDescendant() - spawn.id = name - - return spawn -} - -// --------------------------------------------------------------------------- -// getters -// --------------------------------------------------------------------------- - -// lineage runs the fn on every valueNode in the ancestry tree, -// starting at the root and ending at the dataNode. -func (dn *dataNode) lineage(fn func(id string, vs map[string]any)) { - if dn == nil { - return - } - - if dn.parent != nil { - dn.parent.lineage(fn) - } - - fn(dn.id, dn.values) -} - -// In returns the default dataNode from the context. -// TODO: turn return an interface instead of a dataNode, have dataNodes -// and errors both comply with that wrapper. -func In(ctx context.Context) *dataNode { - return nodeFromCtx(ctx) -} - -// Map flattens the tree of dataNode.values into a map. Descendant nodes -// take priority over ancestors in cases of collision. -func (dn *dataNode) Map() map[string]any { - var ( - m = map[string]any{} - nodeIDs = []string{} - ) - - dn.lineage(func(id string, vs map[string]any) { - if len(id) > 0 { - nodeIDs = append(nodeIDs, id) - } - - for k, v := range vs { - m[k] = v - } - }) - - if len(nodeIDs) > 0 { - m["clues_trace"] = strings.Join(nodeIDs, ",") - } - - if len(dn.agents) == 0 { - return m - } - - agentVals := map[string]map[string]any{} - - for _, agent := range dn.agents { - agentVals[agent.id] = agent.data.Map() - } - - m["agents"] = agentVals - - return m -} - -// Slice flattens the tree of dataNode.values into a Slice where all even -// indices contain the keys, and all odd indices contain values. Descendant -// nodes take priority over ancestors in cases of collision. -func (dn *dataNode) Slice() []any { - m := dn.Map() - s := make([]any, 0, 2*len(m)) - - for k, v := range m { - s = append(s, k, v) - } - - return s -} - -// --------------------------------------------------------------------------- -// initialization -// --------------------------------------------------------------------------- - -// init sets up persistent clients in the clues ecosystem such as otel. -// Initialization is NOT required. It is an optional step that end -// users can take if and when they want those clients running in their -// clues instance. -// -// Multiple initializations will no-op. -func (dn *dataNode) init( - ctx context.Context, - name string, - config OTELConfig, -) error { - if dn == nil { - return nil - } - - // if any of these already exist, initialization was previously called. - if dn.otel != nil { - return nil - } - - cli, err := newOTELClient(ctx, name, config) - - dn.otel = cli - - return Stack(err).OrNil() -} - -// --------------------------------------------------------------------------- -// comments -// --------------------------------------------------------------------------- - -type comment struct { - // the func name in which the comment was created. - Caller string - // the name of the file owning the caller. - File string - // the comment message itself. - Message string -} - -// shorthand for checking if an empty comment was generated. -func (c comment) isEmpty() bool { - return len(c.Message) == 0 -} - -// newComment formats the provided values, and grabs the caller and trace -// info according to the depth. Depth is a skip-caller count, and any func -// calling this one should provide either `1` (for itself) or `depth+1` (if -// it was already given a depth value). -func newComment( - depth int, - template string, - values ...any, -) comment { - caller := getCaller(depth + 1) - _, _, parentFileLine := getDirAndFile(depth + 1) - - return comment{ - Caller: caller, - File: parentFileLine, - Message: fmt.Sprintf(template, values...), - } -} - -// addComment creates a new dataNode with a comment but no other properties. -func (dn *dataNode) addComment( - depth int, - msg string, - vs ...any, -) *dataNode { - if len(msg) == 0 { - return dn - } - - spawn := dn.spawnDescendant() - spawn.id = makeNodeID() - spawn.comment = newComment(depth+1, msg, vs...) - - return spawn -} - -// comments allows us to put a stringer on a slice of comments. -type comments []comment - -// String formats the slice of comments as a stack, much like you'd see -// with an error stacktrace. Comments are listed top-to-bottom from first- -// to-last. -// -// The format for each comment in the stack is: -// -// - : -// -func (cs comments) String() string { - result := []string{} - - for _, c := range cs { - result = append(result, c.Caller+" - "+c.File) - result = append(result, "\t"+c.Message) - } - - return strings.Join(result, "\n") -} - -// Comments retrieves the full ancestor comment chain. -// The return value is ordered from the first added comment (closest to -// the root) to the most recent one (closest to the leaf). -func (dn *dataNode) Comments() comments { - result := comments{} - - if !dn.comment.isEmpty() { - result = append(result, dn.comment) - } - - for dn.parent != nil { - dn = dn.parent - if !dn.comment.isEmpty() { - result = append(result, dn.comment) - } - } - - slices.Reverse(result) - - return result -} - -// --------------------------------------------------------------------------- -// agents -// --------------------------------------------------------------------------- - -type agent struct { - // the name of the agent - id string - - // dataNode is used here instead of a basic value map so that - // we can extend the usage of agents in the future by allowing - // the full set of dataNode behavior. We'll need a builder for that, - // but we'll get there eventually. - data *dataNode -} - -// addAgent adds a new named agent to the dataNode. -func (dn *dataNode) addAgent(name string) *dataNode { - spawn := dn.spawnDescendant() - - if len(spawn.agents) == 0 { - spawn.agents = map[string]*agent{} - } - - spawn.agents[name] = &agent{ - id: name, - // no spawn here, this needs an isolated node - data: &dataNode{}, - } - - return spawn -} - -// --------------------------------------------------------------------------- -// ctx handling -// --------------------------------------------------------------------------- - -type cluesCtxKey string - -const defaultCtxKey cluesCtxKey = "default_clues_ctx_key" - -func ctxKey(namespace string) cluesCtxKey { - return cluesCtxKey(namespace) -} - -// nodeFromCtx pulls the datanode within a given namespace out of the context. -func nodeFromCtx(ctx context.Context) *dataNode { - dn := ctx.Value(defaultCtxKey) - - if dn == nil { - return &dataNode{} - } - - return dn.(*dataNode) -} - -// setNodeInCtx embeds the dataNode in the context, and returns the updated context. -func setNodeInCtx(ctx context.Context, dn *dataNode) context.Context { - return context.WithValue(ctx, defaultCtxKey, dn) -} - -// ------------------------------------------------------------ -// span handling -// ------------------------------------------------------------ - -// traceMapCarrierBase defines the structures that support -// otel traceMapCarrier behavior. A traceMapCarrier is used -// to pass and receive traces using message delivery headers -// and other metadata. -type traceMapCarrierBase interface { - map[string]string | http.Header -} - -// asTraceMapCarrier converts a traceMapCarrier interface to -// its propagation package implementation for that structure. -// ie: map becomes a MapCarrier, headers become HeaderCarriers. -func asTraceMapCarrier[C traceMapCarrierBase]( - carrier C, -) propagation.TextMapCarrier { - if carrier == nil { - return propagation.MapCarrier{} - } - - if mss, ok := any(carrier).(map[string]string); ok { - return propagation.MapCarrier(mss) - } - - if hh, ok := any(carrier).(http.Header); ok { - return propagation.HeaderCarrier(hh) - } - - return propagation.MapCarrier{} -} - -// injectTrace adds the current trace details to the provided -// carrier. If otel is not initialized, no-ops. -// -// The carrier data is mutated by this call. -func (dn *dataNode) injectTrace( - ctx context.Context, - carrier propagation.TextMapCarrier, -) { - if dn == nil { - return - } - - otel.GetTextMapPropagator().Inject(ctx, carrier) -} - -// receiveTrace extracts the current trace details from the -// carrier and adds them to the context. If otel is not -// initialized, no-ops. -// -// The carrier data is mutated by this call. -func (dn *dataNode) receiveTrace( - ctx context.Context, - carrier propagation.TextMapCarrier, -) context.Context { - if dn == nil { - return ctx - } - - return otel.GetTextMapPropagator().Extract(ctx, carrier) -} - -// addSpan adds a new otel span. If the otel client is nil, no-ops. -// Attrs can be added to the span with addSpanAttrs. This span will -// continue to be used for that purpose until replaced with another -// span, which will appear in a separate context (and thus a separate, -// dataNode). -func (dn *dataNode) addSpan( - ctx context.Context, - name string, -) (context.Context, *dataNode) { - if dn == nil || dn.otel == nil { - return ctx, dn - } - - ctx, span := dn.otel.tracer.Start(ctx, name) - - spawn := dn.spawnDescendant() - spawn.span = span - - return ctx, spawn -} - -// closeSpan closes the otel span and removes it span from the data node. -// If no span is present, no ops. -func (dn *dataNode) closeSpan(ctx context.Context) *dataNode { - if dn == nil || dn.span == nil { - return dn - } - - dn.span.End() - - spawn := dn.spawnDescendant() - spawn.span = nil - - return spawn -} - -// addSpanAttributes adds the values to the current span. If the span -// is nil (such as if otel wasn't initialized or no span has been generated), -// this call no-ops. -func (dn *dataNode) addSpanAttributes( - values map[string]any, -) { - if dn == nil || dn.span == nil { - return - } - - for k, v := range values { - dn.span.SetAttributes(attribute.String(k, stringify.Marshal(v, false))) - } -} - -// OTELLogger gets the otel logger instance from the otel client. -// Returns nil if otel wasn't initialized. -func (dn *dataNode) OTELLogger() otellog.Logger { - if dn == nil || dn.otel == nil { - return nil - } - - return dn.otel.logger -} - -// --------------------------------------------------------------------------- -// helpers -// --------------------------------------------------------------------------- - -// makeNodeID generates a random hash of 8 characters for use as a node ID. -func makeNodeID() string { - uns := uuid.NewString() - return uns[:4] + uns[len(uns)-4:] -} - -// getDirAndFile retrieves the file and line number of the caller. -// Depth is the skip-caller count. Clues funcs that call this one should -// provide either `1` (if they do not already have a depth value), or `depth+1` -// otherwise`. -// -// formats: -// dir `absolute/os/path/to/parent/folder` -// fileAndLine `:` -// parentAndFileAndLine `/:` -func getDirAndFile( - depth int, -) (dir, fileAndLine, parentAndFileAndLine string) { - _, file, line, _ := runtime.Caller(depth + 1) - dir, file = path.Split(file) - - fileLine := fmt.Sprintf("%s:%d", file, line) - parentFileLine := fileLine - - parent := path.Base(dir) - if len(parent) > 0 { - parentFileLine = path.Join(parent, fileLine) - } - - return dir, fileLine, parentFileLine -} - -// getCaller retrieves the func name of the caller. Depth is the skip-caller -// count. Clues funcs that call this one should provide either `1` (if they -// do not already have a depth value), or `depth+1` otherwise.` -func getCaller(depth int) string { - pc, _, _, ok := runtime.Caller(depth + 1) - if !ok { - return "" - } - - funcPath := runtime.FuncForPC(pc).Name() - - // the funcpath base looks something like this: - // prefix.funcName[...].foo.bar - // with the [...] only appearing for funcs with generics. - base := path.Base(funcPath) - - // so when we split it into parts by '.', we get - // [prefix, funcName[, ], foo, bar] - parts := strings.Split(base, ".") - - // in certain conditions we'll only get the funcName - // itself, without the other parts. In that case, we - // just need to strip the generic portion from the base. - if len(parts) < 2 { - return strings.ReplaceAll(base, "[...]", "") - } - - // in most cases we'll take the 1th index (the func - // name) and trim off the bracket that remains from - // splitting on a period. - return strings.TrimSuffix(parts[1], "[") -} - -// --------------------------------------------------------------------------- -// serialization -// --------------------------------------------------------------------------- - -// nodeCore contains the serializable set of data in a dataNode. -type nodeCore struct { - OTELServiceName string `json:"otelServiceName"` - // TODO: investigate if map[string]string is really the best structure here. - // maybe we can get away with a map[string]any, or a []byte slice? - Values map[string]string `json:"values"` - Comments []comment `json:"comments"` -} - -// Bytes serializes the dataNode to a slice of bytes. -// Only attributes and comments are maintained. All -// values are stringified in the process. -// -// Node hierarchy, clients (such as otel), agents, and -// hooks (such as labelCounter) are all sliced from the -// result. -func (dn *dataNode) Bytes() ([]byte, error) { - if dn == nil { - return []byte{}, nil - } - - var serviceName string - - if dn.otel != nil { - serviceName = dn.otel.serviceName - } - - core := nodeCore{ - OTELServiceName: serviceName, - Values: map[string]string{}, - Comments: dn.Comments(), - } - - for k, v := range dn.Map() { - core.Values[k] = stringify.Marshal(v, false) - } - - return json.Marshal(core) -} - -// FromBytes deserializes the bytes to a new dataNode. -// No clients, agents, or hooks are initialized in this process. -func FromBytes(bs []byte) (*dataNode, error) { - core := nodeCore{} - - err := json.Unmarshal(bs, &core) - if err != nil { - return nil, err - } - - node := dataNode{ - // FIXME: do something with the serialized commments. - // I'm punting on this for now because I want to figure - // out the best middle ground between avoiding a slice of - // comments in each node for serialization sake (they - // are supposed to be one-comment-per-node to use the tree - // for ordering instead of the parameter), and keeping - // the full comment history available. Probably just - // need to introduce a delimiter. - } - - if len(core.Values) > 0 { - node.values = map[string]any{} - } - - for k, v := range core.Values { - node.values[k] = v - } - - if len(core.OTELServiceName) > 0 { - node.otel = &otelClient{ - serviceName: core.OTELServiceName, - } - } - - return &node, nil -} diff --git a/err.go b/err.go index 465710f..93ad513 100644 --- a/err.go +++ b/err.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" + "github.com/alcionai/clues/internal/node" "github.com/alcionai/clues/internal/stringify" "golang.org/x/exp/maps" ) @@ -41,7 +42,7 @@ type Err struct { // data is the record of contextual data produced, // presumably, at the time the error is created or wrapped. - data *dataNode + data *node.Node } // --------------------------------------------------------------------------- @@ -56,15 +57,15 @@ func newErr( m map[string]any, traceDepth int, ) *Err { - _, _, file := getDirAndFile(traceDepth + 1) + _, _, file := node.GetDirAndFile(traceDepth + 1) return &Err{ e: e, file: file, - caller: getCaller(traceDepth + 1), + caller: node.GetCaller(traceDepth + 1), msg: msg, // no ID needed for err data nodes - data: &dataNode{values: m}, + data: &node.Node{Values: m}, } } @@ -97,15 +98,15 @@ func toStack( stack []error, traceDepth int, ) *Err { - _, _, file := getDirAndFile(traceDepth + 1) + _, _, file := node.GetDirAndFile(traceDepth + 1) return &Err{ e: e, file: file, - caller: getCaller(traceDepth + 1), + caller: node.GetCaller(traceDepth + 1), stack: stack, - // no ID needed for err dataNodes - data: &dataNode{}, + // no ID needed for err nodes + data: &node.Node{}, } } @@ -211,19 +212,19 @@ func stackAncestorsOntoSelf(err error) []error { return errs } -// InErr returns the map of contextual values in the error. +// InErr returns the structured data in the error. // Each error in the stack is unwrapped and all maps are // unioned. In case of collision, lower level error data // take least priority. // TODO: remove this in favor of a type-independent In() -// that returns an interface which both dataNodes and Err +// that returns an interface which both nodes and Err // comply with. -func InErr(err error) *dataNode { +func InErr(err error) *node.Node { if isNilErrIface(err) { - return &dataNode{} + return &node.Node{} } - return &dataNode{values: inErr(err)} + return &node.Node{Values: inErr(err)} } func inErr(err error) map[string]any { @@ -246,12 +247,12 @@ func inErr(err error) map[string]any { // the error. Each error in the stack is unwrapped and all // maps are unioned. In case of collision, lower level error // data take least priority. -func (err *Err) Values() *dataNode { +func (err *Err) Values() *node.Node { if isNilErrIface(err) { - return &dataNode{} + return &node.Node{} } - return &dataNode{values: err.values()} + return &node.Node{Values: err.values()} } func (err *Err) values() map[string]any { @@ -359,18 +360,18 @@ func Labels(err error) map[string]struct{} { // ------------------------------------------------------------ // Comments retrieves all comments in the error. -func (err *Err) Comments() comments { +func (err *Err) Comments() node.CommentHistory { return Comments(err) } // Comments retrieves all comments in the error. -func Comments(err error) comments { +func Comments(err error) node.CommentHistory { if isNilErrIface(err) { - return comments{} + return node.CommentHistory{} } ancs := ancestors(err) - result := comments{} + result := node.CommentHistory{} for _, ancestor := range ancs { ce, ok := ancestor.(*Err) @@ -890,7 +891,7 @@ func (err *Err) With(kvs ...any) *Err { } if len(kvs) > 0 { - err.data = err.data.addValues(stringify.Normalize(kvs...)) + err.data = err.data.AddValues(stringify.Normalize(kvs...)) } return err @@ -911,7 +912,7 @@ func (err *Err) WithMap(m map[string]any) *Err { } if len(m) > 0 { - err.data = err.data.addValues(m) + err.data = err.data.AddValues(m) } return err @@ -951,8 +952,8 @@ func (err *Err) SkipCaller(depth int) *Err { depth = 0 } - _, _, err.file = getDirAndFile(depth + 1) - err.caller = getCaller(depth + 1) + _, _, err.file = node.GetDirAndFile(depth + 1) + err.caller = node.GetCaller(depth + 1) return err } @@ -1030,8 +1031,8 @@ func (err *Err) Comment(msg string, vs ...any) *Err { return &Err{ e: err, - // have to do a new dataNode here, or else comments will duplicate - data: &dataNode{comment: newComment(1, msg, vs...)}, + // have to do a new node here, or else comments will duplicate + data: &node.Node{Comment: node.NewComment(1, msg, vs...)}, } } @@ -1056,8 +1057,8 @@ func Comment(err error, msg string, vs ...any) *Err { return &Err{ e: err, - // have to do a new dataNode here, or else comments will duplicate - data: &dataNode{comment: newComment(1, msg, vs...)}, + // have to do a new node here, or else comments will duplicate + data: &node.Node{Comment: node.NewComment(1, msg, vs...)}, } } diff --git a/err_fmt_test.go b/err_fmt_test.go index d9ad8d9..74c38f1 100644 --- a/err_fmt_test.go +++ b/err_fmt_test.go @@ -5,7 +5,6 @@ import ( stderr "errors" "fmt" "regexp" - "strings" "testing" "github.com/pkg/errors" @@ -38,12 +37,6 @@ type checkFmt struct { reExpect *regexp.Regexp } -func prettyStack(s string) string { - s = strings.ReplaceAll(s, "\n", string('\n')) - s = strings.ReplaceAll(s, "\t", " ") - return s -} - func (c checkFmt) check(t *testing.T, err error) { t.Run(c.tmpl, func(t *testing.T) { result := fmt.Sprintf(c.tmpl, err) @@ -85,10 +78,10 @@ func makeOnion(base error, mid, top func(error) error) error { func self(err error) error { return err } var ( - errStd = stderr.New("an error") - errErrs = errors.New("an error") - fmtErrf = fmt.Errorf("an error") - cluErr = clues.New("an error") + errStd = stderr.New("an error") + errErrs = errors.New("an error") + errFMTErrf = fmt.Errorf("an error") + cluErr = clues.New("an error") cluesWrap = func(err error) error { return clues.Wrap(err, "clues wrap") } cluesPlainStack = func(err error) error { return clues.Stack(err) } @@ -167,7 +160,7 @@ func TestFmt(t *testing.T) { }, { name: "litmus wrap fmt.Errorf", - onion: makeOnion(fmtErrf, + onion: makeOnion(errFMTErrf, func(err error) error { return errors.Wrap(err, "errors wrap") }, self), expect: expect{ @@ -247,7 +240,7 @@ func TestFmt(t *testing.T) { }, { name: "fmt.Errorf", - onion: makeOnion(fmtErrf, self, self), + onion: makeOnion(errFMTErrf, self, self), expect: expect{ v: "an error", hash: `&errors.errorString{s:"an error"}`, @@ -309,7 +302,7 @@ func TestFmt(t *testing.T) { }, { name: "clues.Wrap fmt.Errorf", - onion: makeOnion(fmtErrf, cluesWrap, self), + onion: makeOnion(errFMTErrf, cluesWrap, self), expect: expect{ v: "clues wrap: an error", hash: "clues wrap: an error", @@ -372,7 +365,7 @@ func TestFmt(t *testing.T) { }, { name: "clues.PlainStack fmt.Errorf", - onion: makeOnion(fmtErrf, cluesPlainStack, self), + onion: makeOnion(errFMTErrf, cluesPlainStack, self), expect: expect{ v: "an error", hash: "an error", @@ -447,7 +440,7 @@ func TestFmt(t *testing.T) { }, { name: "clues.Stack fmt.Errorf", - onion: makeOnion(fmtErrf, cluesStack, self), + onion: makeOnion(errFMTErrf, cluesStack, self), expect: expect{ v: "sentinel: an error", hash: "sentinel: an error", @@ -537,7 +530,7 @@ func TestFmt(t *testing.T) { }, { name: "clues.Wrap clues.Stack fmt.Errorf", - onion: makeOnion(fmtErrf, cluesStack, cluesWrap), + onion: makeOnion(errFMTErrf, cluesStack, cluesWrap), expect: expect{ v: "clues wrap: sentinel: an error", hash: "clues wrap: sentinel: an error", @@ -629,7 +622,7 @@ func TestFmt(t *testing.T) { }, { name: "clues.Stack clues.Wrap fmt.Errorf", - onion: makeOnion(fmtErrf, cluesWrap, cluesStack), + onion: makeOnion(errFMTErrf, cluesWrap, cluesStack), expect: expect{ v: "sentinel: clues wrap: an error", hash: "sentinel: clues wrap: an error", @@ -1081,7 +1074,7 @@ func TestFmt_nestedFuncs(t *testing.T) { { name: "clues.Wrap fmt.Errorf", fn: topWrap, - source: fmtErrf, + source: errFMTErrf, expect: expect{ v: "top wrap: mid wrap: bottom wrap: an error", hash: "top wrap: mid wrap: bottom wrap: an error", @@ -1157,7 +1150,7 @@ func TestFmt_nestedFuncs(t *testing.T) { { name: "clues.PlainStack fmt.Errorf", fn: topPlainStack, - source: fmtErrf, + source: errFMTErrf, expect: expect{ v: "an error", hash: "an error", @@ -1240,7 +1233,7 @@ func TestFmt_nestedFuncs(t *testing.T) { { name: "clues.Stack fmt.Errorf", fn: topStack, - source: fmtErrf, + source: errFMTErrf, expect: expect{ v: "top: mid: bottom: an error", hash: "top: mid: bottom: an error", diff --git a/err_test.go b/err_test.go index a1e9238..84869ca 100644 --- a/err_test.go +++ b/err_test.go @@ -1132,63 +1132,6 @@ func TestOrNil(t *testing.T) { } } -type labelCounter map[string]int64 - -func (tla labelCounter) Add(l string, i int64) { - tla[l] = tla[l] + i -} - -var labelTable = []struct { - name string - labels []string - expect map[string]int64 -}{ - { - name: "no labels", - labels: []string{}, - expect: map[string]int64{}, - }, - { - name: "single label", - labels: []string{"un"}, - expect: map[string]int64{ - "un": 1, - }, - }, - { - name: "multiple labels", - labels: []string{"un", "deux"}, - expect: map[string]int64{ - "un": 1, - "deux": 1, - }, - }, - { - name: "duplicated label", - labels: []string{"un", "un"}, - expect: map[string]int64{ - "un": 1, - }, - }, - { - name: "multiple duplicated labels", - labels: []string{"un", "un", "deux", "deux"}, - expect: map[string]int64{ - "un": 1, - "deux": 1, - }, - }, - { - name: "empty string labels", - labels: []string{"", "", "un", "deux"}, - expect: map[string]int64{ - "": 1, - "un": 1, - "deux": 1, - }, - }, -} - // --------------------------------------------------------------------------- // helpers // --------------------------------------------------------------------------- diff --git a/errcore.go b/errcore.go index b6ea874..e213023 100644 --- a/errcore.go +++ b/errcore.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/alcionai/clues/internal/node" "github.com/alcionai/clues/internal/stringify" "golang.org/x/exp/maps" ) @@ -17,7 +18,7 @@ type ErrCore struct { Msg string `json:"msg"` Labels map[string]struct{} `json:"labels"` Values map[string]any `json:"values"` - Comments comments `json:"comments"` + Comments node.CommentHistory `json:"comments"` } // Core transforms the error into an ErrCore. diff --git a/internal/node/agents.go b/internal/node/agents.go new file mode 100644 index 0000000..a299634 --- /dev/null +++ b/internal/node/agents.go @@ -0,0 +1,33 @@ +package node + +// --------------------------------------------------------------------------- +// agents +// --------------------------------------------------------------------------- + +type Agent struct { + // the name of the agent + ID string + + // Data is used here instead of a basic value map so that + // we can extend the usage of agents in the future by allowing + // the full set of node behavior. We'll need a builder for that, + // but we'll get there eventually. + Data *Node +} + +// AddAgent adds a new named agent to the node. +func (dn *Node) AddAgent(name string) *Node { + spawn := dn.SpawnDescendant() + + if len(spawn.Agents) == 0 { + spawn.Agents = map[string]*Agent{} + } + + spawn.Agents[name] = &Agent{ + ID: name, + // no spawn here, this needs an isolated node + Data: &Node{}, + } + + return spawn +} diff --git a/internal/node/comments.go b/internal/node/comments.go new file mode 100644 index 0000000..a38a518 --- /dev/null +++ b/internal/node/comments.go @@ -0,0 +1,105 @@ +package node + +import ( + "fmt" + "slices" + "strings" +) + +// --------------------------------------------------------------------------- +// comments +// --------------------------------------------------------------------------- + +type Comment struct { + // the func name in which the comment was created. + Caller string + // the name of the file owning the caller. + File string + // the comment message itself. + Message string +} + +// shorthand for checking if an empty comment was generated. +func (c Comment) IsEmpty() bool { + return len(c.Message) == 0 +} + +// NewComment formats the provided values, and grabs the caller and trace +// info according to the depth. Depth is a skip-caller count, and any func +// calling this one should provide either `1` (for itself) or `depth+1` (if +// it was already given a depth value). +func NewComment( + depth int, + template string, + values ...any, +) Comment { + caller := GetCaller(depth + 1) + _, _, parentFileLine := GetDirAndFile(depth + 1) + + return Comment{ + Caller: caller, + File: parentFileLine, + Message: fmt.Sprintf(template, values...), + } +} + +// AddComment creates a new nodewith a comment but no other properties. +func (dn *Node) AddComment( + depth int, + msg string, + vs ...any, +) *Node { + if len(msg) == 0 { + return dn + } + + spawn := dn.SpawnDescendant() + spawn.ID = randomNodeID() + spawn.Comment = NewComment(depth+1, msg, vs...) + + return spawn +} + +// CommentHistory allows us to put a stringer on a slice of CommentHistory. +type CommentHistory []Comment + +// String formats the slice of comments as a stack, much like you'd see +// with an error stacktrace. Comments are listed top-to-bottom from first- +// to-last. +// +// The format for each comment in the stack is: +// +// - : +// +func (cs CommentHistory) String() string { + result := []string{} + + for _, c := range cs { + result = append(result, c.Caller+" - "+c.File) + result = append(result, "\t"+c.Message) + } + + return strings.Join(result, "\n") +} + +// Comments retrieves the full ancestor comment chain. +// The return value is ordered from the first added comment (closest to +// the root) to the most recent one (closest to the leaf). +func (dn *Node) Comments() CommentHistory { + result := CommentHistory{} + + if !dn.Comment.IsEmpty() { + result = append(result, dn.Comment) + } + + for dn.Parent != nil { + dn = dn.Parent + if !dn.Comment.IsEmpty() { + result = append(result, dn.Comment) + } + } + + slices.Reverse(result) + + return result +} diff --git a/internal/node/node.go b/internal/node/node.go new file mode 100644 index 0000000..59e6ccb --- /dev/null +++ b/internal/node/node.go @@ -0,0 +1,351 @@ +package node + +import ( + "context" + "encoding/json" + "strings" + + "github.com/alcionai/clues/internal/stringify" + "github.com/google/uuid" + "go.opentelemetry.io/otel/trace" + "golang.org/x/exp/maps" +) + +// --------------------------------------------------------------------------- +// data nodes +// --------------------------------------------------------------------------- + +// Node contains the data tracked by both clues in contexts and in errors. +// +// These nodes compose a tree, such that nodes can walk their ancestry path from +// leaf (the current node) to root (the highest ancestor), but not from root to +// child. This allows clues to establish sets of common ancestor data with unique +// branches for individual descendants, making the addition of new data inherently +// theadsafe. +// +// For collisions during aggregation, distance from the root denotes priority, +// with the root having the lowest priority. IE: if a child overwrites a key +// declared by an ancestor, the child's entry takes priority. +type Node struct { + Parent *Node + + // OTEL contains the client instance for the in memory OTEL runtime. It is only + // present if the end user calls the clues initialization step. + OTEL *OTELClient + + // Span is the current otel Span. + // Spans are kept separately from the otelClient because we want the client to + // maintain a consistent reference to otel initialization, while the Span can + // get replaced at arbitrary points. + Span trace.Span + + // ids are optional and are used primarily as tracing markers. + // if empty, the trace for that node will get skipped when building the + // full trace along the node's ancestry path in the tree. + ID string + + // Values are they arbitrary key:value pairs that appear in clues when callers + // use the Add(ctx, k, v) or err.With(k, v) adders. Each key-value pair added + // to the node is used to produce the final set of Values() in the node, + // with lower nodes in the tree taking priority over higher nodes for any + // collision resolution. + Values map[string]any + + // each node can hold a single commment. The history of comments produced + // by the ancestry path through the tree will get concatenated from oldest + // ancestor to the current node to produce the Comment history. + Comment Comment + + // Agents act as proxy node that can relay specific, intentional data + // additions. They're namespaced so that additions to the Agents don't accidentally + // clobber other values in the node. This also allows Agents to protect + // variations of data from each other, in case users need to compare differences + // on the same keys. That's not the goal for Agents, exactly, but it is capable. + Agents map[string]*Agent +} + +// SpawnDescendant generates a new node that is a descendant of the current +// node. A descendant maintains a pointer to its parent, and carries any genetic +// necessities (ie, copies of fields) that must be present for continued functionality. +func (dn *Node) SpawnDescendant() *Node { + agents := maps.Clone(dn.Agents) + + if agents == nil && dn.Agents != nil { + agents = map[string]*Agent{} + } + + return &Node{ + Parent: dn, + OTEL: dn.OTEL, + Span: dn.Span, + Agents: agents, + } +} + +// --------------------------------------------------------------------------- +// setters +// --------------------------------------------------------------------------- + +// AddValues adds all entries in the map to the node's values. +// automatically propagates values onto the current span. +func (dn *Node) AddValues(m map[string]any) *Node { + if m == nil { + m = map[string]any{} + } + + spawn := dn.SpawnDescendant() + spawn.SetValues(m) + spawn.AddSpanAttributes(m) + + return spawn +} + +// SetValues is generally a helper called by addValues. In +// certain corner cases (like agents) it may get called directly. +func (dn *Node) SetValues(m map[string]any) { + if len(m) == 0 { + return + } + + if len(dn.Values) == 0 { + dn.Values = map[string]any{} + } + + maps.Copy(dn.Values, m) +} + +// AppendToTree adds a new leaf with the provided name. +func (dn *Node) AppendToTree(name string) *Node { + if name == "" { + name = randomNodeID() + } + + spawn := dn.SpawnDescendant() + spawn.ID = name + + return spawn +} + +// --------------------------------------------------------------------------- +// getters +// --------------------------------------------------------------------------- + +// RunLineage runs the fn on every valueNode in the ancestry tree, +// starting at the root and ending at the node. +func (dn *Node) RunLineage(fn func(id string, vs map[string]any)) { + if dn == nil { + return + } + + if dn.Parent != nil { + dn.Parent.RunLineage(fn) + } + + fn(dn.ID, dn.Values) +} + +// Map flattens the tree of node.values into a map. Descendant nodes +// take priority over ancestors in cases of collision. +func (dn *Node) Map() map[string]any { + var ( + m = map[string]any{} + nodeIDs = []string{} + ) + + dn.RunLineage(func(id string, vs map[string]any) { + if len(id) > 0 { + nodeIDs = append(nodeIDs, id) + } + + for k, v := range vs { + m[k] = v + } + }) + + if len(nodeIDs) > 0 { + m["clues_trace"] = strings.Join(nodeIDs, ",") + } + + if len(dn.Agents) == 0 { + return m + } + + agentVals := map[string]map[string]any{} + + for _, agent := range dn.Agents { + agentVals[agent.ID] = agent.Data.Map() + } + + m["agents"] = agentVals + + return m +} + +// Slice flattens the tree of node.values into a Slice where all even +// indices contain the keys, and all odd indices contain values. Descendant +// nodes take priority over ancestors in cases of collision. +func (dn *Node) Slice() []any { + m := dn.Map() + s := make([]any, 0, 2*len(m)) + + for k, v := range m { + s = append(s, k, v) + } + + return s +} + +// --------------------------------------------------------------------------- +// initialization +// --------------------------------------------------------------------------- + +// Init sets up persistent clients in the clues ecosystem such as otel. +// Initialization is NOT required. It is an optional step that end +// users can take if and when they want those clients running in their +// clues instance. +// +// Multiple initializations will no-op. +func (dn *Node) InitOTEL( + ctx context.Context, + name string, + config OTELConfig, +) error { + if dn == nil { + return nil + } + + // if any of these already exist, initialization was previously called. + if dn.OTEL != nil { + return nil + } + + cli, err := NewOTELClient(ctx, name, config) + + dn.OTEL = cli + + if err != nil { + return err + } + + return nil +} + +// --------------------------------------------------------------------------- +// ctx handling +// --------------------------------------------------------------------------- + +type CluesCtxKey string + +const defaultCtxKey CluesCtxKey = "default_clues_ctx_key" + +func CtxKey(namespace string) CluesCtxKey { + return CluesCtxKey(namespace) +} + +// FromCtx pulls the node within a given namespace out of the context. +func FromCtx(ctx context.Context) *Node { + dn := ctx.Value(defaultCtxKey) + + if dn == nil { + return &Node{} + } + + return dn.(*Node) +} + +// EmbedInCtx adds the node in the context, and returns the updated context. +func EmbedInCtx(ctx context.Context, dn *Node) context.Context { + return context.WithValue(ctx, defaultCtxKey, dn) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +// randomNodeID generates a random hash of 8 characters for use as a node ID. +func randomNodeID() string { + uns := uuid.NewString() + return uns[:4] + uns[len(uns)-4:] +} + +// --------------------------------------------------------------------------- +// serialization +// --------------------------------------------------------------------------- + +// nodeCore contains the serializable set of data in a Node. +type nodeCore struct { + OTELServiceName string `json:"otelServiceName"` + // TODO: investigate if map[string]string is really the best structure here. + // maybe we can get away with a map[string]any, or a []byte slice? + Values map[string]string `json:"values"` + Comments []Comment `json:"comments"` +} + +// Bytes serializes the Node to a slice of bytes. +// Only attributes and comments are maintained. All +// values are stringified in the process. +// +// Node hierarchy, clients (such as otel), agents, and +// hooks (such as labelCounter) are all sliced from the +// result. +func (dn *Node) Bytes() ([]byte, error) { + if dn == nil { + return []byte{}, nil + } + + var serviceName string + + if dn.OTEL != nil { + serviceName = dn.OTEL.serviceName + } + + core := nodeCore{ + OTELServiceName: serviceName, + Values: map[string]string{}, + Comments: dn.Comments(), + } + + for k, v := range dn.Map() { + core.Values[k] = stringify.Marshal(v, false) + } + + return json.Marshal(core) +} + +// FromBytes deserializes the bytes to a new Node. +// No clients, agents, or hooks are initialized in this process. +func FromBytes(bs []byte) (*Node, error) { + core := nodeCore{} + + err := json.Unmarshal(bs, &core) + if err != nil { + return nil, err + } + + node := Node{ + // FIXME: do something with the serialized commments. + // I'm punting on this for now because I want to figure + // out the best middle ground between avoiding a slice of + // comments in each node for serialization sake (they + // are supposed to be one-comment-per-node to use the tree + // for ordering instead of the parameter), and keeping + // the full comment history available. Probably just + // need to introduce a delimiter. + } + + if len(core.Values) > 0 { + node.Values = map[string]any{} + } + + for k, v := range core.Values { + node.Values[k] = v + } + + if len(core.OTELServiceName) > 0 { + node.OTEL = &OTELClient{ + serviceName: core.OTELServiceName, + } + } + + return &node, nil +} diff --git a/datanode_test.go b/internal/node/node_test.go similarity index 79% rename from datanode_test.go rename to internal/node/node_test.go index 25f4cec..ccfe5f6 100644 --- a/datanode_test.go +++ b/internal/node/node_test.go @@ -1,4 +1,4 @@ -package clues +package node import ( "context" @@ -12,15 +12,15 @@ import ( // tests // --------------------------------------------------------------------------- -func TestDataNode_Init(t *testing.T) { +func TestNode_Init(t *testing.T) { table := []struct { name string - node *dataNode + node *Node ctx context.Context }{ { name: "nil ctx", - node: &dataNode{}, + node: &Node{}, ctx: nil, }, { @@ -30,14 +30,14 @@ func TestDataNode_Init(t *testing.T) { }, { name: "context.Context", - node: &dataNode{}, + node: &Node{}, ctx: context.Background(), }, } for _, test := range table { t.Run(test.name, func(t *testing.T) { - err := test.node.init(test.ctx, test.name, OTELConfig{}) + err := test.node.InitOTEL(test.ctx, test.name, OTELConfig{}) require.NoError(t, err) }) } @@ -46,14 +46,14 @@ func TestDataNode_Init(t *testing.T) { func TestBytes(t *testing.T) { table := []struct { name string - node func() *dataNode + node func() *Node expectSerialized []byte - expectDeserialized *dataNode + expectDeserialized *Node expectDeserializeErr require.ErrorAssertionFunc }{ { name: "nil", - node: func() *dataNode { + node: func() *Node { return nil }, expectSerialized: []byte{}, @@ -62,25 +62,25 @@ func TestBytes(t *testing.T) { }, { name: "empty", - node: func() *dataNode { - return &dataNode{} + node: func() *Node { + return &Node{} }, expectSerialized: []byte(`{"otelServiceName":"","values":{},"comments":[]}`), - expectDeserialized: &dataNode{}, + expectDeserialized: &Node{}, expectDeserializeErr: require.NoError, }, { name: "with values", - node: func() *dataNode { - return &dataNode{ - otel: &otelClient{ + node: func() *Node { + return &Node{ + OTEL: &OTELClient{ serviceName: "serviceName", }, - values: map[string]any{ + Values: map[string]any{ "fisher": "flannigan", "fitzbog": nil, }, - comment: comment{ + Comment: Comment{ Caller: "i am caller", File: "i am file", Message: "i am message", @@ -90,11 +90,11 @@ func TestBytes(t *testing.T) { expectSerialized: []byte(`{"otelServiceName":"serviceName",` + `"values":{"fisher":"flannigan","fitzbog":""},` + `"comments":[{"Caller":"i am caller","File":"i am file","Message":"i am message"}]}`), - expectDeserialized: &dataNode{ - otel: &otelClient{ + expectDeserialized: &Node{ + OTEL: &OTELClient{ serviceName: "serviceName", }, - values: map[string]any{ + Values: map[string]any{ "fisher": "flannigan", "fitzbog": "", }, diff --git a/internal/node/otel.go b/internal/node/otel.go new file mode 100644 index 0000000..66a2215 --- /dev/null +++ b/internal/node/otel.go @@ -0,0 +1,322 @@ +package node + +import ( + "context" + "fmt" + "net/http" + + "github.com/alcionai/clues/internal/stringify" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdkTrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// ------------------------------------------------------------ +// client +// ------------------------------------------------------------ + +type OTELClient struct { + grpcConn *grpc.ClientConn + traceProvider *sdkTrace.TracerProvider + tracer trace.Tracer + logger otellog.Logger + serviceName string +} + +func (cli *OTELClient) Close(ctx context.Context) error { + if cli == nil { + return nil + } + + if cli.traceProvider != nil { + err := cli.traceProvider.ForceFlush(ctx) + if err != nil { + fmt.Println("forcing trace provider flush:", err) + } + + err = cli.traceProvider.Shutdown(ctx) + if err != nil { + return fmt.Errorf("shutting down otel trace provider: %w", err) + } + } + + if cli.grpcConn != nil { + err := cli.grpcConn.Close() + if err != nil { + return fmt.Errorf("closing grpc connection: %w", err) + } + } + + return nil +} + +// ------------------------------------------------------------ +// config +// ------------------------------------------------------------ + +type OTELConfig struct { + // specify the endpoint location to use for grpc communication. + // If empty, no telemetry exporter will be generated. + // ex: localhost:4317 + // ex: 0.0.0.0:4317 + GRPCEndpoint string +} + +// ------------------------------------------------------------ +// initializers +// ------------------------------------------------------------ + +// NewOTELClient bootstraps the OpenTelemetry pipeline to run against a +// local server instance. If it does not return an error, make sure +// to call the client.Close() method for proper cleanup. +// The service name is used to match traces across backends. +func NewOTELClient( + ctx context.Context, + serviceName string, + config OTELConfig, +) (*OTELClient, error) { + // -- Resource + srvResource, err := resource.New(ctx, resource.WithAttributes( + semconv.ServiceNameKey.String(serviceName))) + if err != nil { + return nil, fmt.Errorf("creating otel resource: %w", err) + } + + // -- Exporter + + conn, err := grpc.NewClient( + config.GRPCEndpoint, + // Note the use of insecure transport here. TLS is recommended in production. + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("creating new grpc connection: %w", err) + } + + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) + if err != nil { + return nil, fmt.Errorf("creating a trace exporter: %w", err) + } + + // -- TracerProvider + + // Register the trace exporter with a TracerProvider, using a batch + // span processor to aggregate spans before export. + batchSpanProcessor := sdkTrace.NewBatchSpanProcessor(exporter) + + tracerProvider := sdkTrace.NewTracerProvider( + sdkTrace.WithResource(srvResource), + sdkTrace.WithSampler(sdkTrace.AlwaysSample()), + sdkTrace.WithSpanProcessor(batchSpanProcessor), + sdkTrace.WithRawSpanLimits(sdkTrace.SpanLimits{ + AttributeValueLengthLimit: -1, + AttributeCountLimit: -1, + AttributePerEventCountLimit: -1, + AttributePerLinkCountLimit: -1, + EventCountLimit: -1, + LinkCountLimit: -1, + })) + + // set global propagator to traceContext (the default is no-op). + otel.SetTextMapPropagator(propagation.TraceContext{}) + otel.SetTracerProvider(tracerProvider) + + // -- Logger + + // generate a logger provider + logProvider := global.GetLoggerProvider() + + // -- Client + + client := OTELClient{ + grpcConn: conn, + traceProvider: tracerProvider, + tracer: tracerProvider.Tracer(serviceName + "/tracer"), + logger: logProvider.Logger(serviceName), + } + + // Shutdown will flush any remaining spans and shut down the exporter. + return &client, nil +} + +// ------------------------------------------------------------ +// annotations. basically otel's version of With() +// Not currently used; we're just mashing everything in as a +// string right now, same as clues does. +// ------------------------------------------------------------ + +type Annotation struct { + kind string + k string + v any +} + +func NewAttribute(k string, v any) Annotation { + return Annotation{ + kind: "attribute", + k: k, + v: v, + } +} + +func (a Annotation) IsAttribute() bool { + return a.kind == "attribute" +} + +func (a Annotation) KV() otellog.KeyValue { + if a.kind != "attribute" { + return otellog.KeyValue{} + } + + // FIXME: needs extensive type support + switch a.v.(type) { + case int: + return otellog.Int(a.k, a.v.(int)) + case int64: + return otellog.Int64(a.k, a.v.(int64)) + case string: + return otellog.String(a.k, a.v.(string)) + case bool: + return otellog.Bool(a.k, a.v.(bool)) + default: // everything else gets stringified + return otellog.String(a.k, stringify.Marshal(a.v, false)) + } +} + +type Annotationer interface { + IsAttribute() bool + KV() attribute.KeyValue +} + +// ------------------------------------------------------------ +// span handling +// ------------------------------------------------------------ + +// TraceMapCarrierBase defines the structures that support +// otel TraceMapCarrier behavior. A traceMapCarrier is used +// to pass and receive traces using message delivery headers +// and other metadata. +type TraceMapCarrierBase interface { + map[string]string | http.Header +} + +// AsTraceMapCarrier converts a traceMapCarrier interface to +// its propagation package implementation for that structure. +// ie: map becomes a MapCarrier, headers become HeaderCarriers. +func AsTraceMapCarrier[C TraceMapCarrierBase]( + carrier C, +) propagation.TextMapCarrier { + if carrier == nil { + return propagation.MapCarrier{} + } + + if mss, ok := any(carrier).(map[string]string); ok { + return propagation.MapCarrier(mss) + } + + if hh, ok := any(carrier).(http.Header); ok { + return propagation.HeaderCarrier(hh) + } + + return propagation.MapCarrier{} +} + +// injectTrace adds the current trace details to the provided +// carrier. If otel is not initialized, no-ops. +// +// The carrier data is mutated by this call. +func (dn *Node) InjectTrace( + ctx context.Context, + carrier propagation.TextMapCarrier, +) { + if dn == nil { + return + } + + otel.GetTextMapPropagator().Inject(ctx, carrier) +} + +// receiveTrace extracts the current trace details from the +// carrier and adds them to the context. If otel is not +// initialized, no-ops. +// +// The carrier data is mutated by this call. +func (dn *Node) ReceiveTrace( + ctx context.Context, + carrier propagation.TextMapCarrier, +) context.Context { + if dn == nil { + return ctx + } + + return otel.GetTextMapPropagator().Extract(ctx, carrier) +} + +// AddSpan adds a new otel span. If the otel client is nil, no-ops. +// Attrs can be added to the span with addSpanAttrs. This span will +// continue to be used for that purpose until replaced with another +// span, which will appear in a separate context (and thus a separate, +// node). +func (dn *Node) AddSpan( + ctx context.Context, + name string, +) (context.Context, *Node) { + if dn == nil || dn.OTEL == nil { + return ctx, dn + } + + ctx, span := dn.OTEL.tracer.Start(ctx, name) + + spawn := dn.SpawnDescendant() + spawn.Span = span + + return ctx, spawn +} + +// CloseSpan closes the otel span and removes it span from the data node. +// If no span is present, no ops. +func (dn *Node) CloseSpan(ctx context.Context) *Node { + if dn == nil || dn.Span == nil { + return dn + } + + dn.Span.End() + + spawn := dn.SpawnDescendant() + spawn.Span = nil + + return spawn +} + +// AddSpanAttributes adds the values to the current span. If the span +// is nil (such as if otel wasn't initialized or no span has been generated), +// this call no-ops. +func (dn *Node) AddSpanAttributes( + values map[string]any, +) { + if dn == nil || dn.Span == nil { + return + } + + for k, v := range values { + dn.Span.SetAttributes(attribute.String(k, stringify.Marshal(v, false))) + } +} + +// OTELLogger gets the otel logger instance from the otel client. +// Returns nil if otel wasn't initialized. +func (dn *Node) OTELLogger() otellog.Logger { + if dn == nil || dn.OTEL == nil { + return nil + } + + return dn.OTEL.logger +} diff --git a/internal/node/stacktrace.go b/internal/node/stacktrace.go new file mode 100644 index 0000000..21e81f3 --- /dev/null +++ b/internal/node/stacktrace.go @@ -0,0 +1,67 @@ +package node + +import ( + "fmt" + "path" + "runtime" + "strings" +) + +// GetDirAndFile retrieves the file and line number of the caller. +// Depth is the skip-caller count. Clues funcs that call this one should +// provide either `1` (if they do not already have a depth value), or `depth+1` +// otherwise`. +// +// formats: +// dir `absolute/os/path/to/parent/folder` +// fileAndLine `:` +// parentAndFileAndLine `/:` +func GetDirAndFile( + depth int, +) (dir, fileAndLine, parentAndFileAndLine string) { + _, file, line, _ := runtime.Caller(depth + 1) + dir, file = path.Split(file) + + fileLine := fmt.Sprintf("%s:%d", file, line) + parentFileLine := fileLine + + parent := path.Base(dir) + if len(parent) > 0 { + parentFileLine = path.Join(parent, fileLine) + } + + return dir, fileLine, parentFileLine +} + +// GetCaller retrieves the func name of the caller. Depth is the skip-caller +// count. Clues funcs that call this one should provide either `1` (if they +// do not already have a depth value), or `depth+1` otherwise.` +func GetCaller(depth int) string { + pc, _, _, ok := runtime.Caller(depth + 1) + if !ok { + return "" + } + + funcPath := runtime.FuncForPC(pc).Name() + + // the funcpath base looks something like this: + // prefix.funcName[...].foo.bar + // with the [...] only appearing for funcs with generics. + base := path.Base(funcPath) + + // so when we split it into parts by '.', we get + // [prefix, funcName[, ], foo, bar] + parts := strings.Split(base, ".") + + // in certain conditions we'll only get the funcName + // itself, without the other parts. In that case, we + // just need to strip the generic portion from the base. + if len(parts) < 2 { + return strings.ReplaceAll(base, "[...]", "") + } + + // in most cases we'll take the 1th index (the func + // name) and trim off the bracket that remains from + // splitting on a period. + return strings.TrimSuffix(parts[1], "[") +} diff --git a/otel.go b/otel.go index b92e752..77397f1 100644 --- a/otel.go +++ b/otel.go @@ -1,190 +1,25 @@ package clues -import ( - "context" - "fmt" +import "github.com/alcionai/clues/internal/node" - "github.com/alcionai/clues/internal/stringify" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - otellog "go.opentelemetry.io/otel/log" - "go.opentelemetry.io/otel/log/global" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdkTrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.26.0" - "go.opentelemetry.io/otel/trace" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -type otelClient struct { - serviceName string - - grpcConn *grpc.ClientConn - traceProvider *sdkTrace.TracerProvider - tracer trace.Tracer - logger otellog.Logger -} - -func (cli *otelClient) close(ctx context.Context) error { - if cli == nil { - return nil - } - - if cli.traceProvider != nil { - err := cli.traceProvider.ForceFlush(ctx) - if err != nil { - fmt.Println("forcing trace provider flush:", err) - } - - err = cli.traceProvider.Shutdown(ctx) - if err != nil { - return WrapWC(ctx, err, "shutting down otel trace provider") - } - } - - if cli.grpcConn != nil { - err := cli.grpcConn.Close() - if err != nil { - return WrapWC(ctx, err, "closing grpc connection") - } - } +var ErrMissingOtelGRPCEndpoint = New("missing otel grpc endpoint").NoTrace() - return nil -} - -// ------------------------------------------------------------ -// initializers -// ------------------------------------------------------------ +const ( + DefaultOTELGRPCEndpoint = "localhost:4317" +) type OTELConfig struct { // specify the endpoint location to use for grpc communication. // If empty, no telemetry exporter will be generated. // ex: localhost:4317 // ex: 0.0.0.0:4317 + // ex: opentelemetry-collector.monitoring.svc.cluster.local:4317 GRPCEndpoint string } -// newOTELClient bootstraps the OpenTelemetry pipeline to run against a -// local server instance. If it does not return an error, make sure -// to call the client.Close() method for proper cleanup. -// The service name is used to match traces across backends. -func newOTELClient( - ctx context.Context, - serviceName string, - config OTELConfig, -) (*otelClient, error) { - // -- Resource - srvResource, err := resource.New(ctx, resource.WithAttributes( - semconv.ServiceNameKey.String(serviceName))) - if err != nil { - return nil, WrapWC(ctx, err, "creating otel resource") - } - - // -- Exporter - - conn, err := grpc.NewClient( - config.GRPCEndpoint, - // Note the use of insecure transport here. TLS is recommended in production. - grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - return nil, WrapWC(ctx, err, "creating new grpc connection") - } - - exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn)) - if err != nil { - return nil, WrapWC(ctx, err, "creating a trace exporter") - } - - // -- TracerProvider - - // Register the trace exporter with a TracerProvider, using a batch - // span processor to aggregate spans before export. - batchSpanProcessor := sdkTrace.NewBatchSpanProcessor(exporter) - - tracerProvider := sdkTrace.NewTracerProvider( - sdkTrace.WithResource(srvResource), - sdkTrace.WithSampler(sdkTrace.AlwaysSample()), - sdkTrace.WithSpanProcessor(batchSpanProcessor), - sdkTrace.WithRawSpanLimits(sdkTrace.SpanLimits{ - AttributeValueLengthLimit: -1, - AttributeCountLimit: -1, - AttributePerEventCountLimit: -1, - AttributePerLinkCountLimit: -1, - EventCountLimit: -1, - LinkCountLimit: -1, - })) - - // set global propagator to traceContext (the default is no-op). - otel.SetTextMapPropagator(propagation.TraceContext{}) - otel.SetTracerProvider(tracerProvider) - - // -- Logger - - // generate a logger provider - logProvider := global.GetLoggerProvider() - - // -- Client - - client := otelClient{ - serviceName: serviceName, - grpcConn: conn, - traceProvider: tracerProvider, - tracer: tracerProvider.Tracer(serviceName + "/tracer"), - logger: logProvider.Logger(serviceName), - } - - // Shutdown will flush any remaining spans and shut down the exporter. - return &client, nil -} - -// ------------------------------------------------------------ -// annotations. basically otel's version of With() -// Not currently used; we're just mashing everything in as a -// string right now, same as clues does. -// ------------------------------------------------------------ - -type annotation struct { - kind string - k string - v any -} - -func NewAttribute(k string, v any) annotation { - return annotation{ - kind: "attribute", - k: k, - v: v, +// clues.OTELConfig is a passthrough to the internal otel config. +func (oc OTELConfig) toInternalConfig() node.OTELConfig { + return node.OTELConfig{ + GRPCEndpoint: oc.GRPCEndpoint, } } - -func (a annotation) IsAttribute() bool { - return a.kind == "attribute" -} - -func (a annotation) KV() otellog.KeyValue { - if a.kind != "attribute" { - return otellog.KeyValue{} - } - - // FIXME: needs extensive type support - switch a.v.(type) { - case int: - return otellog.Int(a.k, a.v.(int)) - case int64: - return otellog.Int64(a.k, a.v.(int64)) - case string: - return otellog.String(a.k, a.v.(string)) - case bool: - return otellog.Bool(a.k, a.v.(bool)) - default: // everything else gets stringified - return otellog.String(a.k, stringify.Marshal(a.v, false)) - } -} - -type Annotationer interface { - IsAttribute() bool - KV() attribute.KeyValue -}