From 605bf78cc278a11c56cc680fbbdf980ae8fc798b Mon Sep 17 00:00:00 2001 From: xhd2015 Date: Fri, 12 Apr 2024 21:35:52 +0800 Subject: [PATCH 1/3] enable trace filter --- runtime/test/trace_marshal/decyclic_test.go | 85 +++++++++++++ runtime/test/trace_marshal/marshal_test.go | 22 ++++ runtime/trace/marshal.go | 128 ++++++++++++++++++++ runtime/trace/stack.go | 48 +++++--- runtime/trace/trace.go | 62 ++++++---- 5 files changed, 307 insertions(+), 38 deletions(-) create mode 100644 runtime/test/trace_marshal/decyclic_test.go diff --git a/runtime/test/trace_marshal/decyclic_test.go b/runtime/test/trace_marshal/decyclic_test.go new file mode 100644 index 00000000..439cfce2 --- /dev/null +++ b/runtime/test/trace_marshal/decyclic_test.go @@ -0,0 +1,85 @@ +package trace_marshal + +import ( + "fmt" + "reflect" + "testing" + + "github.com/xhd2015/xgo/runtime/trace" +) + +type _struct struct { + self *_struct +} + +func TestDecyclicUnexported(t *testing.T) { + s := &_struct{} + s.self = s + + vs := trace.Decyclic(s) + if vs.(*_struct).self != nil { + t.Fatalf("expect vs.self to be nil") + } +} + +type _structMap struct { + m map[string]*_structMap +} + +func TestDecyclicMapValue(t *testing.T) { + s := &_structMap{ + m: make(map[string]*_structMap), + } + s.m["self"] = s + + v1 := reflect.ValueOf(s).Interface() + if v1.(*_structMap).m["self"] == nil { + t.Fatalf("expect v1.m['self'] to be non nil") + } + + vs := trace.Decyclic(s) + if vs.(*_structMap).m["self"] != nil { + t.Fatalf("expect vs.m['self'] to be nil") + } +} + +type _interface struct { + self interface{} +} + +func TestDecyclicInterface(t *testing.T) { + s := &_interface{} + s.self = s + + vs := trace.Decyclic(s) + if vs.(*_interface).self != nil { + t.Fatalf("expect vs.self to be nil") + } +} + +type _valueStruct struct { + A int +} + +func TestDecyclicValueStruct(t *testing.T) { + s := _valueStruct{ + A: 123, + } + + vs := trace.Decyclic(s) + vsA := vs.(_valueStruct).A + if vsA != 123 { + t.Fatalf("vs.A: %v", vsA) + } +} + +func TestDecyclicMapInterface(t *testing.T) { + m := make(map[string]interface{}) + m["A"] = 123 + + vs := trace.Decyclic(m) + vsA := vs.(map[string]interface{})["A"] + if fmt.Sprint(vsA) != "123" { + t.Fatalf("vs.A: %v", vsA) + } +} diff --git a/runtime/test/trace_marshal/marshal_test.go b/runtime/test/trace_marshal/marshal_test.go index bdd15adb..8bbb1bf4 100644 --- a/runtime/test/trace_marshal/marshal_test.go +++ b/runtime/test/trace_marshal/marshal_test.go @@ -67,6 +67,28 @@ func TestMarshalAnyJSON(t *testing.T) { } } +type cyclic struct { + Self *cyclic + Name string +} + +func TestMarshalCyclicJSON(t *testing.T) { + c := &cyclic{ + Name: "cyclic", + } + c.Self = c + + res, err := trace.MarshalAnyJSON(c) + if err != nil { + t.Fatal(err) + } + resStr := string(res) + expect := `{"Self":null,"Name":"cyclic"}` + if resStr != expect { + t.Fatalf("expect res to be %q, actual: %q", expect, resStr) + } +} + func exampleReturnFunc() context.CancelFunc { _, f := context.WithTimeout(context.TODO(), 10*time.Millisecond) return f diff --git a/runtime/trace/marshal.go b/runtime/trace/marshal.go index fe8f82d0..1eb8bd1b 100644 --- a/runtime/trace/marshal.go +++ b/runtime/trace/marshal.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "reflect" + "unsafe" "github.com/xhd2015/xgo/runtime/core" "github.com/xhd2015/xgo/runtime/functab" @@ -22,6 +23,11 @@ func MarshalAnyJSON(v interface{}) ([]byte, error) { return json.Marshal(v) } + // can be done via filter + if false { + v = Decyclic(v) + } + // get the unmarshalable function unmarshalable := getMarshaler(funcInfo.Func, reflect.TypeOf(unmarshalableFunc)) var data []byte @@ -43,6 +49,7 @@ func MarshalAnyJSON(v interface{}) ([]byte, error) { }, func() { data, err = json.Marshal(v) }) + return data, err } @@ -60,3 +67,124 @@ func getMarshaler(newTypeEncoder interface{}, v reflect.Type) interface{} { }) return res } + +type decyclicer struct { + seen map[uintptr]struct{} +} + +func Decyclic(v interface{}) interface{} { + if v == nil { + return nil + } + d := &decyclicer{ + seen: map[uintptr]struct{}{}, + } + d.clear(reflect.ValueOf(v), func(r reflect.Value) { + v = r.Interface() + }) + return v +} + +func makeAddrable(v reflect.Value, set func(r reflect.Value)) reflect.Value { + if v.CanAddr() { + return v + } + p := reflect.New(v.Type()) + p.Elem().Set(v) + x := p.Elem() + set(x) + return x +} + +func (c *decyclicer) clear(v reflect.Value, set func(r reflect.Value)) { + // fmt.Printf("clear: %v\n", v.Type()) + switch v.Kind() { + case reflect.Ptr: + if v.IsNil() { + return + } + + // only pointer can create cyclic + ptr := v.Pointer() + if ptr == 0 { + return + } + if _, ok := c.seen[ptr]; ok { + // fmt.Printf("found : 0x%x -> %v\n", ptr, v.Interface()) + set(reflect.Zero(v.Type())) + return + } + c.seen[ptr] = struct{}{} + defer delete(c.seen, ptr) + + v = makeAddrable(v, set) + c.clear(v.Elem(), func(r reflect.Value) { + v.Elem().Set(r) + }) + case reflect.Interface: + if v.IsNil() { + return + } + v = makeAddrable(v, set) + c.clear(v.Elem(), func(r reflect.Value) { + // NOTE: interface{} is special + // we can directly can call v.Set + // instead of v.Elem().Set() + v.Set(r) + if v.Elem().Kind() == reflect.Ptr && v.Elem().IsNil() { + // fmt.Printf("found isNil\n") + // avoid {nil-value,non-nil type} + set(reflect.Zero(v.Type())) + } + }) + case reflect.Array, reflect.Slice: + switch v.Type().Elem().Kind() { + case reflect.Int64, reflect.Int, reflect.Int32, reflect.Int16, reflect.Int8, + reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uint8, + reflect.Float64, reflect.Float32, + reflect.String, + reflect.Bool: + return + } + v = makeAddrable(v, set) + for i := 0; i < v.Len(); i++ { + e := v.Index(i) + c.clear(e, func(r reflect.Value) { + e.Set(r) + }) + } + case reflect.Map: + v = makeAddrable(v, set) + iter := v.MapRange() + // sets := [][2]reflect.Value{} + for iter.Next() { + vi := v.MapIndex(iter.Key()) + c.clear(vi, func(r reflect.Value) { + v.SetMapIndex(iter.Key(), r) + }) + } + case reflect.Struct: + // fmt.Printf("struct \n") + // make struct addrable + v = makeAddrable(v, set) + + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + if field.CanSet() { + c.clear(field, func(r reflect.Value) { + field.Set(r) + }) + } else { + e := reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())) + c.clear(e.Elem(), func(r reflect.Value) { + e.Elem().Set(r) + }) + // panic(fmt.Errorf("cannot set: %v", field)) + } + } + case reflect.Chan, reflect.Func: + // ignore + default: + // int + } +} diff --git a/runtime/trace/stack.go b/runtime/trace/stack.go index d23662b3..7b01825a 100644 --- a/runtime/trace/stack.go +++ b/runtime/trace/stack.go @@ -19,40 +19,49 @@ type Stack struct { Begin int64 // us End int64 // us - Args core.Object - Results core.Object - Panic bool - Error error - // Recv interface{} - // Args []interface{} - // Results []interface{} + Args core.Object + Results core.Object + Panic bool + Error error Children []*Stack } -func (c *Root) Export() *RootExport { +// allow skip some packages +// +// for example: google.golang.org/protobuf/internal/order +type ExportOptions struct { + FilterStack func(stack *StackExport) *StackExport +} + +func (c *Root) Export(opts *ExportOptions) *RootExport { if c == nil { return nil } return &RootExport{ Begin: c.Begin, - Children: (stacks)(c.Children).Export(), + Children: (stacks)(c.Children).Export(opts), } } type stacks []*Stack -func (c stacks) Export() []*StackExport { +func (c stacks) Export(opts *ExportOptions) []*StackExport { if c == nil { return nil } - list := make([]*StackExport, len(c)) + list := make([]*StackExport, 0, len(c)) for i := 0; i < len(c); i++ { - list[i] = c[i].Export() + stackPoint := c[i] + exportStack := stackPoint.Export(opts) + if exportStack == nil { + continue + } + list = append(list, exportStack) } return list } -func (c *Stack) Export() *StackExport { +func (c *Stack) Export(opts *ExportOptions) *StackExport { if c == nil { return nil } @@ -60,19 +69,24 @@ func (c *Stack) Export() *StackExport { if c.Error != nil { errMsg = c.Error.Error() } - return &StackExport{ - FuncInfo: ExportFuncInfo(c.FuncInfo), + stack := &StackExport{ + FuncInfo: ExportFuncInfo(c.FuncInfo, opts), Begin: c.Begin, End: c.End, Args: c.Args, Results: c.Results, Panic: c.Panic, Error: errMsg, - Children: (stacks)(c.Children).Export(), + Children: ((stacks)(c.Children)).Export(opts), + } + + if opts != nil && opts.FilterStack != nil { + return opts.FilterStack(stack) } + return stack } -func ExportFuncInfo(c *core.FuncInfo) *FuncInfoExport { +func ExportFuncInfo(c *core.FuncInfo, opts *ExportOptions) *FuncInfoExport { if c == nil { return nil } diff --git a/runtime/trace/trace.go b/runtime/trace/trace.go index e03bd5b2..e06d9fb3 100644 --- a/runtime/trace/trace.go +++ b/runtime/trace/trace.go @@ -75,13 +75,7 @@ var interceptorSet int32 // globally. Otherwise locally func Enable() func() { if __xgo_link_init_finished() { - var name string - key := uintptr(__xgo_link_getcurg()) - tinfo, ok := testInfoMapping.Load(key) - if ok { - name = tinfo.(*testInfo).name - } - return enableLocal(&collectOpts{name: name}) + return enableLocal(nil) } enabledGlobally = true setupInterceptor() @@ -100,10 +94,18 @@ func Collect(f func()) { collect(f, &collectOpts{}) } +func Begin(f func()) func() { + if !__xgo_link_init_finished() { + panic("Begin cannot be called from init") + } + return enableLocal(&collectOpts{}) +} + type collectOpts struct { - name string - onComplete func(root *Root) - root *Root + name string + onComplete func(root *Root) + root *Root + exportOptions *ExportOptions } func Options() *collectOpts { @@ -120,10 +122,19 @@ func (c *collectOpts) OnComplete(f func(root *Root)) *collectOpts { return c } +func (c *collectOpts) WithExport(expOpts *ExportOptions) *collectOpts { + c.exportOptions = expOpts + return c +} + func (c *collectOpts) Collect(f func()) { collect(f, c) } +func (c *collectOpts) Begin() func() { + return enableLocal(c) +} + func setupInterceptor() { if !atomic.CompareAndSwapInt32(&interceptorSet, 0, 1) { return @@ -252,7 +263,7 @@ func setupInterceptor() { // global stackMap.Delete(key) - emitTraceNoErr("", root) + emitTraceNoErr("", root, nil) return nil } // pop stack @@ -269,8 +280,8 @@ type optStack struct { } func collect(f func(), collOpts *collectOpts) { - cancel := enableLocal(collOpts) - defer cancel() + finish := enableLocal(collOpts) + defer finish() f() } @@ -281,7 +292,16 @@ func enableLocal(collOpts *collectOpts) func() { setupInterceptor() key := uintptr(__xgo_link_getcurg()) if collOpts.name == "" { - collOpts.name = fmt.Sprintf("g_%x", uint(key)) + var name string + key := uintptr(__xgo_link_getcurg()) + tinfo, ok := testInfoMapping.Load(key) + if ok { + name = tinfo.(*testInfo).name + } + if name == "" { + name = fmt.Sprintf("g_%x", uint(key)) + } + collOpts.name = name } if collOpts.root == nil { collOpts.root = &Root{ @@ -311,7 +331,7 @@ func enableLocal(collOpts *collectOpts) func() { if collOpts.onComplete != nil { collOpts.onComplete(root) } else { - emitTraceNoErr(collOpts.name, root) + emitTraceNoErr(collOpts.name, root, collOpts.exportOptions) } } } @@ -328,7 +348,7 @@ func SetMarshalStack(fn func(root *Root) ([]byte, error)) { marshalStack = fn } -func fmtStack(root *Root) (data []byte, err error) { +func fmtStack(root *Root, opts *ExportOptions) (data []byte, err error) { defer func() { if e := recover(); e != nil { if pe, ok := e.(error); ok { @@ -342,11 +362,11 @@ func fmtStack(root *Root) (data []byte, err error) { if marshalStack != nil { return marshalStack(root) } - return MarshalAnyJSON(root.Export()) + return MarshalAnyJSON(root.Export(opts)) } -func emitTraceNoErr(name string, root *Root) { - emitTrace(name, root) +func emitTraceNoErr(name string, root *Root, opts *ExportOptions) { + emitTrace(name, root, opts) } func getNow() (now time.Time) { @@ -364,7 +384,7 @@ func formatTime(t time.Time, layout string) (output string) { // this should also be marked as trap.Skip() // TODO: may add callback for this -func emitTrace(name string, root *Root) error { +func emitTrace(name string, root *Root, opts *ExportOptions) error { xgoTraceOutput := getTraceOutput() if xgoTraceOutput == "off" { return nil @@ -389,7 +409,7 @@ func emitTrace(name string, root *Root) error { fmt.Printf("%s: ", subName) } var traceOut []byte - trace, stackErr := fmtStack(root) + trace, stackErr := fmtStack(root, opts) if stackErr != nil { traceOut = []byte("error:" + stackErr.Error()) } else { From b95076921b299cab29e79a1642abf005462fb732 Mon Sep 17 00:00:00 2001 From: xhd2015 Date: Sat, 13 Apr 2024 12:46:12 +0800 Subject: [PATCH 2/3] add filter options when exporting stacks --- cmd/xgo/version.go | 4 +- runtime/core/version.go | 4 +- runtime/test/trace_marshal/decyclic.go | 177 ++++++++++++++++++ runtime/test/trace_marshal/decyclic_test.go | 35 +++- .../trace_panic_peek/trace_panic_peek_test.go | 2 +- runtime/trace/marshal.go | 142 +------------- runtime/trace/stack.go | 2 + runtime/trace/trace.go | 11 +- 8 files changed, 228 insertions(+), 149 deletions(-) create mode 100644 runtime/test/trace_marshal/decyclic.go diff --git a/cmd/xgo/version.go b/cmd/xgo/version.go index 29a880ca..73195047 100644 --- a/cmd/xgo/version.go +++ b/cmd/xgo/version.go @@ -3,8 +3,8 @@ package main import "fmt" const VERSION = "1.0.22" -const REVISION = "f174b18f76bff4bd8acddffd03835b760dad03d8+1" -const NUMBER = 175 +const REVISION = "b41281e9fca79eb3bbde5e4347ab8f37763bc545+1" +const NUMBER = 176 func getRevision() string { revSuffix := "" diff --git a/runtime/core/version.go b/runtime/core/version.go index 1091b43d..aacad95d 100644 --- a/runtime/core/version.go +++ b/runtime/core/version.go @@ -7,8 +7,8 @@ import ( ) const VERSION = "1.0.22" -const REVISION = "f174b18f76bff4bd8acddffd03835b760dad03d8+1" -const NUMBER = 175 +const REVISION = "b41281e9fca79eb3bbde5e4347ab8f37763bc545+1" +const NUMBER = 176 // these fields will be filled by compiler const XGO_VERSION = "" diff --git a/runtime/test/trace_marshal/decyclic.go b/runtime/test/trace_marshal/decyclic.go new file mode 100644 index 00000000..19b37def --- /dev/null +++ b/runtime/test/trace_marshal/decyclic.go @@ -0,0 +1,177 @@ +package trace_marshal + +import ( + "reflect" + "unsafe" +) + +// cyclic are often caused by tree like data structures +// +// NOTE: this file is a temporiray backup +// decyclic seems has signaficant memory consumption +// making it slow to decyclic when encounters large data +// +// Problems with modifying decyclic: +// +// may cause data race because value is being +// used in another goroutine +type decyclicer struct { + seen map[uintptr]struct{} +} + +func Decyclic(v interface{}) interface{} { + if v == nil { + return nil + } + d := &decyclicer{ + seen: map[uintptr]struct{}{}, + } + arr := []interface{}{v} + d.clear(reflect.ValueOf(arr), func(r reflect.Value) { + panic("should not call slice's set") + }) + return arr[0] +} + +// func makeAddrable(v reflect.Value, set func(r reflect.Value)) reflect.Value { +// if v.CanAddr() { +// return v +// } +// if false { +// p := reflect.New(v.Type()) +// p.Elem().Set(v) +// x := p.Elem() +// set(x) +// return x +// } +// panic("not addressable") +// } + +func (c *decyclicer) clear(v reflect.Value, set func(r reflect.Value)) { + // fmt.Printf("clear: %v\n", v.Type()) + switch v.Kind() { + case reflect.Ptr: + if v.IsNil() { + return + } + // if !v.CanAddr() { + // return + // } + + // only pointer can create cyclic + ptr := v.Pointer() + if ptr == 0 { + return + } + if _, ok := c.seen[ptr]; ok { + // fmt.Printf("found : 0x%x -> %v\n", ptr, v.Interface()) + set(reflect.Zero(v.Type())) + return + } + c.seen[ptr] = struct{}{} + defer delete(c.seen, ptr) + + c.clear(v.Elem(), func(r reflect.Value) { + v.Elem().Set(r) + }) + case reflect.Interface: + if v.IsNil() { + return + } + // if !v.CanAddr() { + // return + // } + c.clear(v.Elem(), func(r reflect.Value) { + // NOTE: interface{} is special + // we can directly can call v.Set + // instead of v.Elem().Set() + v.Set(r) + if v.Elem().Kind() == reflect.Ptr && v.Elem().IsNil() { + // fmt.Printf("found isNil\n") + // avoid {nil-value,non-nil type} + set(reflect.Zero(v.Type())) + } + }) + case reflect.Array, reflect.Slice: + switch v.Type().Elem().Kind() { + case reflect.Int64, reflect.Int, reflect.Int32, reflect.Int16, + reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16, + reflect.Float64, reflect.Float32, + reflect.String, + reflect.Bool: + return + case reflect.Int8, reflect.Uint8: + // []byte -> Uint8 + // ignore some: 10K JSON + n := v.Len() + if v.Kind() == reflect.Slice && n > 10*1024 { + // reserve first 16 and last 16 + // S[:16] ... + S[len(S)-16:] + const reserve = 16 + const ellipse = 3 + const totalLen = reserve*2 + ellipse + newSlice := reflect.MakeSlice(v.Type(), totalLen, totalLen) + for i := 0; i < reserve; i++ { + newSlice.Index(i).Set(v.Index(i)) + } + for i := 0; i < ellipse; i++ { + if v.Kind() == reflect.Uint8 { + newSlice.Index(reserve + i).SetUint('.') + } else if v.Kind() == reflect.Int8 { + newSlice.Index(reserve + i).SetInt('.') + } + } + for i := 0; i < reserve; i++ { + newSlice.Index(reserve + ellipse + i).Set(v.Index(n - reserve + i)) + } + set(newSlice) + } + return + } + for i := 0; i < v.Len(); i++ { + e := v.Index(i) + c.clear(e, func(r reflect.Value) { + e.Set(r) + }) + } + case reflect.Map: + if !v.CanAddr() { + return + } + iter := v.MapRange() + // sets := [][2]reflect.Value{} + for iter.Next() { + vi := v.MapIndex(iter.Key()) + c.clear(vi, func(r reflect.Value) { + v.SetMapIndex(iter.Key(), r) + }) + } + case reflect.Struct: + if !v.CanAddr() { + return + } + // fmt.Printf("struct \n") + // make struct addrable + // v = makeAddrable(v, set) + // fmt.Printf("struct can addr: %v\n", v.CanAddr()) + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + // fmt.Printf("field: %v %v\n", field, field.CanAddr()) + if field.CanSet() && false { + c.clear(field, func(r reflect.Value) { + field.Set(r) + }) + } else { + e := reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())) + c.clear(e.Elem(), func(r reflect.Value) { + e.Elem().Set(r) + }) + // panic(fmt.Errorf("cannot set: %v", field)) + } + } + case reflect.Chan, reflect.Func: + // ignore + default: + // int + } +} diff --git a/runtime/test/trace_marshal/decyclic_test.go b/runtime/test/trace_marshal/decyclic_test.go index 439cfce2..b02762b3 100644 --- a/runtime/test/trace_marshal/decyclic_test.go +++ b/runtime/test/trace_marshal/decyclic_test.go @@ -4,8 +4,6 @@ import ( "fmt" "reflect" "testing" - - "github.com/xhd2015/xgo/runtime/trace" ) type _struct struct { @@ -16,7 +14,7 @@ func TestDecyclicUnexported(t *testing.T) { s := &_struct{} s.self = s - vs := trace.Decyclic(s) + vs := Decyclic(s) if vs.(*_struct).self != nil { t.Fatalf("expect vs.self to be nil") } @@ -37,22 +35,22 @@ func TestDecyclicMapValue(t *testing.T) { t.Fatalf("expect v1.m['self'] to be non nil") } - vs := trace.Decyclic(s) + vs := Decyclic(s) if vs.(*_structMap).m["self"] != nil { t.Fatalf("expect vs.m['self'] to be nil") } } -type _interface struct { +type _structWithIntf struct { self interface{} } func TestDecyclicInterface(t *testing.T) { - s := &_interface{} + s := &_structWithIntf{} s.self = s - vs := trace.Decyclic(s) - if vs.(*_interface).self != nil { + vs := Decyclic(s) + if vs.(*_structWithIntf).self != nil { t.Fatalf("expect vs.self to be nil") } } @@ -66,7 +64,7 @@ func TestDecyclicValueStruct(t *testing.T) { A: 123, } - vs := trace.Decyclic(s) + vs := Decyclic(s) vsA := vs.(_valueStruct).A if vsA != 123 { t.Fatalf("vs.A: %v", vsA) @@ -77,9 +75,26 @@ func TestDecyclicMapInterface(t *testing.T) { m := make(map[string]interface{}) m["A"] = 123 - vs := trace.Decyclic(m) + vs := Decyclic(m) vsA := vs.(map[string]interface{})["A"] if fmt.Sprint(vsA) != "123" { t.Fatalf("vs.A: %v", vsA) } } + +type byteSliceHolder struct { + slice []byte +} + +func TestDecyclicLargeByteSlice(t *testing.T) { + s := &byteSliceHolder{ + slice: make([]byte, 10*1024+1), + } + vs := Decyclic(s) + vslice := vs.(*byteSliceHolder).slice + + // 16 + ... + 16 + if len(vslice) != 35 { + t.Logf("expect len to be %d, actual: %v", 35, len(vslice)) + } +} diff --git a/runtime/test/trace_panic_peek/trace_panic_peek_test.go b/runtime/test/trace_panic_peek/trace_panic_peek_test.go index 0655fd0b..92e02c7b 100644 --- a/runtime/test/trace_panic_peek/trace_panic_peek_test.go +++ b/runtime/test/trace_panic_peek/trace_panic_peek_test.go @@ -20,7 +20,7 @@ func TestTracePanicPeek(t *testing.T) { var traceData []byte trace.Options().OnComplete(func(root *trace.Root) { var err error - traceData, err = trace.MarshalAnyJSON(root.Export()) + traceData, err = trace.MarshalAnyJSON(root.Export(nil)) if err != nil { t.Fatal(err) } diff --git a/runtime/trace/marshal.go b/runtime/trace/marshal.go index 1eb8bd1b..f273522e 100644 --- a/runtime/trace/marshal.go +++ b/runtime/trace/marshal.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "reflect" - "unsafe" "github.com/xhd2015/xgo/runtime/core" "github.com/xhd2015/xgo/runtime/functab" @@ -17,32 +16,30 @@ import ( // to JSON, when it encounters unmarshalable values // like func, chan, it will bypass these values. func MarshalAnyJSON(v interface{}) ([]byte, error) { - funcInfo := functab.Info("encoding/json", "newTypeEncoder") - if funcInfo == nil { + newTypeEncoder := functab.Info("encoding/json", "newTypeEncoder") + if newTypeEncoder == nil { fmt.Fprintf(os.Stderr, "WARNING: encoding/json.newTypeEncoder not trapped(requires xgo).\n") return json.Marshal(v) } - // can be done via filter - if false { - v = Decyclic(v) - } - // get the unmarshalable function - unmarshalable := getMarshaler(funcInfo.Func, reflect.TypeOf(unmarshalableFunc)) + unmarshalable := getMarshaler(newTypeEncoder.Func, reflect.TypeOf(unmarshalableFunc)) var data []byte var err error // mock the encoding json - trap.WithFuncOverride(funcInfo, &trap.Interceptor{ + trap.WithFuncOverride(newTypeEncoder, &trap.Interceptor{ + Pre: func(ctx context.Context, f *core.FuncInfo, args, result core.Object) (interface{}, error) { + return nil, nil + }, Post: func(ctx context.Context, f *core.FuncInfo, args, result core.Object, data interface{}) error { - if f != funcInfo { + if f != newTypeEncoder { return nil } resField := result.GetFieldIndex(0) // if unmarshalable, replace with an empty struct if reflect.ValueOf(resField.Value()).Pointer() == reflect.ValueOf(unmarshalable).Pointer() { - resField.Set(getMarshaler(funcInfo.Func, reflect.TypeOf(struct{}{}))) + resField.Set(getMarshaler(newTypeEncoder.Func, reflect.TypeOf(struct{}{}))) } return nil }, @@ -67,124 +64,3 @@ func getMarshaler(newTypeEncoder interface{}, v reflect.Type) interface{} { }) return res } - -type decyclicer struct { - seen map[uintptr]struct{} -} - -func Decyclic(v interface{}) interface{} { - if v == nil { - return nil - } - d := &decyclicer{ - seen: map[uintptr]struct{}{}, - } - d.clear(reflect.ValueOf(v), func(r reflect.Value) { - v = r.Interface() - }) - return v -} - -func makeAddrable(v reflect.Value, set func(r reflect.Value)) reflect.Value { - if v.CanAddr() { - return v - } - p := reflect.New(v.Type()) - p.Elem().Set(v) - x := p.Elem() - set(x) - return x -} - -func (c *decyclicer) clear(v reflect.Value, set func(r reflect.Value)) { - // fmt.Printf("clear: %v\n", v.Type()) - switch v.Kind() { - case reflect.Ptr: - if v.IsNil() { - return - } - - // only pointer can create cyclic - ptr := v.Pointer() - if ptr == 0 { - return - } - if _, ok := c.seen[ptr]; ok { - // fmt.Printf("found : 0x%x -> %v\n", ptr, v.Interface()) - set(reflect.Zero(v.Type())) - return - } - c.seen[ptr] = struct{}{} - defer delete(c.seen, ptr) - - v = makeAddrable(v, set) - c.clear(v.Elem(), func(r reflect.Value) { - v.Elem().Set(r) - }) - case reflect.Interface: - if v.IsNil() { - return - } - v = makeAddrable(v, set) - c.clear(v.Elem(), func(r reflect.Value) { - // NOTE: interface{} is special - // we can directly can call v.Set - // instead of v.Elem().Set() - v.Set(r) - if v.Elem().Kind() == reflect.Ptr && v.Elem().IsNil() { - // fmt.Printf("found isNil\n") - // avoid {nil-value,non-nil type} - set(reflect.Zero(v.Type())) - } - }) - case reflect.Array, reflect.Slice: - switch v.Type().Elem().Kind() { - case reflect.Int64, reflect.Int, reflect.Int32, reflect.Int16, reflect.Int8, - reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uint8, - reflect.Float64, reflect.Float32, - reflect.String, - reflect.Bool: - return - } - v = makeAddrable(v, set) - for i := 0; i < v.Len(); i++ { - e := v.Index(i) - c.clear(e, func(r reflect.Value) { - e.Set(r) - }) - } - case reflect.Map: - v = makeAddrable(v, set) - iter := v.MapRange() - // sets := [][2]reflect.Value{} - for iter.Next() { - vi := v.MapIndex(iter.Key()) - c.clear(vi, func(r reflect.Value) { - v.SetMapIndex(iter.Key(), r) - }) - } - case reflect.Struct: - // fmt.Printf("struct \n") - // make struct addrable - v = makeAddrable(v, set) - - for i := 0; i < v.NumField(); i++ { - field := v.Field(i) - if field.CanSet() { - c.clear(field, func(r reflect.Value) { - field.Set(r) - }) - } else { - e := reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())) - c.clear(e.Elem(), func(r reflect.Value) { - e.Elem().Set(r) - }) - // panic(fmt.Errorf("cannot set: %v", field)) - } - } - case reflect.Chan, reflect.Func: - // ignore - default: - // int - } -} diff --git a/runtime/trace/stack.go b/runtime/trace/stack.go index 7b01825a..2bbd03b3 100644 --- a/runtime/trace/stack.go +++ b/runtime/trace/stack.go @@ -31,6 +31,8 @@ type Stack struct { // for example: google.golang.org/protobuf/internal/order type ExportOptions struct { FilterStack func(stack *StackExport) *StackExport + FilterRoot func(root *RootExport) *RootExport + MarshalJSON func(root *RootExport) ([]byte, error) } func (c *Root) Export(opts *ExportOptions) *RootExport { diff --git a/runtime/trace/trace.go b/runtime/trace/trace.go index e06d9fb3..4f210e88 100644 --- a/runtime/trace/trace.go +++ b/runtime/trace/trace.go @@ -362,7 +362,16 @@ func fmtStack(root *Root, opts *ExportOptions) (data []byte, err error) { if marshalStack != nil { return marshalStack(root) } - return MarshalAnyJSON(root.Export(opts)) + exportRoot := root.Export(opts) + if opts != nil { + if opts.FilterRoot != nil { + exportRoot = opts.FilterRoot(exportRoot) + } + if opts.MarshalJSON != nil { + return opts.MarshalJSON(exportRoot) + } + } + return MarshalAnyJSON(exportRoot) } func emitTraceNoErr(name string, root *Root, opts *ExportOptions) { From 26160b5f6548430ae0bbcf8875ac988e777dc93a Mon Sep 17 00:00:00 2001 From: xhd2015 Date: Sat, 13 Apr 2024 21:35:03 +0800 Subject: [PATCH 3/3] add ErrSilent wrapper when marshalling arguments and results --- cmd/xgo/version.go | 4 +- runtime/core/version.go | 4 +- runtime/test/trace/trace_mock_test.go | 2 +- runtime/test/trace_marshal/marshal_test.go | 16 +-- runtime/trace/marshal.go | 6 +- runtime/trace/stack.go | 140 ++++++++++++++++++++- runtime/trace/trace.go | 4 +- 7 files changed, 155 insertions(+), 21 deletions(-) diff --git a/cmd/xgo/version.go b/cmd/xgo/version.go index 73195047..89dbb952 100644 --- a/cmd/xgo/version.go +++ b/cmd/xgo/version.go @@ -3,8 +3,8 @@ package main import "fmt" const VERSION = "1.0.22" -const REVISION = "b41281e9fca79eb3bbde5e4347ab8f37763bc545+1" -const NUMBER = 176 +const REVISION = "b95076921b299cab29e79a1642abf005462fb732+1" +const NUMBER = 177 func getRevision() string { revSuffix := "" diff --git a/runtime/core/version.go b/runtime/core/version.go index aacad95d..c51ef083 100644 --- a/runtime/core/version.go +++ b/runtime/core/version.go @@ -7,8 +7,8 @@ import ( ) const VERSION = "1.0.22" -const REVISION = "b41281e9fca79eb3bbde5e4347ab8f37763bc545+1" -const NUMBER = 176 +const REVISION = "b95076921b299cab29e79a1642abf005462fb732+1" +const NUMBER = 177 // these fields will be filled by compiler const XGO_VERSION = "" diff --git a/runtime/test/trace/trace_mock_test.go b/runtime/test/trace/trace_mock_test.go index 7c03029f..449cf881 100644 --- a/runtime/test/trace/trace_mock_test.go +++ b/runtime/test/trace/trace_mock_test.go @@ -18,7 +18,7 @@ func TestMockedFuncShouldShowInTrace(t *testing.T) { }) h() }) - data, err := trace.MarshalAnyJSON(root.Export()) + data, err := trace.MarshalAnyJSON(root.Export(nil)) if err != nil { t.Fatal(err) } diff --git a/runtime/test/trace_marshal/marshal_test.go b/runtime/test/trace_marshal/marshal_test.go index 8bbb1bf4..3832c599 100644 --- a/runtime/test/trace_marshal/marshal_test.go +++ b/runtime/test/trace_marshal/marshal_test.go @@ -72,20 +72,20 @@ type cyclic struct { Name string } -func TestMarshalCyclicJSON(t *testing.T) { +func TestMarshalCyclicJSONShouldError(t *testing.T) { c := &cyclic{ Name: "cyclic", } c.Self = c - res, err := trace.MarshalAnyJSON(c) - if err != nil { - t.Fatal(err) + // + _, err := trace.MarshalAnyJSON(c) + if err == nil { + t.Fatalf("expect marshal err") } - resStr := string(res) - expect := `{"Self":null,"Name":"cyclic"}` - if resStr != expect { - t.Fatalf("expect res to be %q, actual: %q", expect, resStr) + expectErrMsg := "encountered a cycle via *trace_marshal.cyclic" + if !strings.Contains(err.Error(), expectErrMsg) { + t.Fatalf("expect err: %q, actual: %q", expectErrMsg, err.Error()) } } diff --git a/runtime/trace/marshal.go b/runtime/trace/marshal.go index f273522e..04572e19 100644 --- a/runtime/trace/marshal.go +++ b/runtime/trace/marshal.go @@ -28,9 +28,9 @@ func MarshalAnyJSON(v interface{}) ([]byte, error) { var err error // mock the encoding json trap.WithFuncOverride(newTypeEncoder, &trap.Interceptor{ - Pre: func(ctx context.Context, f *core.FuncInfo, args, result core.Object) (interface{}, error) { - return nil, nil - }, + // Pre: func(ctx context.Context, f *core.FuncInfo, args, result core.Object) (interface{}, error) { + // return nil, nil + // }, Post: func(ctx context.Context, f *core.FuncInfo, args, result core.Object, data interface{}) error { if f != newTypeEncoder { return nil diff --git a/runtime/trace/stack.go b/runtime/trace/stack.go index 2bbd03b3..6cad54aa 100644 --- a/runtime/trace/stack.go +++ b/runtime/trace/stack.go @@ -1,6 +1,8 @@ package trace import ( + "encoding/json" + "fmt" "time" "github.com/xhd2015/xgo/runtime/core" @@ -30,21 +32,87 @@ type Stack struct { // // for example: google.golang.org/protobuf/internal/order type ExportOptions struct { + // suppress error when marshalling + // arguments and results + DisableErrSilent bool + SizeLimit int // 0: default limit 4K + AppearanceLimit int // 0: default limit 100 + FilterStack func(stack *StackExport) *StackExport + FilterRoot func(root *RootExport) *RootExport - MarshalJSON func(root *RootExport) ([]byte, error) + MarshalRoot func(root *RootExport) ([]byte, error) + + stats map[string]map[string]*stat +} +type stat struct { + total int + current int +} + +func (c *ExportOptions) getSizeLimit() int { + if c == nil || c.SizeLimit == 0 { + return 4 * 1024 + } + return c.SizeLimit +} +func (c *ExportOptions) getAppearanceLimit() int { + if c == nil || c.AppearanceLimit == 0 { + return 100 + } + return c.AppearanceLimit } func (c *Root) Export(opts *ExportOptions) *RootExport { if c == nil { return nil } + if opts == nil { + opts = &ExportOptions{} + } + if opts.getAppearanceLimit() > 0 { + opts.stats = getStats(c) + } + return &RootExport{ Begin: c.Begin, Children: (stacks)(c.Children).Export(opts), } } +func getStats(root *Root) map[string]map[string]*stat { + mapping := make(map[string]map[string]*stat) + var traverse func(stack *Stack) + traverse = func(st *Stack) { + if st == nil { + return + } + if st.FuncInfo != nil { + pkg := st.FuncInfo.Pkg + fn := st.FuncInfo.IdentityName + fnMapping := mapping[pkg] + if fnMapping == nil { + fnMapping = make(map[string]*stat) + mapping[pkg] = fnMapping + } + st := fnMapping[fn] + if st == nil { + st = &stat{} + fnMapping[fn] = st + } + st.total++ + } + + for _, st := range st.Children { + traverse(st) + } + } + for _, st := range root.Children { + traverse(st) + } + return mapping +} + type stacks []*Stack func (c stacks) Export(opts *ExportOptions) []*StackExport { @@ -58,6 +126,18 @@ func (c stacks) Export(opts *ExportOptions) []*StackExport { if exportStack == nil { continue } + if exportStack.FuncInfo != nil && opts != nil && opts.stats != nil { + apprLimit := opts.getAppearanceLimit() + if apprLimit > 0 { + fnStat := opts.stats[exportStack.FuncInfo.Pkg][exportStack.FuncInfo.IdentityName] + if fnStat != nil && fnStat.total > apprLimit { + if fnStat.current >= apprLimit { + continue + } + fnStat.current++ + } + } + } list = append(list, exportStack) } return list @@ -67,16 +147,29 @@ func (c *Stack) Export(opts *ExportOptions) *StackExport { if c == nil { return nil } + var errMsg string if c.Error != nil { errMsg = c.Error.Error() } + var args interface{} = c.Args + var results interface{} = c.Results + + sizeLimit := opts.getSizeLimit() + if sizeLimit > 0 { + args = &LimitSize{args, sizeLimit} + results = &LimitSize{results, sizeLimit} + } + if opts == nil || !opts.DisableErrSilent { + args = &ErrSilent{args} + results = &ErrSilent{results} + } stack := &StackExport{ FuncInfo: ExportFuncInfo(c.FuncInfo, opts), Begin: c.Begin, End: c.End, - Args: c.Args, - Results: c.Results, + Args: args, + Results: results, Panic: c.Panic, Error: errMsg, Children: ((stacks)(c.Children)).Export(opts), @@ -111,3 +204,44 @@ func ExportFuncInfo(c *core.FuncInfo, opts *ExportOptions) *FuncInfoExport { Line: c.Line, } } + +// make json err silent +type ErrSilent struct { + Data interface{} +} + +func (c *ErrSilent) MarshalJSON() (data []byte, err error) { + defer func() { + if e := recover(); e != nil { + if pe, ok := e.(error); ok { + err = pe + } else { + err = fmt.Errorf("panic: %v", e) + } + } + if err != nil { + data = []byte(fmt.Sprintf(`{"error":%q}`, err.Error())) + err = nil + } + }() + data, err = json.Marshal(c.Data) + return +} + +// make json err silent +type LimitSize struct { + Data interface{} + Limit int +} + +func (c *LimitSize) MarshalJSON() ([]byte, error) { + data, err := json.Marshal(c.Data) + if err != nil { + return nil, err + } + if c.Limit <= 0 || c.Limit >= len(data) { + return data, nil + } + // shorten + return []byte(fmt.Sprintf(`{"size":%d, "sizeBeforeShrink":%d,"partialData":%q}`, c.Limit, len(data), string(data[:c.Limit]))), nil +} diff --git a/runtime/trace/trace.go b/runtime/trace/trace.go index 4f210e88..92c80df4 100644 --- a/runtime/trace/trace.go +++ b/runtime/trace/trace.go @@ -367,8 +367,8 @@ func fmtStack(root *Root, opts *ExportOptions) (data []byte, err error) { if opts.FilterRoot != nil { exportRoot = opts.FilterRoot(exportRoot) } - if opts.MarshalJSON != nil { - return opts.MarshalJSON(exportRoot) + if opts.MarshalRoot != nil { + return opts.MarshalRoot(exportRoot) } } return MarshalAnyJSON(exportRoot)