Skip to content

Commit

Permalink
make build cache aware of trace flags
Browse files Browse the repository at this point in the history
  • Loading branch information
xhd2015 committed Jul 27, 2024
1 parent 6bd25c2 commit 3281bbd
Show file tree
Hide file tree
Showing 44 changed files with 805 additions and 113 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/go-compatible.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ jobs:
go-version: '${{ matrix.go }}'

- name: Test
run: go run ./script/run-test --reset-instrument --debug -v
run: go run ./script/run-test --install-xgo --reset-instrument --debug -v
3 changes: 2 additions & 1 deletion .github/workflows/go-next.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ jobs:
run: |
curl -fsSL -o go.tar.gz "https://go.dev/dl/go${{matrix.go}}.linux-amd64.tar.gz"
mkdir setup
mkdir setup/gopath
tar -C setup -xzf go.tar.gz
ls setup
GOROOT=$PWD/setup/go PATH=$PWD/setup/go/bin:$PATH go version
- name: Test
run: GOROOT=$PWD/setup/go PATH=$PWD/setup/go/bin:$PATH go run ./script/run-test --reset-instrument --debug -v
run: GOROOT=$PWD/setup/go GOPATH=$PWD/setup/gopath PATH=$PWD/setup/go/bin:$PWD/setup/gopath/bin:$PATH go run ./script/run-test --install-xgo --reset-instrument --debug -v
2 changes: 1 addition & 1 deletion .github/workflows/go-windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ jobs:
go-version: '${{ matrix.go }}'

- name: Test
run: go run ./script/run-test --reset-instrument --debug -v
run: go run ./script/run-test --install-xgo --reset-instrument --debug -v
2 changes: 1 addition & 1 deletion .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
go-version: '${{ matrix.go }}'

- name: Test
run: go run ./script/run-test --reset-instrument --debug -v -cover -coverpkg github.com/xhd2015/xgo/runtime/... -coverprofile cover.out
run: go run ./script/run-test --install-xgo --reset-instrument --debug -v -cover -coverpkg github.com/xhd2015/xgo/runtime/... -coverprofile cover.out

- name: Merge Coverages
run: go run ./cmd/go-tool-coverage merge ./cover-runtime.out ./cover-runtime-sub.out -o cover-runtime-merged.out --exclude-prefix github.com/xhd2015/xgo/runtime/test
Expand Down
5 changes: 5 additions & 0 deletions cmd/xgo/exec_tool/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ func getDebugEnvList(xgoCompilerEnableEnv string) [][2]string {
{XGO_DEBUG_DUMP_AST_FILE, os.Getenv(XGO_DEBUG_DUMP_AST_FILE)},
{XGO_MAIN_MODULE, os.Getenv(XGO_MAIN_MODULE)},
{XGO_COMPILE_PKG_DATA_DIR, os.Getenv(XGO_COMPILE_PKG_DATA_DIR)},

// strace
{XGO_STACK_TRACE, os.Getenv(XGO_STACK_TRACE)},
{XGO_STACK_TRACE_DIR, os.Getenv(XGO_STACK_TRACE_DIR)},

{XGO_STD_LIB_TRAP_DEFAULT_ALLOW, os.Getenv(XGO_STD_LIB_TRAP_DEFAULT_ALLOW)},
{XGO_DEBUG_COMPILE_PKG, os.Getenv(XGO_DEBUG_COMPILE_PKG)},
{XGO_DEBUG_COMPILE_LOG_FILE, os.Getenv(XGO_DEBUG_COMPILE_LOG_FILE)},
Expand Down
6 changes: 6 additions & 0 deletions cmd/xgo/exec_tool/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ const XGO_TOOLCHAIN_VERSION_NUMBER = "XGO_TOOLCHAIN_VERSION_NUMBER"

const XGO_MAIN_MODULE = "XGO_MAIN_MODULE"

// --strace
const XGO_STACK_TRACE = "XGO_STACK_TRACE"

// --strace-dir
const XGO_STACK_TRACE_DIR = "XGO_STACK_TRACE_DIR"

const XGO_COMPILE_PKG_DATA_DIR = "XGO_COMPILE_PKG_DATA_DIR"

const XGO_STD_LIB_TRAP_DEFAULT_ALLOW = "XGO_STD_LIB_TRAP_DEFAULT_ALLOW"
Expand Down
26 changes: 23 additions & 3 deletions cmd/xgo/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ func handleBuild(cmd string, args []string) error {
dumpIR := opts.dumpIR
dumpAST := opts.dumpAST
stackTrace := opts.stackTrace
stackTraceDir := opts.stackTraceDir
trapStdlib := opts.trapStdlib

if cmdExec && len(remainArgs) == 0 {
Expand Down Expand Up @@ -272,6 +273,22 @@ func handleBuild(cmd string, args []string) error {
h.Write(optionsFromFileContent)
buildCacheSuffix += "-" + hex.EncodeToString(h.Sum(nil))
}
if stackTrace == "on" || stackTrace == "true" {
v := stackTrace
if v == "true" {
v = "on"
} else if v == "false" {
v = "off"
}
buildCacheSuffix += "-strace_" + v
}
if stackTraceDir != "" && stackTraceDir != "." && stackTraceDir != "./" {
// this affects the flags package
h := md5.New()
h.Write([]byte(stackTraceDir))
buildCacheSuffix += "-" + hex.EncodeToString(h.Sum(nil))
}

buildCacheDir := filepath.Join(instrumentDir, "build-cache"+buildCacheSuffix)
revisionFile := filepath.Join(instrumentDir, "xgo-revision.txt")
fullSyncRecord := filepath.Join(instrumentDir, "full-sync-record.txt")
Expand Down Expand Up @@ -601,23 +618,26 @@ func handleBuild(cmd string, args []string) error {
if vscodeDebugFile != "" {
xgoDebugVscode = vscodeDebugFile + vscodeDebugFileSuffix
}
execCmd.Env = append(execCmd.Env, "XGO_DEBUG_VSCODE="+xgoDebugVscode)
execCmd.Env = append(execCmd.Env, exec_tool.XGO_DEBUG_VSCODE+"="+xgoDebugVscode)

// debug compile package
execCmd.Env = append(execCmd.Env, exec_tool.XGO_DEBUG_COMPILE_PKG+"="+debugCompilePkg)
execCmd.Env = append(execCmd.Env, exec_tool.XGO_DEBUG_COMPILE_LOG_FILE+"="+debugCompileLogFile)

// stack trace
if stackTrace != "" {
execCmd.Env = append(execCmd.Env, "XGO_STACK_TRACE="+stackTrace)
execCmd.Env = append(execCmd.Env, exec_tool.XGO_STACK_TRACE+"="+stackTrace)
}
if stackTraceDir != "" {
execCmd.Env = append(execCmd.Env, exec_tool.XGO_STACK_TRACE_DIR+"="+stackTraceDir)
}

// trap stdlib
var trapStdlibEnv string
if trapStdlib {
trapStdlibEnv = "true"
}
execCmd.Env = append(execCmd.Env, "XGO_STD_LIB_TRAP_DEFAULT_ALLOW="+trapStdlibEnv)
execCmd.Env = append(execCmd.Env, exec_tool.XGO_STD_LIB_TRAP_DEFAULT_ALLOW+"="+trapStdlibEnv)

// compiler options (make abs)
var absOptionsFromFile string
Expand Down
83 changes: 67 additions & 16 deletions cmd/xgo/option.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,13 @@ type options struct {
// --strace, --strace=on, --strace=off
// --stack-stackTrace, --stack-stackTrace=off, --stack-stackTrace=on
// to be used in test mode
// the parsed value is either on or off, mapping:
// "",true, on => on
// false, off => off
// other => error
stackTrace string
// --strace-dir
stackTraceDir string

remainArgs []string

Expand Down Expand Up @@ -120,6 +126,7 @@ func parseOptions(cmd string, args []string) (*options, error) {
var overlay string
var modfile string
var stackTrace string
var stackTraceDir string
var trapStdlib bool

var remainArgs []string
Expand Down Expand Up @@ -314,6 +321,22 @@ func parseOptions(cmd string, args []string) (*options, error) {
continue
}

// strace dir
stackTraceDirVal, ok, err := tryParseRequiredValue("--strace-dir", args, &i)
if err != nil {
return nil, err
}
if !ok {
stackTraceDirVal, ok, err = tryParseRequiredValue("--stack-trace-dir", args, &i)
if err != nil {
return nil, err
}
}
if ok {
stackTraceDir = stackTraceDirVal
continue
}

argVal, ok := parseStackTraceFlag(arg)
if ok {
stackTrace = argVal
Expand Down Expand Up @@ -419,18 +442,20 @@ func parseOptions(cmd string, args []string) (*options, error) {
// default true
syncWithLink: syncWithLink == nil || *syncWithLink,

mod: mod,
gcflags: gcflags,
overlay: overlay,
modfile: modfile,
stackTrace: stackTrace,
trapStdlib: trapStdlib,
mod: mod,
gcflags: gcflags,
overlay: overlay,
modfile: modfile,
stackTrace: stackTrace,
stackTraceDir: stackTraceDir,
trapStdlib: trapStdlib,

remainArgs: remainArgs,
testArgs: testArgs,
}, nil
}

// parse: --opt=x, --opt x, --opt, but not --opt -x
func tryParseOption(flag string, args []string, i *int) (string, bool) {
v, j, ok := tryParseOptionalValue(flag, args, *i)
if !ok {
Expand All @@ -440,23 +465,49 @@ func tryParseOption(flag string, args []string, i *int) (string, bool) {
return v, true
}

// parse: --opt=x, --opt x, but not --opt -x
// parse: --opt=x, --opt x, even --opt -x
func tryParseRequiredValue(flag string, args []string, i *int) (string, bool, error) {
v, j, ok, err := tryParseValue(flag, args, *i, false)
if err != nil {
return "", false, err
}
if !ok {
return "", false, nil
}
*i = j
return v, true, nil
}

func tryParseOptionalValue(flag string, args []string, i int) (string, int, bool) {
val, i, ok, err := tryParseValue(flag, args, i, true)
if err != nil {
panic(err)
}
return val, i, ok
}

func tryParseValue(flag string, args []string, i int, optional bool) (string, int, bool, error) {
arg := args[i]
if !strings.HasPrefix(arg, flag) {
return "", i, false
return "", i, false, nil
}
suffix := strings.TrimPrefix(arg, flag)
suffix := arg[len(flag):]
if suffix == "" {
if i >= len(args) || strings.HasPrefix(args[i], "-") {
return "", i, true
if optional {
if i >= len(args) || strings.HasPrefix(args[i], "-") {
return "", i, true, nil
}
} else {
if i >= len(args) {
return "", i, false, fmt.Errorf("%s: requires value", flag)
}
}
return args[i+1], i + 1, true
return args[i+1], i + 1, true, nil
}
if !strings.HasPrefix(suffix, "=") {
return "", i, false
return "", i, false, nil
}
return suffix[1:], i, true
return suffix[1:], i, true, nil
}

func parseStackTraceFlag(arg string) (string, bool) {
Expand All @@ -476,10 +527,10 @@ func parseStackTraceFlag(arg string) (string, bool) {
return "", false
}
val := arg[len(stackTracePrefix)+1:]
if val == "" || val == "on" {
if val == "" || val == "on" || val == "true" {
return "on", true
}
if val == "off" {
if val == "off" || val == "false" {
return "off", true
}
panic(fmt.Errorf("unrecognized value %s: %s, expects on|off", arg, val))
Expand Down
50 changes: 30 additions & 20 deletions cmd/xgo/runtime_gen/trace/trace.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,28 +13,17 @@ import (

"github.com/xhd2015/xgo/runtime/core"
"github.com/xhd2015/xgo/runtime/trap"
"github.com/xhd2015/xgo/runtime/trap/flags"
)

// hold goroutine stacks, keyed by goroutine ptr
var stackMap sync.Map // uintptr(goroutine) -> *Root
var testInfoMapping sync.Map // uintptr(goroutine) -> *testInfo

// persist the --strace flag when invoking xgo test
// stack trace options:
//
// on: automatically collect when test starts and ends
const __xgo_injected_StraceFlag = ""

// options:
//
// true: stdlib is by default allowed
const __xgo_injected_StdlibTrapDefaultAllow = ""

var skipStdlibTraceByDefault = __xgo_injected_StdlibTrapDefaultAllow == "true"
var skipStdlibTraceByDefault = flags.TRAP_STDLIB == "true"

type testInfo struct {
name string

name string
onFinish func()
}

Expand All @@ -49,7 +38,7 @@ func init() {
name: name,
}
testInfoMapping.LoadOrStore(key, tInfo)
if __xgo_injected_StraceFlag == "on" {
if flags.STRACE == "on" || flags.STRACE == "true" {
tInfo.onFinish = Begin()
}
})
Expand Down Expand Up @@ -512,7 +501,20 @@ func fmtStack(root *Root, opts *ExportOptions) (data []byte, err error) {
}

func emitTraceNoErr(name string, root *Root, opts *ExportOptions) {
emitTrace(name, root, opts)
var err error
defer func() {
if e := recover(); e != nil {
if pe, ok := e.(error); ok {
err = pe
} else {
err = fmt.Errorf("panic: %v", e)
}
}
if err != nil {
fmt.Fprintf(os.Stderr, "emit trace: name=%s %v", name, err)
}
}()
err = emitTrace(name, root, opts)
}

func formatTime(t time.Time, layout string) (output string) {
Expand All @@ -532,6 +534,7 @@ func emitTrace(name string, root *Root, opts *ExportOptions) error {
}
useStdout := xgoTraceOutput == "stdout"
subName := name
canUseFlagDir := true
if name == "" {
traceIDNum := int64(1)
ghex := fmt.Sprintf("g_%x", __xgo_link_getcurg())
Expand All @@ -542,6 +545,7 @@ func emitTrace(name string, root *Root, opts *ExportOptions) error {
} else if useStdout {
subName = fmt.Sprintf("%s/%s", ghex, traceID)
} else {
canUseFlagDir = false
subName = filepath.Join(xgoTraceOutput, ghex, traceID)
}
}
Expand All @@ -563,11 +567,17 @@ func emitTrace(name string, root *Root, opts *ExportOptions) error {
}

subFile := subName + ".json"
subDir := filepath.Dir(subFile)
err := os.MkdirAll(subDir, 0755)
if err != nil {
return err
if canUseFlagDir && flags.STRACE_DIR != "" {
subFile = filepath.Join(flags.STRACE_DIR, subFile)
} else {
subDir := filepath.Dir(subFile)
err := os.MkdirAll(subDir, 0755)
if err != nil {
return err
}
}

var err error
trap.Direct(func() {
err = WriteFile(subFile, traceOut, 0755)
})
Expand Down
5 changes: 5 additions & 0 deletions cmd/xgo/runtime_gen/trap/flags/build_cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package flags

// if any flag changed, this file should be replaced to
// cause rebuild
const _ = "__BUILD_CACHE__"
Loading

0 comments on commit 3281bbd

Please sign in to comment.