From 0a5cae8da71438ccff74156469b063a132cbe5c1 Mon Sep 17 00:00:00 2001 From: xhd2015 Date: Sat, 6 Apr 2024 23:39:09 +0800 Subject: [PATCH] add pkgdata recording --- cmd/xgo/exec_tool/debug.go | 1 + cmd/xgo/exec_tool/env.go | 2 + cmd/xgo/main.go | 14 ++- cmd/xgo/version.go | 4 +- patch/ctxt/ctx.go | 1 + patch/pkgdata/pkgdata.go | 172 +++++++++++++++++++++++++++ patch/syntax/syntax.go | 2 +- patch/syntax/vars.go | 102 ++++++++++++---- runtime/core/version.go | 4 +- runtime/test/patch/patch_var_test.go | 43 ++++++- runtime/test/patch/sub/sub.go | 3 + 11 files changed, 308 insertions(+), 40 deletions(-) create mode 100644 patch/pkgdata/pkgdata.go create mode 100644 runtime/test/patch/sub/sub.go diff --git a/cmd/xgo/exec_tool/debug.go b/cmd/xgo/exec_tool/debug.go index 6e0cd9bf..4799cbc8 100644 --- a/cmd/xgo/exec_tool/debug.go +++ b/cmd/xgo/exec_tool/debug.go @@ -19,6 +19,7 @@ func getDebugEnv(xgoCompilerEnableEnv string) map[string]string { XGO_DEBUG_DUMP_AST_FILE: os.Getenv(XGO_DEBUG_DUMP_AST_FILE), "GOCACHE": os.Getenv("GOCACHE"), XGO_MAIN_MODULE: os.Getenv(XGO_MAIN_MODULE), + XGO_COMPILE_PKG_DATA_DIR: os.Getenv(XGO_COMPILE_PKG_DATA_DIR), "GOROOT": "../..", "PATH": "../../bin:${env:PATH}", "XGO_COMPILER_ENABLE": xgoCompilerEnableEnv, diff --git a/cmd/xgo/exec_tool/env.go b/cmd/xgo/exec_tool/env.go index aac3a7f9..374875d6 100644 --- a/cmd/xgo/exec_tool/env.go +++ b/cmd/xgo/exec_tool/env.go @@ -13,3 +13,5 @@ const XGO_TOOLCHAIN_REVISION = "XGO_TOOLCHAIN_REVISION" const XGO_TOOLCHAIN_VERSION_NUMBER = "XGO_TOOLCHAIN_VERSION_NUMBER" const XGO_MAIN_MODULE = "XGO_MAIN_MODULE" + +const XGO_COMPILE_PKG_DATA_DIR = "XGO_COMPILE_PKG_DATA_DIR" diff --git a/cmd/xgo/main.go b/cmd/xgo/main.go index 641f5a50..e795a073 100644 --- a/cmd/xgo/main.go +++ b/cmd/xgo/main.go @@ -232,6 +232,7 @@ func handleBuild(cmd string, args []string) error { compilerBin := filepath.Join(instrumentDir, "compile"+exeSuffix) compilerBuildID := filepath.Join(instrumentDir, "compile.buildid.txt") instrumentGoroot := filepath.Join(instrumentDir, goVersionName) + packageDataDir := filepath.Join(instrumentDir, "pkgdata") // gcflags can cause the build cache to invalidate // so separate them with normal one @@ -272,7 +273,7 @@ func handleBuild(cmd string, args []string) error { var revisionChanged bool if !noInstrument && !noSetup { - err = ensureDirs(binDir, logDir, instrumentDir) + err = ensureDirs(binDir, logDir, instrumentDir, packageDataDir) if err != nil { return err } @@ -363,7 +364,7 @@ func handleBuild(cmd string, args []string) error { } } logDebug("resolved main module: %s", mainModule) - execCmdEnv = append(execCmdEnv, "XGO_MAIN_MODULE="+mainModule) + execCmdEnv = append(execCmdEnv, exec_tool.XGO_MAIN_MODULE+"="+mainModule) // GOCACHE="$shdir/build-cache" PATH=$goroot/bin:$PATH GOROOT=$goroot DEBUG_PKG=$debug go build -toolexec="$shdir/exce_tool $cmd" "${build_flags[@]}" "$@" buildCmdArgs := []string{cmd} if toolExecFlag != "" { @@ -418,6 +419,7 @@ func handleBuild(cmd string, args []string) error { if !noInstrument { execCmd.Env = append(execCmd.Env, "GOCACHE="+buildCacheDir) execCmd.Env = append(execCmd.Env, "XGO_COMPILER_BIN="+compilerBin) + execCmd.Env = append(execCmd.Env, exec_tool.XGO_COMPILE_PKG_DATA_DIR+"="+packageDataDir) // xgo versions execCmd.Env = append(execCmd.Env, "XGO_TOOLCHAIN_VERSION="+VERSION) execCmd.Env = append(execCmd.Env, "XGO_TOOLCHAIN_REVISION="+REVISION) @@ -570,7 +572,7 @@ func checkGoroot(goroot string) (string, error) { return "", err } -func ensureDirs(binDir string, logDir string, instrumentDir string) error { +func ensureDirs(binDir string, logDir string, instrumentDir string, packageDataDir string) error { err := os.MkdirAll(binDir, 0755) if err != nil { return fmt.Errorf("create ~/.xgo/bin: %w", err) @@ -581,7 +583,11 @@ func ensureDirs(binDir string, logDir string, instrumentDir string) error { } err = os.MkdirAll(instrumentDir, 0755) if err != nil { - return fmt.Errorf("create ~/.xgo/%s: %w", filepath.Base(instrumentDir), err) + return fmt.Errorf("create %s: %w", filepath.Base(instrumentDir), err) + } + err = os.MkdirAll(packageDataDir, 0755) + if err != nil { + return fmt.Errorf("create %s: %w", filepath.Base(packageDataDir), err) } return nil } diff --git a/cmd/xgo/version.go b/cmd/xgo/version.go index 5465bddf..59dae190 100644 --- a/cmd/xgo/version.go +++ b/cmd/xgo/version.go @@ -3,8 +3,8 @@ package main import "fmt" const VERSION = "1.0.18" -const REVISION = "2c003e4f895ef51b9c22e7c5bb5018dd6007ff96+1" -const NUMBER = 164 +const REVISION = "86b60236af7fe7147b90073421f57187fbf6990a+1" +const NUMBER = 165 func getRevision() string { revSuffix := "" diff --git a/patch/ctxt/ctx.go b/patch/ctxt/ctx.go index 8532b89e..ee1c744e 100644 --- a/patch/ctxt/ctx.go +++ b/patch/ctxt/ctx.go @@ -11,6 +11,7 @@ const XgoRuntimePkg = XgoModule + "/runtime" const XgoRuntimeCorePkg = XgoModule + "/runtime/core" var XgoMainModule = os.Getenv("XGO_MAIN_MODULE") +var XgoCompilePkgDataDir = os.Getenv("XGO_COMPILE_PKG_DATA_DIR") func SkipPackageTrap() bool { pkgPath := GetPkgPath() diff --git a/patch/pkgdata/pkgdata.go b/patch/pkgdata/pkgdata.go new file mode 100644 index 00000000..ad7d2252 --- /dev/null +++ b/patch/pkgdata/pkgdata.go @@ -0,0 +1,172 @@ +package pkgdata + +import ( + "cmd/compile/internal/base" + xgo_ctxt "cmd/compile/internal/xgo_rewrite_internal/patch/ctxt" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +type PackageData struct { + Vars map[string]bool + Consts map[string]bool + Funcs map[string]bool +} + +var pkgDataMapping map[string]*PackageData + +func GetPkgData(pkgPath string) *PackageData { + data, ok := pkgDataMapping[pkgPath] + if ok { + return data + } + data, err := load(pkgPath) + if err != nil { + base.Errorf("load package data: %s %v", pkgPath, err) + return nil + } + if pkgDataMapping == nil { + pkgDataMapping = make(map[string]*PackageData, 1) + } + pkgDataMapping[pkgPath] = data + return data +} +func WritePkgData(pkgPath string, pkgData *PackageData) error { + file := getPkgDataFile(pkgPath) + + err := os.MkdirAll(filepath.Dir(file), 0755) + if err != nil { + return err + } + w, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return err + } + defer w.Close() + + writeSection := func(section string, m map[string]bool) error { + if len(m) == 0 { + return nil + } + _, err := io.WriteString(w, section) + if err != nil { + return err + } + _, err = io.WriteString(w, "\n") + if err != nil { + return err + } + for k := range m { + _, err := io.WriteString(w, k) + if err != nil { + return err + } + _, err = io.WriteString(w, "\n") + if err != nil { + return err + } + } + return nil + } + err = writeSection("[const]", pkgData.Consts) + if err != nil { + return err + } + writeSection("[var]", pkgData.Vars) + if err != nil { + return err + } + writeSection("[func]", pkgData.Funcs) + if err != nil { + return err + } + + return nil +} + +func load(pkgPath string) (*PackageData, error) { + if xgo_ctxt.XgoCompilePkgDataDir == "" { + return nil, fmt.Errorf("XGO_COMPILE_PKG_DATA_DIR not set") + } + file := getPkgDataFile(pkgPath) + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + return parsePkgData(string(data)) +} + +type Section int + +const ( + Section_Func Section = 1 + Section_Var Section = 2 + Section_Const Section = 3 +) + +func getPkgDataFile(pkgPath string) string { + fsPath := pkgPath + if filepath.Separator != '/' { + split := strings.Split(pkgPath, "/") + fsPath = filepath.Join(split...) + } + return filepath.Join(xgo_ctxt.XgoCompilePkgDataDir, fsPath, "__xgo_pkgdata__.txt") +} + +// [func] +func parsePkgData(content string) (*PackageData, error) { + lines := strings.Split(content, "\n") + n := len(lines) + + p := &PackageData{} + var section Section + for i := 0; i < n; i++ { + line := strings.TrimSpace(lines[i]) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + switch line { + case "[func]": + section = Section_Func + case "[var]": + section = Section_Var + case "[const]": + section = Section_Const + default: + if section == 0 { + break + } + name := line + idx := strings.Index(line, " ") + if idx >= 0 { + name = line[:idx] + } + if name == "" { + break + } + switch section { + case Section_Func: + if p.Funcs == nil { + p.Funcs = make(map[string]bool, 1) + } + p.Funcs[name] = true + case Section_Var: + if p.Vars == nil { + p.Vars = make(map[string]bool, 1) + } + p.Vars[name] = true + case Section_Const: + if p.Consts == nil { + p.Consts = make(map[string]bool, 1) + } + p.Consts[name] = true + default: + // ignore others + } + } + } + return p, nil +} diff --git a/patch/syntax/syntax.go b/patch/syntax/syntax.go index 35815fb0..b4d3314e 100644 --- a/patch/syntax/syntax.go +++ b/patch/syntax/syntax.go @@ -170,7 +170,7 @@ func registerFuncs(fileList []*syntax.File, addFile func(name string, r io.Reade rewriteStdAndGenericFuncs(funcDelcs, pkgPath) if varTrap { - trapVariables(fileList, funcDelcs) + trapVariables(pkgPath, fileList, funcDelcs) // debug // fmt.Fprintf(os.Stderr, "ast:") // syntax.Fdump(os.Stderr, fileList[0]) diff --git a/patch/syntax/vars.go b/patch/syntax/vars.go index 9d60ccbc..af2cd7de 100644 --- a/patch/syntax/vars.go +++ b/patch/syntax/vars.go @@ -4,6 +4,7 @@ import ( "cmd/compile/internal/base" "cmd/compile/internal/syntax" xgo_ctxt "cmd/compile/internal/xgo_rewrite_internal/patch/ctxt" + "cmd/compile/internal/xgo_rewrite_internal/patch/pkgdata" "fmt" "os" "strconv" @@ -55,10 +56,25 @@ func (c *vis) Visit(node syntax.Node) (w syntax.Visitor) { return nil } -func trapVariables(fileList []*syntax.File, funcDelcs []*DeclInfo) { +func trapVariables(pkgPath string, fileList []*syntax.File, funcDelcs []*DeclInfo) { names := make(map[string]*DeclInfo, len(funcDelcs)) + varNames := make(map[string]bool) + constNames := make(map[string]bool) for _, funcDecl := range funcDelcs { - names[funcDecl.IdentityName()] = funcDecl + identityName := funcDecl.IdentityName() + names[identityName] = funcDecl + if funcDecl.Kind == Kind_Var || funcDecl.Kind == Kind_VarPtr { + varNames[identityName] = true + } else if funcDecl.Kind == Kind_Const { + constNames[identityName] = true + } + } + err := pkgdata.WritePkgData(pkgPath, &pkgdata.PackageData{ + Consts: constNames, + Vars: varNames, + }) + if err != nil { + base.Fatalf("write pkg data: %v", err) } // iterate each file, find variable reference, for _, file := range fileList { @@ -72,7 +88,7 @@ func trapVariables(fileList []*syntax.File, funcDelcs []*DeclInfo) { continue } ctx := &BlockContext{} - ctx.traverseNode(fnDecl.Body, names, imports) + fnDecl.Body = ctx.traverseBlockStmt(fnDecl.Body, names, imports) } } } @@ -115,6 +131,8 @@ type BlockContext struct { Names map[string]bool + OperationParent map[syntax.Node]*syntax.Operation + // to be inserted InsertList []syntax.Stmt @@ -338,7 +356,13 @@ func (ctx *BlockContext) traverseExpr(node syntax.Expr, globaleNames map[string] case *syntax.ParenExpr: node.X = ctx.traverseExpr(node.X, globaleNames, imports) case *syntax.SelectorExpr: - return ctx.trapSelector(node, node, false, globaleNames, imports) + newNode, selIsName := ctx.trapSelector(node, node, false, globaleNames, imports) + if newNode != nil { + return newNode + } + if !selIsName { + node.X = ctx.traverseExpr(node.X, globaleNames, imports) + } case *syntax.IndexExpr: node.X = ctx.traverseExpr(node.X, globaleNames, imports) node.Index = ctx.traverseExpr(node.Index, globaleNames, imports) @@ -357,21 +381,32 @@ func (ctx *BlockContext) traverseExpr(node syntax.Expr, globaleNames map[string] return res case *syntax.Operation: // take addr? - if node.Op != syntax.And || node.Y != nil { - node.X = ctx.traverseExpr(node.X, globaleNames, imports) - node.Y = ctx.traverseExpr(node.Y, globaleNames, imports) - return node + if node.Op == syntax.And && node.Y == nil { + // &a, + switch x := node.X.(type) { + case *syntax.Name: + return ctx.trapAddrNode(node, x, globaleNames) + case *syntax.SelectorExpr: + newNode, selIsName := ctx.trapSelector(node, x, true, globaleNames, imports) + if newNode != nil { + return newNode + } + if selIsName { + return node + } + } } - // &a, - switch x := node.X.(type) { - case *syntax.Name: - return ctx.trapAddrNode(node, x, globaleNames) - case *syntax.SelectorExpr: - return ctx.trapSelector(node, x, true, globaleNames, imports) - default: - node.X = ctx.traverseExpr(node.X, globaleNames, imports) - node.Y = ctx.traverseExpr(node.Y, globaleNames, imports) + if node.X != nil && node.Y != nil { + if ctx.OperationParent == nil { + ctx.OperationParent = make(map[syntax.Node]*syntax.Operation) + } + ctx.OperationParent[node.X] = node + ctx.OperationParent[node.Y] = node } + // x op y + node.X = ctx.traverseExpr(node.X, globaleNames, imports) + node.Y = ctx.traverseExpr(node.Y, globaleNames, imports) + return node case *syntax.CallExpr: // NOTE: we skip capturing a name as a function // node.Fun = ctx.traverseExpr(node.Fun, globaleNames, imports) @@ -426,7 +461,17 @@ func (c *BlockContext) trapValueNode(node *syntax.Name, globaleNames map[string] } // TODO: what about dot import? decl := globaleNames[name] - if decl == nil || !decl.Kind.IsVarOrConst() { + if decl == nil { + return node + } + if decl.Kind == Kind_Var || decl.Kind == Kind_VarPtr { + // good to go + } else if decl.Kind == Kind_Const { + if _, ok := c.OperationParent[node]; ok { + // directly inside an operation + return node + } + } else { return node } preStmts, tmpVarName := trapVar(node, syntax.NewName(node.Pos(), XgoLocalPkgName), node.Value, false) @@ -434,31 +479,36 @@ func (c *BlockContext) trapValueNode(node *syntax.Name, globaleNames map[string] return syntax.NewName(node.Pos(), tmpVarName) } -func (ctx *BlockContext) trapSelector(node syntax.Expr, sel *syntax.SelectorExpr, takeAddr bool, globaleNames map[string]*DeclInfo, imports map[string]string) syntax.Expr { +func (ctx *BlockContext) trapSelector(node syntax.Expr, sel *syntax.SelectorExpr, takeAddr bool, globaleNames map[string]*DeclInfo, imports map[string]string) (newExpr syntax.Expr, selIsName bool) { // form: pkg.var nameNode, ok := sel.X.(*syntax.Name) if !ok { - sel.X = ctx.traverseExpr(sel.X, globaleNames, imports) - return node + return nil, false } name := nameNode.Value if ctx.Has(name) { // local name - sel.X = ctx.traverseExpr(sel.X, globaleNames, imports) - return node + return nil, true } // import path pkgPath := imports[name] if pkgPath == "" { sel.X = ctx.trapValueNode(nameNode, globaleNames) - return node + return nil, true } if !allowPkgVarTrap(pkgPath) { - return node + return nil, true + } + pkgData := pkgdata.GetPkgData(pkgPath) + if pkgData.Consts[sel.Sel.Value] { + // is const and inside operation + if _, ok := ctx.OperationParent[node]; ok { + return nil, true + } } preStmts, tmpVarName := trapVar(node, newStringLit(pkgPath), sel.Sel.Value, takeAddr) ctx.InsertList = append(ctx.InsertList, preStmts...) - return syntax.NewName(node.Pos(), tmpVarName) + return syntax.NewName(node.Pos(), tmpVarName), true } func (c *BlockContext) trapAddrNode(node *syntax.Operation, nameNode *syntax.Name, globaleNames map[string]*DeclInfo) syntax.Expr { diff --git a/runtime/core/version.go b/runtime/core/version.go index c665affb..c588e223 100644 --- a/runtime/core/version.go +++ b/runtime/core/version.go @@ -7,8 +7,8 @@ import ( ) const VERSION = "1.0.18" -const REVISION = "2c003e4f895ef51b9c22e7c5bb5018dd6007ff96+1" -const NUMBER = 164 +const REVISION = "86b60236af7fe7147b90073421f57187fbf6990a+1" +const NUMBER = 165 // these fields will be filled by compiler const XGO_VERSION = "" diff --git a/runtime/test/patch/patch_var_test.go b/runtime/test/patch/patch_var_test.go index 1af5091d..82aec209 100644 --- a/runtime/test/patch/patch_var_test.go +++ b/runtime/test/patch/patch_var_test.go @@ -3,8 +3,10 @@ package patch import ( "fmt" "testing" + "unsafe" "github.com/xhd2015/xgo/runtime/mock" + "github.com/xhd2015/xgo/runtime/test/patch/sub" ) var a int = 123 @@ -45,10 +47,11 @@ func TestPatchVarWrongTypeShouldFailTest(t *testing.T) { } } -const pkgName = "github.com/xhd2015/xgo/runtime/test/patch" +const pkgPath = "github.com/xhd2015/xgo/runtime/test/patch" +const subPkgPath = "github.com/xhd2015/xgo/runtime/test/patch/sub" func TestPatchVarByNameTest(t *testing.T) { - mock.PatchByName(pkgName, "a", func() int { + mock.PatchByName(pkgPath, "a", func() int { return 456 }) b := a @@ -58,7 +61,7 @@ func TestPatchVarByNameTest(t *testing.T) { } func TestPatchVarByNamePtrTest(t *testing.T) { - mock.PatchByName(pkgName, "a", func() *int { + mock.PatchByName(pkgPath, "a", func() *int { x := 456 return &x }) @@ -72,7 +75,7 @@ func TestPatchVarByNamePtrTest(t *testing.T) { const testVersion = "1.0" func TestPatchConstByNamePtrTest(t *testing.T) { - mock.PatchByName(pkgName, "testVersion", func() string { + mock.PatchByName(pkgPath, "testVersion", func() string { return "1.5" }) version := testVersion @@ -87,7 +90,7 @@ func TestPatchConstByNameWrongTypeShouldFail(t *testing.T) { defer func() { pe = recover() }() - mock.PatchByName(pkgName, "a", func() string { + mock.PatchByName(pkgPath, "a", func() string { return "1.5" }) }() @@ -100,3 +103,33 @@ func TestPatchConstByNameWrongTypeShouldFail(t *testing.T) { t.Fatalf("expect err %q, actual: %q", expectMsg, msg) } } + +const N = 50 + +func TestPatchConstOperationShouldCompileAndSkipMock(t *testing.T) { + // should have no effect + mock.PatchByName(pkgPath, "N", func() int { + return 10 + }) + // because N is used inside an operation + // it's type is not yet determined, so + // should not rewrite it + size := N * unsafe.Sizeof(int(0)) + if size != 400 { + t.Logf("expect N not patched and size to be %d, actual: %d\n", 400, size) + } +} + +func TestPatchOtherPkgConstOperationShouldCompileAndSkipMock(t *testing.T) { + // should have no effect + mock.PatchByName(subPkgPath, "N", func() int { + return 10 + }) + // because N is used inside an operation + // it's type is not yet determined, so + // should not rewrite it + size := sub.N * unsafe.Sizeof(int(0)) + if size != 400 { + t.Logf("expect N not patched and size to be %d, actual: %d\n", 400, size) + } +} diff --git a/runtime/test/patch/sub/sub.go b/runtime/test/patch/sub/sub.go new file mode 100644 index 00000000..8348c677 --- /dev/null +++ b/runtime/test/patch/sub/sub.go @@ -0,0 +1,3 @@ +package sub + +const N = 50