From e5c469a4874776cfe3eddd457f54bcd27a5e8b08 Mon Sep 17 00:00:00 2001 From: AnmolxSingh Date: Wed, 26 Feb 2025 22:39:26 +0530 Subject: [PATCH 1/6] ast main file Signed-off-by: AnmolxSingh --- cmd/config-doc-gen/main.go | 206 +++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 cmd/config-doc-gen/main.go diff --git a/cmd/config-doc-gen/main.go b/cmd/config-doc-gen/main.go new file mode 100644 index 00000000000..50bedc2fd7f --- /dev/null +++ b/cmd/config-doc-gen/main.go @@ -0,0 +1,206 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "strings" +) + +// FieldDoc represents documentation for a struct field. +type FieldDoc struct { + Name string `json:"name"` + Type string `json:"type"` + Tag string `json:"tag,omitempty"` + DefaultValue interface{} `json:"default_value,omitempty"` + Comment string `json:"comment,omitempty"` +} + +// StructDoc represents documentation for a struct. +type StructDoc struct { + Name string `json:"name"` + Fields []FieldDoc `json:"fields"` + Comment string `json:"comment,omitempty"` +} + +func main() { + // List of target directories + targetDirs := []string{ + "cmd/jaeger/internal/extension/jaegerquery", + "pkg/es/config", + } + + var allStructs []StructDoc + + // Iterate over each target directory + for _, dir := range targetDirs { + fmt.Printf("Parsing directory: %s\n", dir) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Process only Go source files + if !info.IsDir() && filepath.Ext(path) == ".go" { + structs, err := parseFile(path) + if err != nil { + log.Printf("Error parsing file %s: %v", path, err) + return err + } + allStructs = append(allStructs, structs...) + } + return nil + }) + if err != nil { + log.Fatalf("Error walking the path %s: %v", dir, err) + } + } + + // Serialize the collected struct documentation to JSON + outputFile, err := os.Create("struct_docs.json") + if err != nil { + log.Fatalf("Error creating output file: %v", err) + } + defer outputFile.Close() + + encoder := json.NewEncoder(outputFile) + encoder.SetIndent("", " ") + if err := encoder.Encode(allStructs); err != nil { + log.Fatalf("Error encoding JSON: %v", err) + } + + fmt.Println("Struct documentation has been written to struct_docs.json") +} + +// parseFile parses a Go source file and extracts struct information. +func parseFile(filePath string) ([]StructDoc, error) { + var structs []StructDoc + + // Create a new token file set + fset := token.NewFileSet() + + // Parse the Go source file + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("failed to parse file %s: %w", filePath, err) + } + + // Traverse the AST to find struct type declarations + ast.Inspect(node, func(n ast.Node) bool { + // Check for type declarations + genDecl, ok := n.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + return true + } + + // Iterate over the type specifications + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + + // Check if the type is a struct + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + continue + } + + structDoc := StructDoc{ + Name: typeSpec.Name.Name, + Comment: extractComment(genDecl.Doc), + } + + // Iterate over the struct fields + for _, field := range structType.Fields.List { + fieldType := exprToString(fset, field.Type) + fieldTag := extractTag(field.Tag) + defaultValue := extractDefaultValue(field.Tag, fieldType) + + for _, name := range field.Names { + structDoc.Fields = append(structDoc.Fields, FieldDoc{ + Name: name.Name, + Type: fieldType, + Tag: fieldTag, + DefaultValue: defaultValue, + Comment: extractComment(field.Doc), + }) + } + } + structs = append(structs, structDoc) + } + return false + }) + + return structs, nil +} + +// extractComment retrieves the text from a CommentGroup. +func extractComment(cg *ast.CommentGroup) string { + if cg != nil { + return strings.TrimSpace(cg.Text()) + } + return "" +} + +// exprToString converts an ast.Expr to its string representation. +func exprToString(fset *token.FileSet, expr ast.Expr) string { + var buf bytes.Buffer + if err := printer.Fprint(&buf, fset, expr); err != nil { + return "" + } + return buf.String() +} + +// extractTag extracts the tag value from a BasicLit. +func extractTag(tag *ast.BasicLit) string { + if tag != nil { + return strings.Trim(tag.Value, "`") + } + return "" +} + +// extractDefaultValue parses the tag to find a default value if specified. +func extractDefaultValue(tag *ast.BasicLit, fieldType string) interface{} { + if tag == nil { + return nil + } + tagValue := extractTag(tag) + structTag := reflect.StructTag(tagValue) + defaultValueStr := structTag.Get("default") + if defaultValueStr == "" { + return nil + } + + // Convert the default value string to the appropriate type + switch fieldType { + case "string": + return defaultValueStr + case "int", "int8", "int16", "int32", "int64": + var intValue int64 + fmt.Sscanf(defaultValueStr, "%d", &intValue) + return intValue + case "uint", "uint8", "uint16", "uint32", "uint64": + var uintValue uint64 + fmt.Sscanf(defaultValueStr, "%d", &uintValue) + return uintValue + case "float32", "float64": + var floatValue float64 + fmt.Sscanf(defaultValueStr, "%f", &floatValue) + return floatValue + case "bool": + var boolValue bool + fmt.Sscanf(defaultValueStr, "%t", &boolValue) + return boolValue + // Add more types as needed + default: + return nil + } +} From e24150886cfd107b29e898131f5e2ad464ca438b Mon Sep 17 00:00:00 2001 From: AnmolxSingh Date: Wed, 26 Feb 2025 23:39:32 +0530 Subject: [PATCH 2/6] license added Signed-off-by: AnmolxSingh --- cmd/config-doc-gen/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/config-doc-gen/main.go b/cmd/config-doc-gen/main.go index 50bedc2fd7f..365c856d85b 100644 --- a/cmd/config-doc-gen/main.go +++ b/cmd/config-doc-gen/main.go @@ -1,3 +1,6 @@ +// Copyright (c) 2025 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + package main import ( From 7a1909c895e69a9174798067fc171e4c9b36bdf8 Mon Sep 17 00:00:00 2001 From: Anmol <166167480+AnmolxSingh@users.noreply.github.com> Date: Thu, 27 Feb 2025 01:22:19 +0530 Subject: [PATCH 3/6] Update cmd/config-doc-gen/main.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Anmol <166167480+AnmolxSingh@users.noreply.github.com> --- cmd/config-doc-gen/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/config-doc-gen/main.go b/cmd/config-doc-gen/main.go index 365c856d85b..44701371cbf 100644 --- a/cmd/config-doc-gen/main.go +++ b/cmd/config-doc-gen/main.go @@ -188,7 +188,9 @@ func extractDefaultValue(tag *ast.BasicLit, fieldType string) interface{} { return defaultValueStr case "int", "int8", "int16", "int32", "int64": var intValue int64 - fmt.Sscanf(defaultValueStr, "%d", &intValue) + if _, err := fmt.Sscanf(defaultValueStr, "%d", &intValue); err != nil { + return nil + } return intValue case "uint", "uint8", "uint16", "uint32", "uint64": var uintValue uint64 From 4a48db764d30fa3748ef7225aea6ff454d1d35cd Mon Sep 17 00:00:00 2001 From: Anmol <166167480+AnmolxSingh@users.noreply.github.com> Date: Thu, 27 Feb 2025 17:50:50 +0530 Subject: [PATCH 4/6] Update cmd/config-doc-gen/main.go Co-authored-by: Yuri Shkuro Signed-off-by: Anmol <166167480+AnmolxSingh@users.noreply.github.com> --- cmd/config-doc-gen/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/config-doc-gen/main.go b/cmd/config-doc-gen/main.go index 44701371cbf..fe62b7432b6 100644 --- a/cmd/config-doc-gen/main.go +++ b/cmd/config-doc-gen/main.go @@ -189,7 +189,7 @@ func extractDefaultValue(tag *ast.BasicLit, fieldType string) interface{} { case "int", "int8", "int16", "int32", "int64": var intValue int64 if _, err := fmt.Sscanf(defaultValueStr, "%d", &intValue); err != nil { - return nil + return err } return intValue case "uint", "uint8", "uint16", "uint32", "uint64": From 5f3fa6ab51d5098745807a67956cac63af20546a Mon Sep 17 00:00:00 2001 From: AnmolxSingh Date: Sat, 1 Mar 2025 01:14:36 +0530 Subject: [PATCH 5/6] recursive struc doc added Signed-off-by: AnmolxSingh --- cmd/config-doc-gen/main.go | 61 ++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/cmd/config-doc-gen/main.go b/cmd/config-doc-gen/main.go index fe62b7432b6..07b5c1846dd 100644 --- a/cmd/config-doc-gen/main.go +++ b/cmd/config-doc-gen/main.go @@ -82,10 +82,47 @@ func main() { fmt.Println("Struct documentation has been written to struct_docs.json") } +// processField processes a struct field and handles nested structs. +func processField(fset *token.FileSet, field *ast.Field, structs map[string]StructDoc) []FieldDoc { + var fieldDocs []FieldDoc + fieldType := exprToString(fset, field.Type) + fieldTag := extractTag(field.Tag) + defaultValue := extractDefaultValue(field.Tag, fieldType) + + for _, name := range field.Names { + fieldDoc := FieldDoc{ + Name: name.Name, + Type: fieldType, + Tag: fieldTag, + DefaultValue: defaultValue, + Comment: extractComment(field.Doc), + } + fieldDocs = append(fieldDocs, fieldDoc) + } + + // Handle nested struct + if ident, ok := field.Type.(*ast.Ident); ok && ident.Obj != nil { + if typeSpec, ok := ident.Obj.Decl.(*ast.TypeSpec); ok { + if structType, isStruct := typeSpec.Type.(*ast.StructType); isStruct { + nestedStructDoc := StructDoc{ + Name: ident.Name, + Comment: extractComment(typeSpec.Doc), + } + for _, nestedField := range structType.Fields.List { + nestedStructDoc.Fields = append(nestedStructDoc.Fields, processField(fset, nestedField, structs)...) + } + structs[ident.Name] = nestedStructDoc + } + } + } + + return fieldDocs +} + // parseFile parses a Go source file and extracts struct information. func parseFile(filePath string) ([]StructDoc, error) { var structs []StructDoc - + structMap := make(map[string]StructDoc) // Create a new token file set fset := token.NewFileSet() @@ -123,25 +160,19 @@ func parseFile(filePath string) ([]StructDoc, error) { // Iterate over the struct fields for _, field := range structType.Fields.List { - fieldType := exprToString(fset, field.Type) - fieldTag := extractTag(field.Tag) - defaultValue := extractDefaultValue(field.Tag, fieldType) - - for _, name := range field.Names { - structDoc.Fields = append(structDoc.Fields, FieldDoc{ - Name: name.Name, - Type: fieldType, - Tag: fieldTag, - DefaultValue: defaultValue, - Comment: extractComment(field.Doc), - }) - } + structDoc.Fields = append(structDoc.Fields, processField(fset, field, structMap)...) } - structs = append(structs, structDoc) + // Store processed struct in the map + structMap[structDoc.Name] = structDoc + } return false }) + // Convert map to slice + for _, structDoc := range structMap { + structs = append(structs, structDoc) + } return structs, nil } From cbdb269d59ebd6fd99ccbe02c2782400bfda5795 Mon Sep 17 00:00:00 2001 From: AnmolxSingh Date: Sun, 2 Mar 2025 17:40:36 +0530 Subject: [PATCH 6/6] removed default tag Signed-off-by: AnmolxSingh --- cmd/config-doc-gen/main.go | 205 +++++++++++++++++++++++-------------- 1 file changed, 129 insertions(+), 76 deletions(-) diff --git a/cmd/config-doc-gen/main.go b/cmd/config-doc-gen/main.go index 07b5c1846dd..687e076a785 100644 --- a/cmd/config-doc-gen/main.go +++ b/cmd/config-doc-gen/main.go @@ -14,7 +14,6 @@ import ( "log" "os" "path/filepath" - "reflect" "strings" ) @@ -87,15 +86,13 @@ func processField(fset *token.FileSet, field *ast.Field, structs map[string]Stru var fieldDocs []FieldDoc fieldType := exprToString(fset, field.Type) fieldTag := extractTag(field.Tag) - defaultValue := extractDefaultValue(field.Tag, fieldType) for _, name := range field.Names { fieldDoc := FieldDoc{ - Name: name.Name, - Type: fieldType, - Tag: fieldTag, - DefaultValue: defaultValue, - Comment: extractComment(field.Doc), + Name: name.Name, + Type: fieldType, + Tag: fieldTag, + Comment: extractComment(field.Doc), } fieldDocs = append(fieldDocs, fieldDoc) } @@ -123,52 +120,68 @@ func processField(fset *token.FileSet, field *ast.Field, structs map[string]Stru func parseFile(filePath string) ([]StructDoc, error) { var structs []StructDoc structMap := make(map[string]StructDoc) + defaultValues := make(map[string]map[string]interface{}) // Store default values for structs + // Create a new token file set fset := token.NewFileSet() - - // Parse the Go source file node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) if err != nil { return nil, fmt.Errorf("failed to parse file %s: %w", filePath, err) } - // Traverse the AST to find struct type declarations + // Traverse the AST to find struct type declarations and functions ast.Inspect(node, func(n ast.Node) bool { - // Check for type declarations - genDecl, ok := n.(*ast.GenDecl) - if !ok || genDecl.Tok != token.TYPE { - return true - } - - // Iterate over the type specifications - for _, spec := range genDecl.Specs { - typeSpec, ok := spec.(*ast.TypeSpec) - if !ok { - continue + switch n := n.(type) { + case *ast.GenDecl: // Handle struct declarations + if n.Tok != token.TYPE { + return true } + for _, spec := range n.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + continue + } - // Check if the type is a struct - structType, ok := typeSpec.Type.(*ast.StructType) - if !ok { - continue - } + structDoc := StructDoc{ + Name: typeSpec.Name.Name, + Comment: extractComment(n.Doc), + } - structDoc := StructDoc{ - Name: typeSpec.Name.Name, - Comment: extractComment(genDecl.Doc), - } + for _, field := range structType.Fields.List { + structDoc.Fields = append(structDoc.Fields, processField(fset, field, structMap)...) + } - // Iterate over the struct fields - for _, field := range structType.Fields.List { - structDoc.Fields = append(structDoc.Fields, processField(fset, field, structMap)...) + structMap[structDoc.Name] = structDoc } - // Store processed struct in the map - structMap[structDoc.Name] = structDoc + case *ast.FuncDecl: // Handle `createDefaultConfig()` + if n.Name.Name == "createDefaultConfig" { + structName, defaults := extractDefaultValues(fset, n) + + if structName != "" { + defaultValues[structName] = defaults + } + } } - return false + return true }) + // Apply extracted defaults to structs + for structName, defaults := range defaultValues { + if structDoc, exists := structMap[structName]; exists { + for i, field := range structDoc.Fields { + if val, found := defaults[field.Name]; found { + structDoc.Fields[i].DefaultValue = val + } + } + structMap[structName] = structDoc + } + } + // Convert map to slice for _, structDoc := range structMap { structs = append(structs, structDoc) @@ -176,6 +189,86 @@ func parseFile(filePath string) ([]StructDoc, error) { return structs, nil } +// function to extract default value from struct +func extractDefaultValues(fset *token.FileSet, fn *ast.FuncDecl) (string, map[string]interface{}) { + defaults := make(map[string]interface{}) + var structName string + + if fn.Body.List == nil { + return "", defaults + } + + for _, stmt := range fn.Body.List { + retStmt, ok := stmt.(*ast.ReturnStmt) + if !ok || len(retStmt.Results) == 0 { + continue + } + + unaryExpr, ok := retStmt.Results[0].(*ast.UnaryExpr) + if !ok { + continue + } + + compLit, ok := unaryExpr.X.(*ast.CompositeLit) + if !ok { + continue + } + + if ident, ok := compLit.Type.(*ast.Ident); ok { + structName = ident.Name + } + + for _, elt := range compLit.Elts { + kvExpr, ok := elt.(*ast.KeyValueExpr) + if !ok { + continue + } + + // Pass a valid fset instead of nil + fieldName := exprToString(fset, kvExpr.Key) + defaultValue := extractValue(kvExpr.Value) + defaults[fieldName] = defaultValue + } + } + + return structName, defaults +} + +// function to extract value +func extractValue(expr ast.Expr) interface{} { + switch v := expr.(type) { + case *ast.BasicLit: // Handle basic types + switch v.Kind { + case token.STRING: + return strings.Trim(v.Value, `"`) + case token.INT: + var intValue int + fmt.Sscanf(v.Value, "%d", &intValue) + return intValue + case token.FLOAT: + var floatValue float64 + fmt.Sscanf(v.Value, "%f", &floatValue) + return floatValue + default: + return v.Value + } + case *ast.Ident: // Handle boolean values + if v.Name == "true" { + return true + } else if v.Name == "false" { + return false + } + return v.Name // Could be a constant + case *ast.CallExpr: // Handle function calls + if fun, ok := v.Fun.(*ast.SelectorExpr); ok { + return fmt.Sprintf("function_call: %s.%s", fun.X, fun.Sel.Name) + } else if fun, ok := v.Fun.(*ast.Ident); ok { + return fmt.Sprintf("function_call: %s", fun.Name) + } + } + return nil +} + // extractComment retrieves the text from a CommentGroup. func extractComment(cg *ast.CommentGroup) string { if cg != nil { @@ -200,43 +293,3 @@ func extractTag(tag *ast.BasicLit) string { } return "" } - -// extractDefaultValue parses the tag to find a default value if specified. -func extractDefaultValue(tag *ast.BasicLit, fieldType string) interface{} { - if tag == nil { - return nil - } - tagValue := extractTag(tag) - structTag := reflect.StructTag(tagValue) - defaultValueStr := structTag.Get("default") - if defaultValueStr == "" { - return nil - } - - // Convert the default value string to the appropriate type - switch fieldType { - case "string": - return defaultValueStr - case "int", "int8", "int16", "int32", "int64": - var intValue int64 - if _, err := fmt.Sscanf(defaultValueStr, "%d", &intValue); err != nil { - return err - } - return intValue - case "uint", "uint8", "uint16", "uint32", "uint64": - var uintValue uint64 - fmt.Sscanf(defaultValueStr, "%d", &uintValue) - return uintValue - case "float32", "float64": - var floatValue float64 - fmt.Sscanf(defaultValueStr, "%f", &floatValue) - return floatValue - case "bool": - var boolValue bool - fmt.Sscanf(defaultValueStr, "%t", &boolValue) - return boolValue - // Add more types as needed - default: - return nil - } -}