switch to libopenapi

kin-openapi sometimes produces weird output for the statefulset resource - and only for that. No clue why but libopenapi seems to be running stable. Tested this by running the code generator multiple times without getting different results

Signed-off-by: Sebastian Hoß <[email protected]>

go.mod

@@ -3,20 +3,21 @@
 
 module github.com/metio/terraform-provider-k8s
 
-go 1.22.0
+go 1.23.0
 
-toolchain go1.22.2
+toolchain go1.23.1
 
 require (
-	github.com/getkin/kin-openapi v0.127.0
 	github.com/gruntwork-io/terratest v0.47.1
 	github.com/hashicorp/terraform-plugin-docs v0.19.4
 	github.com/hashicorp/terraform-plugin-framework v1.11.0
 	github.com/hashicorp/terraform-plugin-framework-validators v0.13.0
 	github.com/hashicorp/terraform-plugin-go v0.23.0
 	github.com/hashicorp/terraform-plugin-log v0.9.0
 	github.com/iancoleman/strcase v0.3.0
+	github.com/pb33f/libopenapi v0.18.1
 	github.com/stretchr/testify v1.9.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apiextensions-apiserver v0.31.1
 	k8s.io/apimachinery v0.31.1
 	k8s.io/client-go v0.31.1
@@ -42,13 +43,16 @@ require (
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/aws/aws-sdk-go v1.44.122 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dprotaso/go-yit v0.0.0-20240618133044-5a0af90af097 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fatih/color v1.16.0 // indirect
@@ -60,6 +64,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-test/deep v1.0.8 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
@@ -95,7 +100,6 @@ require (
 	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/invopop/yaml v0.3.1 // indirect
 	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -116,12 +120,10 @@ require (
 	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/oklog/run v1.1.0 // indirect
-	github.com/perimeterx/marshmallow v1.1.5 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -135,6 +137,8 @@ require (
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect
+	github.com/wk8/go-ordered-map/v2 v2.1.9-0.20240815153524-6ea36470d1bd // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/yuin/goldmark v1.7.1 // indirect
@@ -167,7 +171,6 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.31.1 // indirect
 	k8s.io/cli-runtime v0.31.1 // indirect
 	k8s.io/component-base v0.31.1 // indirect

tools/generator/main.go

@@ -33,7 +33,7 @@ func main() {
 	var data []*generator.TemplateData
 	if *parseOpenAPIv2 {
 		openapi := generator.ParseOpenAPIv2Files(fmt.Sprintf("%s/openapi_v2/", *schemaDir))
-		data = append(data, generator.ConvertOpenAPIv3(openapi)...)
+		data = append(data, generator.ConvertOpenAPIv2(openapi)...)
 	}
 	if *parseCRDv1 {
 		crd := generator.ParseCRDv1Files(fmt.Sprintf("%s/crd_v1/", *schemaDir))

tools/internal/generator/openapiv3_converter.go → tools/internal/generator/openapiv2_converter.go

@@ -7,43 +7,51 @@ package generator
 
 import (
 	"fmt"
-	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/pb33f/libopenapi/datamodel/high/base"
+	v2high "github.com/pb33f/libopenapi/datamodel/high/v2"
 	"k8s.io/utils/strings/slices"
 	"sort"
 	"strings"
 )
 
-func ConvertOpenAPIv3(schemas []map[string]*openapi3.SchemaRef) []*TemplateData {
+func ConvertOpenAPIv2(schemas []v2high.Swagger) []*TemplateData {
 	data := make([]*TemplateData, 0)
 	for _, schema := range schemas {
-		for name, definition := range schema {
-			if supportedOpenAPIv3Object(name, definition) {
+		for name, definition := range schema.Definitions.Definitions.FromNewest() {
+			if supportedOpenAPIv2Object(name, definition) {
 				namespaced := isNamespacedObject(name)
-				templateData := openAPIv3AsTemplateData(definition, namespaced)
-				data = append(data, templateData)
+				if templateData := openAPIv2AsTemplateData(definition, namespaced); templateData != nil {
+					data = append(data, templateData)
+				}
 			}
 		}
 	}
 	return data
 }
 
-func openAPIv3AsTemplateData(definition *openapi3.SchemaRef, namespaced bool) *TemplateData {
+func openAPIv2AsTemplateData(definition *base.SchemaProxy, namespaced bool) *TemplateData {
 	var group string
 	var version string
 	var kind string
-	if gvkExt, ok := definition.Value.Extensions["x-kubernetes-group-version-kind"]; ok {
-		raw := gvkExt.([]interface{})
-		gvk := raw[0].(map[string]interface{})
-		group = gvk["group"].(string)
-		version = gvk["version"].(string)
-		kind = gvk["kind"].(string)
+	if node, present := definition.Schema().Extensions.Get("x-kubernetes-group-version-kind"); present {
+		var gvk []map[string]string
+		err := node.Decode(&gvk)
+		if err != nil {
+			fmt.Println(err)
+			return nil
+		}
+		group = gvk[0]["group"]
+		version = gvk[0]["version"]
+		kind = gvk[0]["kind"]
+	} else {
+		return nil
 	}
-	schema := definition.Value
+	schema := definition.Schema()
 	// remove manually managed or otherwise ignored properties
-	delete(schema.Properties, "metadata")
-	delete(schema.Properties, "status")
-	delete(schema.Properties, "apiVersion")
-	delete(schema.Properties, "kind")
+	schema.Properties.Delete("metadata")
+	schema.Properties.Delete("status")
+	schema.Properties.Delete("apiVersion")
+	schema.Properties.Delete("kind")
 
 	imports := AdditionalImports{}
 	typeName := resourceTypeName(group, kind, version)
@@ -88,36 +96,38 @@ func openAPIv3AsTemplateData(definition *openapi3.SchemaRef, namespaced bool) *T
 		TerraformModelType: terraformModelType(group, kind, version),
 
 		AdditionalImports: imports,
-		Properties:        openAPIv3Properties(schema, &imports, "", typeName),
+		Properties:        openAPIv2Properties(schema, &imports, "", typeName),
 	}
 }
 
-func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, path string, terraformResourceName string) []*Property {
+func openAPIv2Properties(schema *base.Schema, imports *AdditionalImports, path string, terraformResourceName string) []*Property {
 	props := make([]*Property, 0)
 
 	if schema != nil {
-		for name, prop := range schema.Properties {
-			if prop.Value != nil {
+		for name, prop := range schema.Properties.FromNewest() {
+			if prop.Schema() != nil {
 				propPath := propertyPath(path, name)
 				if ignored, ok := ignoredAttributes[terraformResourceName]; ok {
 					if slices.Contains(ignored, propPath) {
 						continue
 					}
 				}
 
+				schemaType := prop.Schema().Type[0]
+
 				var nestedProperties []*Property
-				if prop.Value.Type.Is(openapi3.TypeArray) && prop.Value.Items != nil && prop.Value.Items.Value != nil && prop.Value.Items.Value.Type.Is(openapi3.TypeObject) {
-					nestedProperties = openAPIv3Properties(prop.Value.Items.Value, imports, propPath, terraformResourceName)
-				} else if prop.Value.Type.Is(openapi3.TypeObject) && prop.Value.AdditionalProperties.Schema != nil && prop.Value.AdditionalProperties.Schema.Value.Type.Is(openapi3.TypeObject) {
-					nestedProperties = openAPIv3Properties(prop.Value.AdditionalProperties.Schema.Value, imports, propPath, terraformResourceName)
+				if schemaType == "array" && prop.Schema().Items != nil && prop.Schema().Items.IsA() && prop.Schema().Items.A.Schema().Type[0] == "object" {
+					nestedProperties = openAPIv2Properties(prop.Schema().Items.A.Schema(), imports, propPath, terraformResourceName)
+				} else if schemaType == "object" && prop.Schema().AdditionalProperties != nil && prop.Schema().AdditionalProperties.IsA() && prop.Schema().AdditionalProperties.A.Schema().Type[0] == "object" {
+					nestedProperties = openAPIv2Properties(prop.Schema().AdditionalProperties.A.Schema(), imports, propPath, terraformResourceName)
 				} else {
-					nestedProperties = openAPIv3Properties(prop.Value, imports, propPath, terraformResourceName)
+					nestedProperties = openAPIv2Properties(prop.Schema(), imports, propPath, terraformResourceName)
 				}
 
-				attributeType, valueType, elementType, goType, customType := translateTypeWith(&openapiv3TypeTranslator{property: prop.Value}, terraformResourceName, propPath)
+				attributeType, valueType, elementType, goType, customType := translateTypeWith(&openapiv2TypeTranslator{property: prop.Schema()}, terraformResourceName, propPath)
 
-				validators := validatorsFor(&openapiv3ValidatorExtractor{
-					property: prop.Value,
+				validators := validatorsFor(&openapiv2ValidatorExtractor{
+					property: prop.Schema(),
 					imports:  imports,
 				}, terraformResourceName, propPath, imports)
 
@@ -135,7 +145,7 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
 					TerraformElementType:   elementType,
 					TerraformCustomType:    customType,
 					TerraformValueType:     valueType,
-					Description:            description(prop.Value.Description),
+					Description:            description(prop.Schema().Description),
 					Required:               slices.Contains(schema.Required, name),
 					Optional:               !slices.Contains(schema.Required, name),
 					Computed:               false,
@@ -152,11 +162,11 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
 		return props[i].Name < props[j].Name
 	})
 
-	if schema.MinProps > 0 && schema.MaxProps != nil {
-		min := schema.MinProps
-		max := schema.MaxProps
+	if schema.MinProperties != nil && schema.MaxProperties != nil {
+		minProperties := *schema.MinProperties
+		maxProperties := schema.MaxProperties
 
-		if min == 1 && *max == 1 {
+		if minProperties == 1 && *maxProperties == 1 {
 			for _, outer := range props {
 				var pathExpressions []string
 				for _, inner := range props {
@@ -170,10 +180,10 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
 				imports.Path = true
 			}
 		}
-	} else if schema.MinProps > 0 && schema.MaxProps == nil {
-		min := schema.MinProps
+	} else if schema.MinProperties != nil && schema.MaxProperties == nil {
+		minProperties := *schema.MinProperties
 
-		if min == 1 {
+		if minProperties == 1 {
 			for _, outer := range props {
 				var pathExpressions []string
 				for _, inner := range props {
@@ -186,7 +196,7 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
 				addValidatorImports(outer, imports)
 				imports.Path = true
 			}
-		} else if min > 1 && min == uint64(len(props)) {
+		} else if minProperties > 1 && minProperties == int64(len(props)) {
 			for _, prop := range props {
 				prop.Required = true
 				prop.Optional = false

tools/internal/generator/openapiv3_filter.go → tools/internal/generator/openapiv2_filter.go

@@ -6,7 +6,7 @@
 package generator
 
 import (
-	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/pb33f/libopenapi/datamodel/high/base"
 	"k8s.io/utils/strings/slices"
 	"strings"
 )
@@ -56,10 +56,10 @@ var supportedKubernetesApiObjects = []string{
 	"io.k8s.kube-aggregator.pkg.apis.apiregistration.v1.APIService",
 }
 
-func supportedOpenAPIv3Object(name string, definition *openapi3.SchemaRef) bool {
+func supportedOpenAPIv2Object(name string, definition *base.SchemaProxy) bool {
 	if !strings.HasPrefix(name, "io.k8s") || slices.Contains(supportedKubernetesApiObjects, name) {
-		if _, ok := definition.Value.Extensions["x-kubernetes-group-version-kind"]; ok {
-			if len(definition.Value.Properties) > 0 {
+		if gvk := definition.Schema().Extensions.GetOrZero("x-kubernetes-group-version-kind"); gvk != nil {
+			if definition.Schema().Properties.Len() > 0 {
 				return true
 			}
 		}

tools/internal/generator/openapiv2_parser.go

@@ -6,46 +6,37 @@
 package generator
 
 import (
-	"encoding/json"
-	"github.com/getkin/kin-openapi/openapi2"
-	"github.com/getkin/kin-openapi/openapi2conv"
-	"github.com/getkin/kin-openapi/openapi3"
+	"fmt"
+	"github.com/pb33f/libopenapi"
+	v2high "github.com/pb33f/libopenapi/datamodel/high/v2"
 	"io/fs"
 	"log"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 )
 
-var openapi3Loader *openapi3.Loader
-
-func init() {
-	openapi3Loader = openapi3.NewLoader()
-	openapi3Loader.IsExternalRefsAllowed = true
-	openapi3Loader.ReadFromURIFunc = func(loader *openapi3.Loader, uri *url.URL) ([]byte, error) {
-		return os.ReadFile(uri.Path)
-	}
-}
-
-func ParseOpenAPIv2Files(basePath string) []map[string]*openapi3.SchemaRef {
-	schemas := make([]map[string]*openapi3.SchemaRef, 0)
+func ParseOpenAPIv2Files(basePath string) []v2high.Swagger {
+	schemas := make([]v2high.Swagger, 0)
 
 	err := filepath.WalkDir(basePath, func(path string, d fs.DirEntry, err error) error {
 		if !d.IsDir() && strings.HasSuffix(path, ".json") {
-			openapiv2, parseErr := parseOpenAPIv2(path)
-			if parseErr != nil {
-				return parseErr
+			fileContent, readErr := os.ReadFile(path)
+			if readErr != nil {
+				return readErr
 			}
-			openapiv3, conversionErr := convertV2toV3(openapiv2)
-			if conversionErr != nil {
-				return conversionErr
+			document, docErr := libopenapi.NewDocument(fileContent)
+			if docErr != nil {
+				return docErr
 			}
-			resolveErr := resolveReferences(path, openapiv3)
-			if resolveErr != nil {
-				return resolveErr
+			docModel, errors := document.BuildV2Model()
+			if len(errors) > 0 {
+				for i := range errors {
+					fmt.Printf("error: %e\n", errors[i])
+				}
+				panic(fmt.Sprintf("cannot create v3 model from document: %d errors reported", len(errors)))
 			}
-			schemas = append(schemas, openapiv3.Components.Schemas)
+			schemas = append(schemas, docModel.Model)
 		}
 		return nil
 	})
@@ -55,33 +46,3 @@ func ParseOpenAPIv2Files(basePath string) []map[string]*openapi3.SchemaRef {
 
 	return schemas
 }
-
-func parseOpenAPIv2(filePath string) (*openapi2.T, error) {
-	input, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, err
-	}
-	var doc openapi2.T
-	err = json.Unmarshal(input, &doc)
-	if err != nil {
-		return nil, err
-	}
-	return &doc, nil
-
-}
-
-func resolveReferences(filePath string, v3 *openapi3.T) error {
-	err := openapi3Loader.ResolveRefsIn(v3, &url.URL{Path: filePath})
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func convertV2toV3(doc *openapi2.T) (*openapi3.T, error) {
-	v3, err := openapi2conv.ToV3(doc)
-	if err != nil {
-		return nil, err
-	}
-	return v3, nil
-}