Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

switch to libopenapi #194

Merged
merged 1 commit into from
Sep 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,21 @@

module github.com/metio/terraform-provider-k8s

go 1.22.0
go 1.23.0

toolchain go1.22.2
toolchain go1.23.1

require (
github.com/getkin/kin-openapi v0.127.0
github.com/gruntwork-io/terratest v0.47.1
github.com/hashicorp/terraform-plugin-docs v0.19.4
github.com/hashicorp/terraform-plugin-framework v1.11.0
github.com/hashicorp/terraform-plugin-framework-validators v0.13.0
github.com/hashicorp/terraform-plugin-go v0.23.0
github.com/hashicorp/terraform-plugin-log v0.9.0
github.com/iancoleman/strcase v0.3.0
github.com/pb33f/libopenapi v0.18.1
github.com/stretchr/testify v1.9.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/apiextensions-apiserver v0.31.1
k8s.io/apimachinery v0.31.1
k8s.io/client-go v0.31.1
Expand All @@ -42,13 +43,16 @@ require (
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/armon/go-radix v1.0.0 // indirect
github.com/aws/aws-sdk-go v1.44.122 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dprotaso/go-yit v0.0.0-20240618133044-5a0af90af097 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
github.com/fatih/color v1.16.0 // indirect
Expand All @@ -60,6 +64,7 @@ require (
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-test/deep v1.0.8 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
Expand Down Expand Up @@ -95,7 +100,6 @@ require (
github.com/huandu/xstrings v1.3.3 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/invopop/yaml v0.3.1 // indirect
github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand All @@ -116,12 +120,10 @@ require (
github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/perimeterx/marshmallow v1.1.5 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
Expand All @@ -135,6 +137,8 @@ require (
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.9-0.20240815153524-6ea36470d1bd // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
github.com/yuin/goldmark v1.7.1 // indirect
Expand Down Expand Up @@ -167,7 +171,6 @@ require (
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.31.1 // indirect
k8s.io/cli-runtime v0.31.1 // indirect
k8s.io/component-base v0.31.1 // indirect
Expand Down
63 changes: 53 additions & 10 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion tools/generator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func main() {
var data []*generator.TemplateData
if *parseOpenAPIv2 {
openapi := generator.ParseOpenAPIv2Files(fmt.Sprintf("%s/openapi_v2/", *schemaDir))
data = append(data, generator.ConvertOpenAPIv3(openapi)...)
data = append(data, generator.ConvertOpenAPIv2(openapi)...)
}
if *parseCRDv1 {
crd := generator.ParseCRDv1Files(fmt.Sprintf("%s/crd_v1/", *schemaDir))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,43 +7,51 @@ package generator

import (
"fmt"
"github.com/getkin/kin-openapi/openapi3"
"github.com/pb33f/libopenapi/datamodel/high/base"
v2high "github.com/pb33f/libopenapi/datamodel/high/v2"
"k8s.io/utils/strings/slices"
"sort"
"strings"
)

func ConvertOpenAPIv3(schemas []map[string]*openapi3.SchemaRef) []*TemplateData {
func ConvertOpenAPIv2(schemas []v2high.Swagger) []*TemplateData {
data := make([]*TemplateData, 0)
for _, schema := range schemas {
for name, definition := range schema {
if supportedOpenAPIv3Object(name, definition) {
for name, definition := range schema.Definitions.Definitions.FromNewest() {
if supportedOpenAPIv2Object(name, definition) {
namespaced := isNamespacedObject(name)
templateData := openAPIv3AsTemplateData(definition, namespaced)
data = append(data, templateData)
if templateData := openAPIv2AsTemplateData(definition, namespaced); templateData != nil {
data = append(data, templateData)
}
}
}
}
return data
}

func openAPIv3AsTemplateData(definition *openapi3.SchemaRef, namespaced bool) *TemplateData {
func openAPIv2AsTemplateData(definition *base.SchemaProxy, namespaced bool) *TemplateData {
var group string
var version string
var kind string
if gvkExt, ok := definition.Value.Extensions["x-kubernetes-group-version-kind"]; ok {
raw := gvkExt.([]interface{})
gvk := raw[0].(map[string]interface{})
group = gvk["group"].(string)
version = gvk["version"].(string)
kind = gvk["kind"].(string)
if node, present := definition.Schema().Extensions.Get("x-kubernetes-group-version-kind"); present {
var gvk []map[string]string
err := node.Decode(&gvk)
if err != nil {
fmt.Println(err)
return nil
}
group = gvk[0]["group"]
version = gvk[0]["version"]
kind = gvk[0]["kind"]
} else {
return nil
}
schema := definition.Value
schema := definition.Schema()
// remove manually managed or otherwise ignored properties
delete(schema.Properties, "metadata")
delete(schema.Properties, "status")
delete(schema.Properties, "apiVersion")
delete(schema.Properties, "kind")
schema.Properties.Delete("metadata")
schema.Properties.Delete("status")
schema.Properties.Delete("apiVersion")
schema.Properties.Delete("kind")

imports := AdditionalImports{}
typeName := resourceTypeName(group, kind, version)
Expand Down Expand Up @@ -88,36 +96,38 @@ func openAPIv3AsTemplateData(definition *openapi3.SchemaRef, namespaced bool) *T
TerraformModelType: terraformModelType(group, kind, version),

AdditionalImports: imports,
Properties: openAPIv3Properties(schema, &imports, "", typeName),
Properties: openAPIv2Properties(schema, &imports, "", typeName),
}
}

func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, path string, terraformResourceName string) []*Property {
func openAPIv2Properties(schema *base.Schema, imports *AdditionalImports, path string, terraformResourceName string) []*Property {
props := make([]*Property, 0)

if schema != nil {
for name, prop := range schema.Properties {
if prop.Value != nil {
for name, prop := range schema.Properties.FromNewest() {
if prop.Schema() != nil {
propPath := propertyPath(path, name)
if ignored, ok := ignoredAttributes[terraformResourceName]; ok {
if slices.Contains(ignored, propPath) {
continue
}
}

schemaType := prop.Schema().Type[0]

var nestedProperties []*Property
if prop.Value.Type.Is(openapi3.TypeArray) && prop.Value.Items != nil && prop.Value.Items.Value != nil && prop.Value.Items.Value.Type.Is(openapi3.TypeObject) {
nestedProperties = openAPIv3Properties(prop.Value.Items.Value, imports, propPath, terraformResourceName)
} else if prop.Value.Type.Is(openapi3.TypeObject) && prop.Value.AdditionalProperties.Schema != nil && prop.Value.AdditionalProperties.Schema.Value.Type.Is(openapi3.TypeObject) {
nestedProperties = openAPIv3Properties(prop.Value.AdditionalProperties.Schema.Value, imports, propPath, terraformResourceName)
if schemaType == "array" && prop.Schema().Items != nil && prop.Schema().Items.IsA() && prop.Schema().Items.A.Schema().Type[0] == "object" {
nestedProperties = openAPIv2Properties(prop.Schema().Items.A.Schema(), imports, propPath, terraformResourceName)
} else if schemaType == "object" && prop.Schema().AdditionalProperties != nil && prop.Schema().AdditionalProperties.IsA() && prop.Schema().AdditionalProperties.A.Schema().Type[0] == "object" {
nestedProperties = openAPIv2Properties(prop.Schema().AdditionalProperties.A.Schema(), imports, propPath, terraformResourceName)
} else {
nestedProperties = openAPIv3Properties(prop.Value, imports, propPath, terraformResourceName)
nestedProperties = openAPIv2Properties(prop.Schema(), imports, propPath, terraformResourceName)
}

attributeType, valueType, elementType, goType, customType := translateTypeWith(&openapiv3TypeTranslator{property: prop.Value}, terraformResourceName, propPath)
attributeType, valueType, elementType, goType, customType := translateTypeWith(&openapiv2TypeTranslator{property: prop.Schema()}, terraformResourceName, propPath)

validators := validatorsFor(&openapiv3ValidatorExtractor{
property: prop.Value,
validators := validatorsFor(&openapiv2ValidatorExtractor{
property: prop.Schema(),
imports: imports,
}, terraformResourceName, propPath, imports)

Expand All @@ -135,7 +145,7 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
TerraformElementType: elementType,
TerraformCustomType: customType,
TerraformValueType: valueType,
Description: description(prop.Value.Description),
Description: description(prop.Schema().Description),
Required: slices.Contains(schema.Required, name),
Optional: !slices.Contains(schema.Required, name),
Computed: false,
Expand All @@ -152,11 +162,11 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
return props[i].Name < props[j].Name
})

if schema.MinProps > 0 && schema.MaxProps != nil {
min := schema.MinProps
max := schema.MaxProps
if schema.MinProperties != nil && schema.MaxProperties != nil {
minProperties := *schema.MinProperties
maxProperties := schema.MaxProperties

if min == 1 && *max == 1 {
if minProperties == 1 && *maxProperties == 1 {
for _, outer := range props {
var pathExpressions []string
for _, inner := range props {
Expand All @@ -170,10 +180,10 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
imports.Path = true
}
}
} else if schema.MinProps > 0 && schema.MaxProps == nil {
min := schema.MinProps
} else if schema.MinProperties != nil && schema.MaxProperties == nil {
minProperties := *schema.MinProperties

if min == 1 {
if minProperties == 1 {
for _, outer := range props {
var pathExpressions []string
for _, inner := range props {
Expand All @@ -186,7 +196,7 @@ func openAPIv3Properties(schema *openapi3.Schema, imports *AdditionalImports, pa
addValidatorImports(outer, imports)
imports.Path = true
}
} else if min > 1 && min == uint64(len(props)) {
} else if minProperties > 1 && minProperties == int64(len(props)) {
for _, prop := range props {
prop.Required = true
prop.Optional = false
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
package generator

import (
"github.com/getkin/kin-openapi/openapi3"
"github.com/pb33f/libopenapi/datamodel/high/base"
"k8s.io/utils/strings/slices"
"strings"
)
Expand Down Expand Up @@ -56,10 +56,10 @@ var supportedKubernetesApiObjects = []string{
"io.k8s.kube-aggregator.pkg.apis.apiregistration.v1.APIService",
}

func supportedOpenAPIv3Object(name string, definition *openapi3.SchemaRef) bool {
func supportedOpenAPIv2Object(name string, definition *base.SchemaProxy) bool {
if !strings.HasPrefix(name, "io.k8s") || slices.Contains(supportedKubernetesApiObjects, name) {
if _, ok := definition.Value.Extensions["x-kubernetes-group-version-kind"]; ok {
if len(definition.Value.Properties) > 0 {
if gvk := definition.Schema().Extensions.GetOrZero("x-kubernetes-group-version-kind"); gvk != nil {
if definition.Schema().Properties.Len() > 0 {
return true
}
}
Expand Down
75 changes: 18 additions & 57 deletions tools/internal/generator/openapiv2_parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,46 +6,37 @@
package generator

import (
"encoding/json"
"github.com/getkin/kin-openapi/openapi2"
"github.com/getkin/kin-openapi/openapi2conv"
"github.com/getkin/kin-openapi/openapi3"
"fmt"
"github.com/pb33f/libopenapi"
v2high "github.com/pb33f/libopenapi/datamodel/high/v2"
"io/fs"
"log"
"net/url"
"os"
"path/filepath"
"strings"
)

var openapi3Loader *openapi3.Loader

func init() {
openapi3Loader = openapi3.NewLoader()
openapi3Loader.IsExternalRefsAllowed = true
openapi3Loader.ReadFromURIFunc = func(loader *openapi3.Loader, uri *url.URL) ([]byte, error) {
return os.ReadFile(uri.Path)
}
}

func ParseOpenAPIv2Files(basePath string) []map[string]*openapi3.SchemaRef {
schemas := make([]map[string]*openapi3.SchemaRef, 0)
func ParseOpenAPIv2Files(basePath string) []v2high.Swagger {
schemas := make([]v2high.Swagger, 0)

err := filepath.WalkDir(basePath, func(path string, d fs.DirEntry, err error) error {
if !d.IsDir() && strings.HasSuffix(path, ".json") {
openapiv2, parseErr := parseOpenAPIv2(path)
if parseErr != nil {
return parseErr
fileContent, readErr := os.ReadFile(path)
if readErr != nil {
return readErr
}
openapiv3, conversionErr := convertV2toV3(openapiv2)
if conversionErr != nil {
return conversionErr
document, docErr := libopenapi.NewDocument(fileContent)
if docErr != nil {
return docErr
}
resolveErr := resolveReferences(path, openapiv3)
if resolveErr != nil {
return resolveErr
docModel, errors := document.BuildV2Model()
if len(errors) > 0 {
for i := range errors {
fmt.Printf("error: %e\n", errors[i])
}
panic(fmt.Sprintf("cannot create v3 model from document: %d errors reported", len(errors)))
}
schemas = append(schemas, openapiv3.Components.Schemas)
schemas = append(schemas, docModel.Model)
}
return nil
})
Expand All @@ -55,33 +46,3 @@ func ParseOpenAPIv2Files(basePath string) []map[string]*openapi3.SchemaRef {

return schemas
}

func parseOpenAPIv2(filePath string) (*openapi2.T, error) {
input, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
var doc openapi2.T
err = json.Unmarshal(input, &doc)
if err != nil {
return nil, err
}
return &doc, nil

}

func resolveReferences(filePath string, v3 *openapi3.T) error {
err := openapi3Loader.ResolveRefsIn(v3, &url.URL{Path: filePath})
if err != nil {
return err
}
return nil
}

func convertV2toV3(doc *openapi2.T) (*openapi3.T, error) {
v3, err := openapi2conv.ToV3(doc)
if err != nil {
return nil, err
}
return v3, nil
}
Loading