Skip to content

Commit

Permalink
Add aliases from compatible paths (#3906)
Browse files Browse the repository at this point in the history
1. Generate a report after generation with just the resource paths for
each token.
2. Manually copy this into the version folder to be used as a source for
the next version.
3. Create a lookup which finds "compatible resources" based on the path
being the same or the module/resource names matching (case
insensitively).
4. Add aliases for the compatible resources.
  • Loading branch information
danielrbradley authored Jan 30, 2025
1 parent e9d778c commit b342515
Show file tree
Hide file tree
Showing 11 changed files with 22,799 additions and 41 deletions.
13 changes: 13 additions & 0 deletions provider/pkg/gen/__snapshots__/gen_aliases_test_v3.snap
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,11 @@
"repository": "https://github.com/pulumi/pulumi-azure-native",
"resources": {
"azure-native:aadiam/v20200301:PrivateEndpointConnection": {
"aliases": [
{
"type": "azure-native:aadiam:PrivateEndpointConnection"
}
],
"description": "Private endpoint connection resource.",
"inputProperties": {
"policyName": {
Expand Down Expand Up @@ -507,6 +512,14 @@
"type": "object"
},
"azure-native:aadiam/v20200301:PrivateLinkForAzureAd": {
"aliases": [
{
"type": "azure-native:aadiam/v20200301preview:PrivateLinkForAzureAd"
},
{
"type": "azure-native:aadiam:PrivateLinkForAzureAd"
}
],
"description": "PrivateLink Policy configuration object.",
"inputProperties": {
"allTenants": {
Expand Down
6 changes: 6 additions & 0 deletions provider/pkg/gen/__snapshots__/gen_dashboard_test.snap
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,14 @@
[TestPortalDashboardGen - 1]
{
"aliases": [
{
"type": "azure-native:portal/v20190101preview:Dashboard"
},
{
"type": "azure-native:portal/v20200901preview:Dashboard"
},
{
"type": "azure-native:portal/v20221201preview:Dashboard"
}
],
"description": "The shared dashboard resource definition.\nAzure REST API version: 2020-09-01-preview.",
Expand Down
92 changes: 92 additions & 0 deletions provider/pkg/gen/compatibleTokens.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package gen

import (
"fmt"
"strings"

"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/collections"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/openapi"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/openapi/paths"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/resources"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/util"
)

type Token string
type ModuleLowered string
type ResourceLowered string
type NormalizedPath string

// CompatibleTokensLookup is a lookup table for compatible tokens based on either name matches or their URI paths in Azure.
// Comparisons are done using normalisation for casing and path parameter naming.
type CompatibleTokensLookup struct {
byLoweredModuleResourceName map[ModuleLowered]map[ResourceLowered]map[Token]struct{}
byNormalizedPath map[NormalizedPath]map[Token]struct{}
}

// NewCompatibleTokensLookup creates a new CompatibleResourceLookup from the collection of tokens mapping to their Azure path.
// Note: tokenPaths are outputted for the current provider within the reports directory.
func NewCompatibleTokensLookup(tokenPaths map[string]string) (*CompatibleTokensLookup, error) {
lookup := CompatibleTokensLookup{
byLoweredModuleResourceName: map[ModuleLowered]map[ResourceLowered]map[Token]struct{}{},
byNormalizedPath: map[NormalizedPath]map[Token]struct{}{},
}
if tokenPaths == nil {
return &lookup, nil
}
for token, path := range util.MapOrdered(tokenPaths) {
moduleName, _, resourceName, err := resources.ParseToken(token)
if err != nil {
return nil, fmt.Errorf("failed to parse token for CompatibleTokensLookup %s: %w", token, err)
}
lookup.add(openapi.ModuleName(moduleName), resourceName, path, token)
}
return &lookup, nil
}

func (lookup *CompatibleTokensLookup) add(moduleName openapi.ModuleName, resourceName openapi.ResourceName, path string, token string) {
normalizedPath := NormalizedPath(paths.NormalizePath(path))
moduleLowered := ModuleLowered(moduleName.Lowered())
resourceLowered := ResourceLowered(strings.ToLower(resourceName))
typedToken := Token(token)

if _, ok := lookup.byLoweredModuleResourceName[moduleLowered]; !ok {
lookup.byLoweredModuleResourceName[moduleLowered] = map[ResourceLowered]map[Token]struct{}{}
}
if _, ok := lookup.byLoweredModuleResourceName[moduleLowered][resourceLowered]; !ok {
lookup.byLoweredModuleResourceName[moduleLowered][resourceLowered] = map[Token]struct{}{}
}

lookup.byLoweredModuleResourceName[moduleLowered][resourceLowered][typedToken] = struct{}{}
if _, ok := lookup.byNormalizedPath[normalizedPath]; !ok {
lookup.byNormalizedPath[normalizedPath] = map[Token]struct{}{}
}
lookup.byNormalizedPath[normalizedPath][typedToken] = struct{}{}
}

func (lookup *CompatibleTokensLookup) IsPopulated() bool {
return lookup != nil && len(lookup.byLoweredModuleResourceName) > 0 && len(lookup.byNormalizedPath) > 0
}

// FindCompatibleTokens returns a list of all compatible resource tokens for a given module, resource, and path based on the names or path matching.
func (lookup *CompatibleTokensLookup) FindCompatibleTokens(moduleName openapi.ModuleName, resourceName openapi.ResourceName, path string) []string {
if lookup == nil || lookup.byLoweredModuleResourceName == nil || lookup.byNormalizedPath == nil {
return nil
}
matches := collections.NewOrderableSet[string]()
moduleLowered := ModuleLowered(moduleName.Lowered())
resourceLowered := ResourceLowered(strings.ToLower(resourceName))
normalizedPath := NormalizedPath(paths.NormalizePath(path))
if byLoweredResourceName, ok := lookup.byLoweredModuleResourceName[moduleLowered]; ok {
if tokens, ok := byLoweredResourceName[resourceLowered]; ok {
for token := range tokens {
matches.Add(string(token))
}
}
}
if resource, ok := lookup.byNormalizedPath[normalizedPath]; ok {
for token := range resource {
matches.Add(string(token))
}
}
return matches.SortedValues()
}
11 changes: 10 additions & 1 deletion provider/pkg/gen/gen_aliases_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,16 @@ func TestAliasesGen(t *testing.T) {
})

t.Run("v3", func(t *testing.T) {
generationResult, err := PulumiSchema(rootDir, modules, versioningStub{}, semver.MustParse("3.0.0"))
versioning := versioningStub{
// These are extracted from versions/v2-token-paths.json
previousTokenPaths: map[string]string{
"azure-native:aadiam/v20200301:PrivateEndpointConnection": "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/microsoft.aadiam/privateLinkForAzureAd/{policyName}/privateEndpointConnections/{privateEndpointConnectionName}",
"azure-native:aadiam:PrivateEndpointConnection": "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/microsoft.aadiam/privateLinkForAzureAd/{policyName}/privateEndpointConnections/{privateEndpointConnectionName}",
"azure-native:aadiam:PrivateLinkForAzureAd": "/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/microsoft.aadiam/privateLinkForAzureAd/{policyName}",
"azure-native:aadiam/v20200301preview:PrivateLinkForAzureAd": "/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/microsoft.aadiam/privateLinkForAzureAd/{policyName}",
},
}
generationResult, err := PulumiSchema(rootDir, modules, versioning, semver.MustParse("3.0.0"))
if err != nil {
t.Fatal(err)
}
Expand Down
11 changes: 10 additions & 1 deletion provider/pkg/gen/gen_dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,16 @@ func TestPortalDashboardGen(t *testing.T) {
},
},
}, openapi.DefaultVersions{}, nil, nil)
generationResult, err := PulumiSchema(rootDir, modules, versioningStub{}, semver.MustParse("3.0.0"))
versioning := versioningStub{
// These are extracted from versions/v2-token-paths.json
previousTokenPaths: map[string]string{
"azure-native:portal/v20190101preview:Dashboard": "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Portal/dashboards/{dashboardName}",
"azure-native:portal/v20200901preview:Dashboard": "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Portal/dashboards/{dashboardName}",
"azure-native:portal/v20221201preview:Dashboard": "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Portal/dashboards/{dashboardName}",
"azure-native:portal:Dashboard": "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Portal/dashboards/{dashboardName}",
},
}
generationResult, err := PulumiSchema(rootDir, modules, versioning, semver.MustParse("3.0.0"))
if err != nil {
t.Fatal(err)
}
Expand Down
61 changes: 43 additions & 18 deletions provider/pkg/gen/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ type Versioning interface {
ShouldInclude(moduleName openapi.ModuleName, version *openapi.ApiVersion, typeName, token string) bool
GetDeprecation(token string) (ResourceDeprecation, bool)
GetAllVersions(openapi.ModuleName, openapi.ResourceName) []openapi.ApiVersion
GetPreviousCompatibleTokensLookup() (*CompatibleTokensLookup, error)
}

type GenerationResult struct {
Expand All @@ -70,6 +71,7 @@ type GenerationResult struct {
// A map of module -> resource -> set of paths, to record resources that have conflicts where the same resource
// maps to more than one API path.
PathConflicts map[openapi.ModuleName]map[openapi.ResourceName]map[string][]openapi.ApiVersion
TokenPaths map[string]string
}

// PulumiSchema will generate a Pulumi schema for the given Azure modules and resources map.
Expand Down Expand Up @@ -283,6 +285,14 @@ func PulumiSchema(rootDir string, modules openapi.AzureModules, versioning Versi
flattenedPropertyConflicts := map[string]map[string]any{}
exampleMap := make(map[string][]resources.AzureAPIExample)
resourcesPathTracker := newResourcesPathConflictsTracker()
previousCompatibleTokensLookup, err := versioning.GetPreviousCompatibleTokensLookup()
if err != nil {
return nil, err
}

if !previousCompatibleTokensLookup.IsPopulated() && providerVersion.Major >= 3 {
return nil, fmt.Errorf("GetPreviousCompatibleTokensLookup is not populated. This is likely due to v%d-token-paths.json being missing or empty", providerVersion.Major-1)
}

for moduleName, moduleVersions := range util.MapOrdered(modules) {
resourcePaths := map[openapi.ResourceName]map[string][]openapi.ApiVersion{}
Expand All @@ -301,19 +311,20 @@ func PulumiSchema(rootDir string, modules openapi.AzureModules, versioning Versi
}

gen := packageGenerator{
pkg: &pkg,
metadata: &metadata,
moduleName: moduleName,
apiVersion: apiVersion,
sdkVersion: sdkVersion,
allVersions: allVersions,
examples: exampleMap,
versioning: versioning,
caseSensitiveTypes: caseSensitiveTypes,
rootDir: rootDir,
flattenedPropertyConflicts: flattenedPropertyConflicts,
majorVersion: int(providerVersion.Major),
resourcePaths: map[openapi.ResourceName]map[string]openapi.ApiVersion{},
pkg: &pkg,
metadata: &metadata,
moduleName: moduleName,
apiVersion: apiVersion,
sdkVersion: sdkVersion,
allVersions: allVersions,
examples: exampleMap,
versioning: versioning,
caseSensitiveTypes: caseSensitiveTypes,
rootDir: rootDir,
flattenedPropertyConflicts: flattenedPropertyConflicts,
majorVersion: int(providerVersion.Major),
resourcePaths: map[openapi.ResourceName]map[string]openapi.ApiVersion{},
previousCompatibleTokensLookup: previousCompatibleTokensLookup,
}

// Populate C#, Java, Python and Go module mapping.
Expand Down Expand Up @@ -353,7 +364,7 @@ func PulumiSchema(rootDir string, modules openapi.AzureModules, versioning Versi
return nil, fmt.Errorf("path conflicts detected. You probably need to add a case to schema.go/dedupResourceNameByPath.\n%+v", resourcesPathTracker.pathConflicts)
}

err := genMixins(&pkg, &metadata)
err = genMixins(&pkg, &metadata)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -405,6 +416,10 @@ version using infrastructure as code, which Pulumi then uses to drive the ARM AP
"packages": javaPackages,
})

tokenPaths := map[string]string{}
for token, resource := range metadata.Resources {
tokenPaths[token] = resource.Path
}
return &GenerationResult{
Schema: &pkg,
Metadata: &metadata,
Expand All @@ -413,6 +428,7 @@ version using infrastructure as code, which Pulumi then uses to drive the ARM AP
TypeCaseConflicts: caseSensitiveTypes.findCaseConflicts(),
FlattenedPropertyConflicts: flattenedPropertyConflicts,
PathConflicts: resourcesPathTracker.pathConflicts,
TokenPaths: tokenPaths,
}, nil
}

Expand Down Expand Up @@ -670,7 +686,8 @@ type packageGenerator struct {
majorVersion int
// A resource -> path -> API version map to record API paths per resource and later detect conflicts.
// Each packageGenerator instance is only used for a single API version, so there won't be conflicting paths here.
resourcePaths map[openapi.ResourceName]map[string]openapi.ApiVersion
resourcePaths map[openapi.ResourceName]map[string]openapi.ApiVersion
previousCompatibleTokensLookup *CompatibleTokensLookup
}

func (g *packageGenerator) genResources(typeName string, resource *openapi.ResourceSpec, nestedResourceBodyRefs []string) error {
Expand Down Expand Up @@ -929,7 +946,7 @@ func (g *packageGenerator) genResourceVariant(apiSpec *openapi.ResourceSpec, res
},
InputProperties: resourceRequest.specs,
RequiredInputs: resourceRequest.requiredSpecs.SortedValues(),
Aliases: g.generateAliases(resource, typeNameAliases...),
Aliases: g.generateAliases(resourceTok, resource, typeNameAliases...),
DeprecationMessage: resource.deprecationMessage,
}
g.pkg.Resources[resourceTok] = resourceSpec
Expand Down Expand Up @@ -1037,9 +1054,14 @@ func isSingleton(resource *resourceVariant) bool {
return resource.PathItem.Delete == nil || customresources.IsSingleton(resource.Path)
}

func (g *packageGenerator) generateAliases(resource *resourceVariant, typeNameAliases ...string) []pschema.AliasSpec {
func (g *packageGenerator) generateAliases(resourceTok string, resource *resourceVariant, typeNameAliases ...string) []pschema.AliasSpec {
typeAliases := collections.NewOrderableSet[string]()

previousCompatibleTokens := g.previousCompatibleTokensLookup.FindCompatibleTokens(resource.ModuleNaming.ResolvedName, resource.typeName, resource.Path)
for _, token := range previousCompatibleTokens {
typeAliases.Add(token)
}

// Add an alias for the same version of the resource, but in its old module.
if resource.ModuleNaming.PreviousName != nil {
typeAliases.Add(generateTok(*resource.ModuleNaming.PreviousName, resource.typeName, g.sdkVersion))
Expand Down Expand Up @@ -1075,7 +1097,10 @@ func (g *packageGenerator) generateAliases(resource *resourceVariant, typeNameAl

var aliasSpecs []pschema.AliasSpec
for _, v := range typeAliases.SortedValues() {
aliasSpecs = append(aliasSpecs, pschema.AliasSpec{Type: &v})
// Skip aliasing to itself.
if v != resourceTok {
aliasSpecs = append(aliasSpecs, pschema.AliasSpec{Type: &v})
}
}
return aliasSpecs
}
Expand Down
27 changes: 18 additions & 9 deletions provider/pkg/gen/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@ import (
var _ Versioning = (*versioningStub)(nil)

type versioningStub struct {
shouldInclude func(moduleName openapi.ModuleName, version *openapi.ApiVersion, typeName, token string) bool
getDeprecations func(token string) (ResourceDeprecation, bool)
getAllVersions func(moduleName openapi.ModuleName, resource string) []openapi.ApiVersion
shouldInclude func(moduleName openapi.ModuleName, version *openapi.ApiVersion, typeName, token string) bool
getDeprecations func(token string) (ResourceDeprecation, bool)
getAllVersions func(moduleName openapi.ModuleName, resource string) []openapi.ApiVersion
previousTokenPaths map[string]string
}

func (v versioningStub) ShouldInclude(moduleName openapi.ModuleName, version *openapi.ApiVersion, typeName, token string) bool {
Expand All @@ -43,15 +44,21 @@ func (v versioningStub) GetAllVersions(moduleName openapi.ModuleName, resource s
return []openapi.ApiVersion{}
}

func (v versioningStub) GetPreviousCompatibleTokensLookup() (*CompatibleTokensLookup, error) {
return NewCompatibleTokensLookup(v.previousTokenPaths)
}

func TestAliases(t *testing.T) {
// Wrap the generation of type aliases in a function to make it easier to test
generateTypeAliases := func(moduleName, typeName string, sdkVersion openapi.SdkVersion, previousModuleName string, typeNameAliases []string, versions []openapi.ApiVersion) []string {
previousCompatibleTokensLookup, _ := NewCompatibleTokensLookup(map[string]string{})
generator := packageGenerator{
pkg: &pschema.PackageSpec{Name: "azure-native"},
sdkVersion: sdkVersion,
versioning: versioningStub{},
moduleName: openapi.ModuleName(moduleName),
majorVersion: 2,
pkg: &pschema.PackageSpec{Name: "azure-native"},
sdkVersion: sdkVersion,
versioning: versioningStub{},
moduleName: openapi.ModuleName(moduleName),
majorVersion: 2,
previousCompatibleTokensLookup: previousCompatibleTokensLookup,
}

resource := &resourceVariant{
Expand All @@ -65,7 +72,9 @@ func TestAliases(t *testing.T) {
resource.ModuleNaming.PreviousName = &previousName
}

aliasSpecs := generator.generateAliases(resource, typeNameAliases...)
resourceTok := generateTok(openapi.ModuleName(moduleName), typeName, sdkVersion)

aliasSpecs := generator.generateAliases(resourceTok, resource, typeNameAliases...)
typeAliases := []string{}
for _, alias := range aliasSpecs {
typeAliases = append(typeAliases, *alias.Type)
Expand Down
3 changes: 3 additions & 0 deletions provider/pkg/versioning/build_schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ type BuildSchemaReports struct {
AllEndpoints map[openapi.ModuleName]map[openapi.ResourceName]map[string]*openapi.Endpoint
InactiveDefaultVersions map[openapi.ModuleName][]openapi.ApiVersion
OldApiVersions map[openapi.ModuleName][]openapi.ApiVersion
TokenPaths map[string]string
}

func (r BuildSchemaReports) WriteTo(outputDir string) ([]string, error) {
Expand All @@ -67,6 +68,7 @@ func (r BuildSchemaReports) WriteTo(outputDir string) ([]string, error) {
"skippedPOSTEndpoints.json": r.SkippedPOSTEndpoints,
"typeCaseConflicts.json": r.TypeCaseConflicts,
"oldApiVersions.json": r.OldApiVersions,
"tokenPaths.json": r.TokenPaths,
})
}

Expand Down Expand Up @@ -142,6 +144,7 @@ func BuildSchema(args BuildSchemaArgs) (*BuildSchemaResult, error) {
buildSchemaReports.TypeCaseConflicts = generationResult.TypeCaseConflicts
buildSchemaReports.FlattenedPropertyConflicts = generationResult.FlattenedPropertyConflicts
buildSchemaReports.PathConflicts = generationResult.PathConflicts
buildSchemaReports.TokenPaths = generationResult.TokenPaths

pkgSpec := generationResult.Schema
metadata := generationResult.Metadata
Expand Down
Loading

0 comments on commit b342515

Please sign in to comment.