Optimize algorithm

cmd/subcmds/optimizeSG.go

@@ -8,7 +8,7 @@ package subcmds
 import (
 	"github.com/spf13/cobra"
 
-	"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
+	sgoptimizer "github.com/np-guard/vpc-network-config-synthesis/pkg/optimize/sg"
 )
 
 const sgNameFlag = "sg-name"
@@ -20,7 +20,7 @@ func newOptimizeSGCommand(args *inArgs) *cobra.Command {
 		Long:  `OptimizeSG attempts to reduce the number of security group rules in a SG without changing the semantic.`,
 		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return optimization(cmd, args, optimize.NewSGOptimizer, true)
+			return optimization(cmd, args, sgoptimizer.NewSGOptimizer, true)
 		},
 	}
 

go.mod

@@ -5,7 +5,7 @@ go 1.23.0
 require (
 	github.com/IBM/vpc-go-sdk v0.63.1
 	github.com/np-guard/cloud-resource-collector v0.17.0
-	github.com/np-guard/models v0.5.2
+	github.com/np-guard/models v0.5.3
 	github.com/spf13/cobra v1.8.1
 )
 

go.sum

@@ -134,8 +134,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/np-guard/cloud-resource-collector v0.17.0 h1:QFXrFeF9ZS3P4B2dnZNvpWXLUq/QGPuwDGTjarByCvs=
 github.com/np-guard/cloud-resource-collector v0.17.0/go.mod h1:vp82iqdSq12EZJxA11oZkVseFVKg0hvPIUA9tVEdnMc=
-github.com/np-guard/models v0.5.2 h1:lty+shExffJpMQyu36a/NBYEky/rjEddQid4GOVHnhs=
-github.com/np-guard/models v0.5.2/go.mod h1:dqRdt5EQID1GmHuYsMOJzg4sS104om6NwEZ6sVO55z8=
+github.com/np-guard/models v0.5.3 h1:XtpLNTyhU1ptmFrYL3MCZestEvRa0uyiYV7YptIeE/k=
+github.com/np-guard/models v0.5.3/go.mod h1:dqRdt5EQID1GmHuYsMOJzg4sS104om6NwEZ6sVO55z8=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=

pkg/io/confio/parse_sgs.go

@@ -36,11 +36,12 @@ func ReadSGs(filename string) (*ir.SGCollection, error) {
 			log.Printf("Warning: missing SG/VPC name in sg at index %d\n", i)
 			continue
 		}
+		sgName := ir.SGName(*sg.Name)
 		vpcName := *sg.VPC.Name
 		if result.SGs[vpcName] == nil {
 			result.SGs[vpcName] = make(map[ir.SGName]*ir.SG)
 		}
-		result.SGs[vpcName][ir.SGName(*sg.Name)] = &ir.SG{InboundRules: inbound, OutboundRules: outbound}
+		result.SGs[vpcName][sgName] = &ir.SG{SGName: sgName, InboundRules: inbound, OutboundRules: outbound}
 	}
 	return result, nil
 }
@@ -99,7 +100,7 @@ func translateSGRuleProtocolIcmp(rule *vpcv1.SecurityGroupRuleSecurityGroupRuleP
 	direction, err1 := translateDirection(*rule.Direction)
 	remote, err2 := translateRemote(rule.Remote)
 	local, err3 := translateLocal(rule.Local)
-	protocol, err4 := netp.ICMPFromTypeAndCode64(rule.Type, rule.Code)
+	protocol, err4 := netp.ICMPFromTypeAndCode64WithoutRFCValidation(rule.Type, rule.Code)
 	if err := errors.Join(err1, err2, err3, err4); err != nil {
 		return nil, err
 	}

pkg/io/tfio/acl.go

@@ -65,7 +65,7 @@ func aclRule(rule *ir.ACLRule, name string) (tf.Block, error) {
 		{Name: "destination", Value: quote(rule.Destination.String())},
 	}
 
-	comment := ""
+	comment := "\n"
 	if rule.Explanation != "" {
 		comment = fmt.Sprintf("# %v", rule.Explanation)
 	}
@@ -109,7 +109,7 @@ func aclCollection(t *ir.ACLCollection, vpc string) (*tf.ConfigFile, error) {
 	var acls = make([]tf.Block, len(sortedACLs))
 	i := 0
 	for _, subnet := range sortedACLs {
-		comment := ""
+		comment := "\n"
 		vpcName := ir.VpcFromScopedResource(subnet)
 		acl := t.ACLs[vpcName][subnet]
 		if len(sortedACLs) > 1 { // not a single nacl

pkg/io/tfio/sg.go

@@ -93,8 +93,8 @@ func sgRule(rule *ir.SGRule, sgName ir.SGName, i int) (tf.Block, error) {
 func sg(sgName, vpcName string) (tf.Block, error) {
 	tfSGName := ir.ChangeScoping(sgName)
 	comment := fmt.Sprintf("\n### SG attached to %s", sgName)
-	if sgName == tfSGName { // its optimization
-		comment = ""
+	if sgName == tfSGName { // optimization mode
+		comment = "\n"
 	}
 	if err := verifyName(tfSGName); err != nil {
 		return tf.Block{}, err

pkg/ir/sg.go

@@ -68,6 +68,11 @@ func (r *SGRule) mustSupersede(other *SGRule) bool {
 	return res
 }
 
+func NewSGRule(direction Direction, remote RemoteType, p netp.Protocol, local *netset.IPBlock, e string) *SGRule {
+	return &SGRule{Direction: direction, Remote: remote, Protocol: p,
+		Local: local, Explanation: e}
+}
+
 func NewSG(sgName SGName) *SG {
 	return &SG{SGName: sgName, InboundRules: []*SGRule{}, OutboundRules: []*SGRule{}, Attached: []ID{}}
 }

pkg/optimize/common.go

@@ -5,9 +5,72 @@ SPDX-License-Identifier: Apache-2.0
 
 package optimize
 
-import "github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
+import (
+	"sort"
+
+	"github.com/np-guard/models/pkg/ds"
+	"github.com/np-guard/models/pkg/netp"
+	"github.com/np-guard/models/pkg/netset"
+
+	"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
+)
 
 type Optimizer interface {
 	// attempts to reduce number of SG/nACL rules
 	Optimize() (ir.Collection, error)
 }
+
+// each IPBlock is a single CIDR. The CIDRs are disjoint.
+func SortPartitionsByIPAddrs[T ds.Set[T]](p []ds.Pair[*netset.IPBlock, T]) []ds.Pair[*netset.IPBlock, T] {
+	cmp := func(i, j int) bool {
+		if p[i].Left.FirstIPAddress() == p[j].Left.FirstIPAddress() {
+			return p[i].Left.LastIPAddress() < p[j].Left.LastIPAddress()
+		}
+		return p[i].Left.FirstIPAddress() < p[j].Left.FirstIPAddress()
+	}
+	sort.Slice(p, cmp)
+	return p
+}
+
+func IcmpsetPartitions(icmpset *netset.ICMPSet) []netp.ICMP {
+	result := make([]netp.ICMP, 0)
+	if icmpset.IsAll() {
+		icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(nil, nil)
+		return []netp.ICMP{icmp}
+	}
+
+	for _, cube := range icmpset.Partitions() {
+		for _, typeInterval := range cube.Left.Intervals() {
+			for _, icmpType := range typeInterval.Elements() {
+				if cube.Right.Equal(netset.AllICMPCodes()) {
+					icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(&icmpType, nil)
+					result = append(result, icmp)
+					continue
+				}
+				for _, codeInterval := range cube.Right.Intervals() {
+					for _, icmpCode := range codeInterval.Elements() {
+						icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(&icmpType, &icmpCode)
+						result = append(result, icmp)
+					}
+				}
+			}
+		}
+	}
+	return result
+}
+
+func IcmpRuleToIcmpSet(icmp netp.ICMP) *netset.ICMPSet {
+	if icmp.TypeCode == nil {
+		return netset.AllICMPSet()
+	}
+	icmpType := int64(icmp.TypeCode.Type)
+	if icmp.TypeCode.Code == nil {
+		return netset.NewICMPSet(icmpType, icmpType, int64(netp.MinICMPCode), int64(netp.MaxICMPCode))
+	}
+	icmpCode := int64(*icmp.TypeCode.Code)
+	return netset.NewICMPSet(icmpType, icmpType, icmpCode, icmpCode)
+}
+
+func AllPorts(tcpudpPorts *netset.PortSet) bool {
+	return tcpudpPorts.Equal(netset.AllPorts())
+}

pkg/optimize/sg/ipCubesToRules.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2023- IBM Inc. All Rights Reserved.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package sgoptimizer
+
+import (
+	"github.com/np-guard/models/pkg/ds"
+	"github.com/np-guard/models/pkg/interval"
+	"github.com/np-guard/models/pkg/netp"
+	"github.com/np-guard/models/pkg/netset"
+
+	"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
+	"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
+)
+
+// any protocol cubes, represented by a single ipblock that will be decomposed
+// into cidrs. Each cidr will be a remote of a single SG rule
+func anyProtocolIPCubesToRules(cubes *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
+	result := make([]*ir.SGRule, 0)
+	for _, cidr := range cubes.SplitToCidrs() {
+		result = append(result, ir.NewSGRule(direction, cidr, netp.AnyProtocol{}, netset.GetCidrAll(), ""))
+	}
+	return result
+}
+
+// tcpudpIPCubesToRules converts cubes representing tcp or udp protocol rules to SG rules
+func tcpudpIPCubesToRules(cubes []ds.Pair[*netset.IPBlock, *netset.PortSet], anyProtocolCubes *netset.IPBlock, direction ir.Direction,
+	isTCP bool) []*ir.SGRule {
+	if len(cubes) == 0 {
+		return []*ir.SGRule{}
+	}
+
+	res := make([]*ir.SGRule, 0)
+	activeRules := make(map[*netset.IPBlock]netp.Protocol) // the key is the first IP
+
+	for i := range cubes {
+		// if it is not possible to continue the rule between the cubes, generate all existing rules
+		if i > 0 && uncoveredHole(cubes[i-1], cubes[i], anyProtocolCubes) {
+			res = append(res, createActiveRules(activeRules, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+			activeRules = make(map[*netset.IPBlock]netp.Protocol)
+		}
+
+		// if there are active rules whose ports are not fully included in the current cube, they will be created
+		// also activePorts will be calculated, which is the ports that are still included in the active rules
+		activePorts := interval.NewCanonicalSet()
+		for startIP, protocol := range activeRules {
+			if tcpudp, ok := protocol.(netp.TCPUDP); ok {
+				if !tcpudp.DstPorts().ToSet().IsSubset(cubes[i].Right) {
+					res = append(res, createNewRules(protocol, startIP, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+					delete(activeRules, startIP)
+				} else {
+					activePorts.AddInterval(tcpudp.DstPorts())
+				}
+			}
+		}
+
+		// if the current cube contains ports that are not contained in active rules, new rules will be created
+		for _, ports := range cubes[i].Right.Intervals() {
+			if !ports.ToSet().IsSubset(activePorts) {
+				p, _ := netp.NewTCPUDP(isTCP, netp.MinPort, netp.MaxPort, int(ports.Start()), int(ports.End()))
+				activeRules[cubes[i].Left.FirstIPAddressObject()] = p
+			}
+		}
+	}
+	// generate all existing rules
+	return append(res, createActiveRules(activeRules, cubes[len(cubes)-1].Left.LastIPAddressObject(), direction)...)
+}
+
+// icmpIPCubesToRules converts cubes representing icmp protocol rules to SG rules
+func icmpIPCubesToRules(cubes []ds.Pair[*netset.IPBlock, *netset.ICMPSet], anyProtocolCubes *netset.IPBlock,
+	direction ir.Direction) []*ir.SGRule {
+	if len(cubes) == 0 {
+		return []*ir.SGRule{}
+	}
+
+	res := make([]*ir.SGRule, 0)
+	activeRules := make(map[*netset.IPBlock]netp.Protocol) // the key is the first IP
+
+	for i := range cubes {
+		// if it is not possible to continue the rule between the cubes, generate all existing rules
+		if i > 0 && uncoveredHole(cubes[i-1], cubes[i], anyProtocolCubes) {
+			res = append(res, createActiveRules(activeRules, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+			activeRules = make(map[*netset.IPBlock]netp.Protocol)
+		}
+
+		// if there are active rules whose icmp values are not fully included in the current cube, they will be created
+		// also activeICMP will be calculated, which is the icmp values that are still included in the active rules
+		activeICMP := netset.EmptyICMPSet()
+		for startIP, protocol := range activeRules {
+			if icmp, ok := protocol.(netp.ICMP); ok {
+				ruleIcmpSet := optimize.IcmpRuleToIcmpSet(icmp)
+				if !ruleIcmpSet.IsSubset(cubes[i].Right) {
+					res = append(res, createNewRules(protocol, startIP, cubes[i-1].Left.LastIPAddressObject(), direction)...)
+					delete(activeRules, startIP)
+				} else {
+					activeICMP.Union(ruleIcmpSet)
+				}
+			}
+		}
+
+		// if the cube contains icmp values that are not contained in  active rules, new rules will be created
+		for _, p := range optimize.IcmpsetPartitions(cubes[i].Right) {
+			if !optimize.IcmpRuleToIcmpSet(p).IsSubset(activeICMP) {
+				activeRules[cubes[i].Left.FirstIPAddressObject()] = p
+			}
+		}
+	}
+
+	// generate all  existing rules
+	return append(res, createActiveRules(activeRules, cubes[len(cubes)-1].Left.LastIPAddressObject(), direction)...)
+}
+
+// uncoveredHole returns true if the rules can not be continued between the two cubes
+// i.e there is a hole between two ipblocks that is not a subset of anyProtocol cubes
+func uncoveredHole[T ds.Set[T]](prevPair, currPair ds.Pair[*netset.IPBlock, T], anyProtocolCubes *netset.IPBlock) bool {
+	prevIPBlock := prevPair.Left
+	currIPBlock := currPair.Left
+	touching, _ := prevIPBlock.TouchingIPRanges(currIPBlock)
+	if touching {
+		return false
+	}
+	holeFirstIP, _ := prevIPBlock.NextIP()
+	holeEndIP, _ := currIPBlock.PreviousIP()
+	hole, _ := netset.IPBlockFromIPRange(holeFirstIP, holeEndIP)
+	return !hole.IsSubset(anyProtocolCubes)
+}
+
+// creates sgRules from SG active rules
+func createActiveRules(activeRules map[*netset.IPBlock]netp.Protocol, lastIP *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
+	res := make([]*ir.SGRule, 0)
+	for firstIP, protocol := range activeRules {
+		res = append(res, createNewRules(protocol, firstIP, lastIP, direction)...)
+	}
+	return res
+}
+
+// createNewRules breaks the startIP-endIP ip range into cidrs and creates SG rules
+func createNewRules(protocol netp.Protocol, startIP, endIP *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
+	res := make([]*ir.SGRule, 0)
+	ipRange, _ := netset.IPBlockFromIPRange(startIP, endIP)
+	for _, cidr := range ipRange.SplitToCidrs() {
+		res = append(res, ir.NewSGRule(direction, cidr, protocol, netset.GetCidrAll(), ""))
+	}
+	return res
+}