Skip to content

Commit

Permalink
Optimize algorithm (#195)
Browse files Browse the repository at this point in the history
  • Loading branch information
YairSlobodin1 authored Dec 18, 2024
1 parent bc8133f commit b4b86f1
Show file tree
Hide file tree
Showing 34 changed files with 9,984 additions and 55 deletions.
4 changes: 2 additions & 2 deletions cmd/subcmds/optimizeSG.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ package subcmds
import (
"github.com/spf13/cobra"

"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
sgoptimizer "github.com/np-guard/vpc-network-config-synthesis/pkg/optimize/sg"
)

const sgNameFlag = "sg-name"
Expand All @@ -20,7 +20,7 @@ func newOptimizeSGCommand(args *inArgs) *cobra.Command {
Long: `OptimizeSG attempts to reduce the number of security group rules in a SG without changing the semantic.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return optimization(cmd, args, optimize.NewSGOptimizer, true)
return optimization(cmd, args, sgoptimizer.NewSGOptimizer, true)
},
}

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.23.0
require (
github.com/IBM/vpc-go-sdk v0.63.1
github.com/np-guard/cloud-resource-collector v0.17.0
github.com/np-guard/models v0.5.2
github.com/np-guard/models v0.5.3
github.com/spf13/cobra v1.8.1
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/np-guard/cloud-resource-collector v0.17.0 h1:QFXrFeF9ZS3P4B2dnZNvpWXLUq/QGPuwDGTjarByCvs=
github.com/np-guard/cloud-resource-collector v0.17.0/go.mod h1:vp82iqdSq12EZJxA11oZkVseFVKg0hvPIUA9tVEdnMc=
github.com/np-guard/models v0.5.2 h1:lty+shExffJpMQyu36a/NBYEky/rjEddQid4GOVHnhs=
github.com/np-guard/models v0.5.2/go.mod h1:dqRdt5EQID1GmHuYsMOJzg4sS104om6NwEZ6sVO55z8=
github.com/np-guard/models v0.5.3 h1:XtpLNTyhU1ptmFrYL3MCZestEvRa0uyiYV7YptIeE/k=
github.com/np-guard/models v0.5.3/go.mod h1:dqRdt5EQID1GmHuYsMOJzg4sS104om6NwEZ6sVO55z8=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
Expand Down
5 changes: 3 additions & 2 deletions pkg/io/confio/parse_sgs.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,12 @@ func ReadSGs(filename string) (*ir.SGCollection, error) {
log.Printf("Warning: missing SG/VPC name in sg at index %d\n", i)
continue
}
sgName := ir.SGName(*sg.Name)
vpcName := *sg.VPC.Name
if result.SGs[vpcName] == nil {
result.SGs[vpcName] = make(map[ir.SGName]*ir.SG)
}
result.SGs[vpcName][ir.SGName(*sg.Name)] = &ir.SG{InboundRules: inbound, OutboundRules: outbound}
result.SGs[vpcName][sgName] = &ir.SG{SGName: sgName, InboundRules: inbound, OutboundRules: outbound}
}
return result, nil
}
Expand Down Expand Up @@ -99,7 +100,7 @@ func translateSGRuleProtocolIcmp(rule *vpcv1.SecurityGroupRuleSecurityGroupRuleP
direction, err1 := translateDirection(*rule.Direction)
remote, err2 := translateRemote(rule.Remote)
local, err3 := translateLocal(rule.Local)
protocol, err4 := netp.ICMPFromTypeAndCode64(rule.Type, rule.Code)
protocol, err4 := netp.ICMPFromTypeAndCode64WithoutRFCValidation(rule.Type, rule.Code)
if err := errors.Join(err1, err2, err3, err4); err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/io/tfio/acl.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func aclRule(rule *ir.ACLRule, name string) (tf.Block, error) {
{Name: "destination", Value: quote(rule.Destination.String())},
}

comment := ""
comment := "\n"
if rule.Explanation != "" {
comment = fmt.Sprintf("# %v", rule.Explanation)
}
Expand Down Expand Up @@ -109,7 +109,7 @@ func aclCollection(t *ir.ACLCollection, vpc string) (*tf.ConfigFile, error) {
var acls = make([]tf.Block, len(sortedACLs))
i := 0
for _, subnet := range sortedACLs {
comment := ""
comment := "\n"
vpcName := ir.VpcFromScopedResource(subnet)
acl := t.ACLs[vpcName][subnet]
if len(sortedACLs) > 1 { // not a single nacl
Expand Down
4 changes: 2 additions & 2 deletions pkg/io/tfio/sg.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ func sgRule(rule *ir.SGRule, sgName ir.SGName, i int) (tf.Block, error) {
func sg(sgName, vpcName string) (tf.Block, error) {
tfSGName := ir.ChangeScoping(sgName)
comment := fmt.Sprintf("\n### SG attached to %s", sgName)
if sgName == tfSGName { // its optimization
comment = ""
if sgName == tfSGName { // optimization mode
comment = "\n"
}
if err := verifyName(tfSGName); err != nil {
return tf.Block{}, err
Expand Down
5 changes: 5 additions & 0 deletions pkg/ir/sg.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@ func (r *SGRule) mustSupersede(other *SGRule) bool {
return res
}

func NewSGRule(direction Direction, remote RemoteType, p netp.Protocol, local *netset.IPBlock, e string) *SGRule {
return &SGRule{Direction: direction, Remote: remote, Protocol: p,
Local: local, Explanation: e}
}

func NewSG(sgName SGName) *SG {
return &SG{SGName: sgName, InboundRules: []*SGRule{}, OutboundRules: []*SGRule{}, Attached: []ID{}}
}
Expand Down
63 changes: 62 additions & 1 deletion pkg/optimize/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,70 @@ SPDX-License-Identifier: Apache-2.0

package optimize

import "github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
import (
"slices"

"github.com/np-guard/models/pkg/ds"
"github.com/np-guard/models/pkg/netp"
"github.com/np-guard/models/pkg/netset"

"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
)

type Optimizer interface {
// attempts to reduce number of SG/nACL rules
Optimize() (ir.Collection, error)
}

// each IPBlock is a single CIDR. The CIDRs are disjoint.
func SortPartitionsByIPAddrs[T ds.Set[T]](p []ds.Pair[*netset.IPBlock, T]) []ds.Pair[*netset.IPBlock, T] {
cmp := func(i, j ds.Pair[*netset.IPBlock, T]) int {
return i.Left.Compare(j.Left)
}
slices.SortFunc(p, cmp)
return p
}

// IcmpsetPartitions breaks the set into ICMP slice, where each element defined as legal in nACL, SG rules
func IcmpsetPartitions(icmpset *netset.ICMPSet) []netp.ICMP {
if icmpset.IsAll() {
icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(nil, nil)
return []netp.ICMP{icmp}
}

result := make([]netp.ICMP, 0)
for _, cube := range icmpset.Partitions() {
for _, typeInterval := range cube.Left.Intervals() {
for _, icmpType := range typeInterval.Elements() {
if cube.Right.Equal(netset.AllICMPCodes()) {
icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(&icmpType, nil)
result = append(result, icmp)
continue
}
for _, codeInterval := range cube.Right.Intervals() {
for _, icmpCode := range codeInterval.Elements() {
icmp, _ := netp.ICMPFromTypeAndCode64WithoutRFCValidation(&icmpType, &icmpCode)
result = append(result, icmp)
}
}
}
}
}
return result
}

func IcmpRuleToIcmpSet(icmp netp.ICMP) *netset.ICMPSet {
if icmp.TypeCode == nil {
return netset.AllICMPSet()
}
icmpType := int64(icmp.TypeCode.Type)
if icmp.TypeCode.Code == nil {
return netset.NewICMPSet(icmpType, icmpType, int64(netp.MinICMPCode), int64(netp.MaxICMPCode))
}
icmpCode := int64(*icmp.TypeCode.Code)
return netset.NewICMPSet(icmpType, icmpType, icmpCode, icmpCode)
}

func IsAllPorts(tcpudpPorts *netset.PortSet) bool {
return tcpudpPorts.Equal(netset.AllPorts())
}
28 changes: 0 additions & 28 deletions pkg/optimize/sg.go

This file was deleted.

147 changes: 147 additions & 0 deletions pkg/optimize/sg/ipCubesToRules.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
/*
Copyright 2023- IBM Inc. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package sgoptimizer

import (
"slices"

"github.com/np-guard/models/pkg/ds"
"github.com/np-guard/models/pkg/interval"
"github.com/np-guard/models/pkg/netp"
"github.com/np-guard/models/pkg/netset"

"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
)

// any protocol IP-segments, represented by a single ipblock that will be decomposed
// into cidrs. Each cidr will be a remote of a single SG rule
func anyProtocolIPCubesToRules(cubes *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
result := make([]*ir.SGRule, 0)
for _, cidr := range cubes.SplitToCidrs() {
result = append(result, ir.NewSGRule(direction, cidr, netp.AnyProtocol{}, netset.GetCidrAll(), ""))
}
return result
}

// tcpudpIPCubesToRules converts cubes representing tcp or udp protocol rules to SG rules
func tcpudpIPCubesToRules(cubes []ds.Pair[*netset.IPBlock, *netset.PortSet], anyProtocolCubes *netset.IPBlock, direction ir.Direction,
isTCP bool) []*ir.SGRule {
if len(cubes) == 0 {
return []*ir.SGRule{}
}

res := make([]*ir.SGRule, 0)
activeRules := make(map[*netset.IPBlock]netp.Protocol) // the key is the first IP

for i := range cubes {
// if it is not possible to continue the rule between the cubes, generate all existing rules
if i > 0 && uncoveredHole(cubes[i-1], cubes[i], anyProtocolCubes) {
res = slices.Concat(res, createActiveRules(activeRules, cubes[i-1].Left.LastIPAddressObject(), direction))
activeRules = make(map[*netset.IPBlock]netp.Protocol)
}

// if there are active rules whose ports are not fully included in the current cube, they will be created
// also activePorts will be calculated, which is the ports that are still included in the active rules
activePorts := interval.NewCanonicalSet()
for startIP, protocol := range activeRules {
tcpudp, _ := protocol.(netp.TCPUDP) // already checked
if !tcpudp.DstPorts().ToSet().IsSubset(cubes[i].Right) {
res = slices.Concat(res, createNewRules(protocol, startIP, cubes[i-1].Left.LastIPAddressObject(), direction))
delete(activeRules, startIP)
} else {
activePorts.AddInterval(tcpudp.DstPorts())
}
}

// if the current cube contains ports that are not contained in active rules, new rules will be created
for _, ports := range cubes[i].Right.Intervals() {
if !ports.ToSet().IsSubset(activePorts) {
p, _ := netp.NewTCPUDP(isTCP, netp.MinPort, netp.MaxPort, int(ports.Start()), int(ports.End()))
activeRules[cubes[i].Left.FirstIPAddressObject()] = p
}
}
}
// generate all existing rules
return slices.Concat(res, createActiveRules(activeRules, cubes[len(cubes)-1].Left.LastIPAddressObject(), direction))
}

// icmpIPCubesToRules converts cubes representing icmp protocol rules to SG rules
func icmpIPCubesToRules(cubes []ds.Pair[*netset.IPBlock, *netset.ICMPSet], anyProtocolCubes *netset.IPBlock,
direction ir.Direction) []*ir.SGRule {
if len(cubes) == 0 {
return []*ir.SGRule{}
}

res := make([]*ir.SGRule, 0)
activeRules := make(map[*netset.IPBlock]netp.Protocol) // the key is the first IP

for i := range cubes {
// if it is not possible to continue the rule between the cubes, generate all existing rules
if i > 0 && uncoveredHole(cubes[i-1], cubes[i], anyProtocolCubes) {
res = slices.Concat(res, createActiveRules(activeRules, cubes[i-1].Left.LastIPAddressObject(), direction))
activeRules = make(map[*netset.IPBlock]netp.Protocol)
}

// if there are active rules whose icmp values are not fully included in the current cube, they will be created
// also activeICMP will be calculated, which is the icmp values that are still included in the active rules
activeICMP := netset.EmptyICMPSet()
for startIP, protocol := range activeRules {
icmp, _ := protocol.(netp.ICMP)
ruleIcmpSet := optimize.IcmpRuleToIcmpSet(icmp)
if !ruleIcmpSet.IsSubset(cubes[i].Right) {
res = slices.Concat(res, createNewRules(protocol, startIP, cubes[i-1].Left.LastIPAddressObject(), direction))
delete(activeRules, startIP)
} else {
activeICMP.Union(ruleIcmpSet)
}
}

// if the cube contains icmp values that are not contained in active rules, new rules will be created
for _, p := range optimize.IcmpsetPartitions(cubes[i].Right) {
if !optimize.IcmpRuleToIcmpSet(p).IsSubset(activeICMP) {
activeRules[cubes[i].Left.FirstIPAddressObject()] = p
}
}
}

// generate all existing rules
return slices.Concat(res, createActiveRules(activeRules, cubes[len(cubes)-1].Left.LastIPAddressObject(), direction))
}

// uncoveredHole returns true if the rules can not be continued between the two cubes
// i.e there is a hole between two ipblocks that is not a subset of anyProtocol cubes
func uncoveredHole[T ds.Set[T]](prevPair, currPair ds.Pair[*netset.IPBlock, T], anyProtocolCubes *netset.IPBlock) bool {
prevIPBlock := prevPair.Left
currIPBlock := currPair.Left
touching, _ := prevIPBlock.TouchingIPRanges(currIPBlock)
if touching {
return false
}
holeFirstIP, _ := prevIPBlock.NextIP()
holeEndIP, _ := currIPBlock.PreviousIP()
hole, _ := netset.IPBlockFromIPRange(holeFirstIP, holeEndIP)
return !hole.IsSubset(anyProtocolCubes)
}

// creates sgRules from SG active rules
func createActiveRules(activeRules map[*netset.IPBlock]netp.Protocol, lastIP *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
res := make([]*ir.SGRule, 0)
for firstIP, protocol := range activeRules {
res = slices.Concat(res, createNewRules(protocol, firstIP, lastIP, direction))
}
return res
}

// createNewRules breaks the startIP-endIP ip range into cidrs and creates SG rules
func createNewRules(protocol netp.Protocol, startIP, endIP *netset.IPBlock, direction ir.Direction) []*ir.SGRule {
res := make([]*ir.SGRule, 0)
ipRange, _ := netset.IPBlockFromIPRange(startIP, endIP)
for _, cidr := range ipRange.SplitToCidrs() {
res = append(res, ir.NewSGRule(direction, cidr, protocol, netset.GetCidrAll(), ""))
}
return res
}
Loading

0 comments on commit b4b86f1

Please sign in to comment.