From cf9e479d029d03e33f8a8eabb83eb9cb77689ffe Mon Sep 17 00:00:00 2001 From: shirim Date: Thu, 20 Feb 2025 16:56:14 +0200 Subject: [PATCH 01/24] synthesis README --- README.md | 2 +- README_Synthesis.md | 53 +++++++++++++++++++++++++++++++++++++++++++++ cmd/commands.go | 2 +- 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 README_Synthesis.md diff --git a/README.md b/README.md index 3c87e573..0904eb13 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ spec: - Egress ``` - +More details [here](README_Synthesis.md) ## NSX Supported API versions and resources diff --git a/README_Synthesis.md b/README_Synthesis.md new file mode 100644 index 00000000..f0de0064 --- /dev/null +++ b/README_Synthesis.md @@ -0,0 +1,53 @@ +# Synthesize k8s policy + +Synthesize a given NSX DFW configuration into an equivalent k8s network policy. + +``` +Flags: + -- synthesis-dump-dir flag to run synthesis; specify directory path to store k8s synthesis results + -- synth flag to run synthesis, even if synthesis-dump-dir is not specified + -- synth-create-dns-policy flag to create a policy allowing access to target env dns pod + -- synthesize-admin-policies flag to synthesize category environment into admin network policies (which included deny, pass and priority) (default false) + -- disjoint-hint comma separated list of NSX groups/tags that are always disjoint in their VM members, + needed for an effective and sound synthesis process, can specify more than one hint + (example: \"--" + disjointHintsFlag + " frontend,backend --" + disjointHintsFlag + " app,web,db\") +``` + +## Overview +Synthesize a given NSX DFW configuration, specifically firewall rules, into an equivalent k8s network policy. +There are two main challenges here: +* *The flattening challenge*: translating allow/deny/pass with priorities into flat allow rules (which is what k8s network policies support) +* *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. +This is important since once new e.g. VMs are added with the relevant tags/labels they should be granted the desired communication permissions and the +desired protection. + +### The flattening challenge +The translation of priortized allow/deny/pass rules into flat allow rules is exponential in the number of terms of the +original rules (to be accurate, the number of allow rules generated for each original allow rule is +exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: +1. Ask the user to provide the tool with lists - _hints_ - of disjoint tags/groups. In the future it is planned to "guess" these +disjoint sets, and ask the user to approve them. +2. Apply various optimization to simplify the resulting rules and to rid redundant rules; the more accurate hints the +tool is provided, the more efficient the hints are + +### The policy preserving challenge +To preserve the original intent of the policy, the synthesized policy refers, where possible, to permanent labeling such +as tags - e.g. _front-end_ - and not to temporarily labeling such as _VM_ names. E.g., a specific rule's _src_ is + defined to be group _aaa_ that is defined as _tag = backend_ and _tag != DB_ then the synthesized policy will refer to the value +of the tag. + +## Currently supported +Currently the tool supports groups defined by expressions over tags; nested expression are not supported at the moment. +If a group is defined +by an expression that we do not yet support, then the synthesized policy will refer just to the group, and the +relevant *VM*s will be granted labels of this group. In the following releases we will expend our expressions support. + +## Debuging +The tool first translates the priortized allow/deny/pass rules into an abstract model that +contains the rules to be syntactically translated to k8s policies; if _synthesize-admin-policies_ is off then all rules must +be flat allow rules, and so all the rules in the abstract model are allow rules; +otherwise rules originating from _environment_ category are translated to admin policies and as such may also contain +priorties, allow, deny and pass; other rules are flat allow rules. Each rule is defined over +_src_, _dst_ and a _connection_. The rules are "Or"ed. +The _src_ and the _dst_ are _Conjunction_, and the _connection_ contains a protocol and potentially _src/dst_ ports. +The synthesis dump directory contains the abstract model, in addition to other debug data, such as the connectivity map. \ No newline at end of file diff --git a/cmd/commands.go b/cmd/commands.go index 70f11440..e32c9d99 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -50,7 +50,7 @@ const ( outputFileHelp = "file path to store analysis results" explainHelp = "flag to explain connectivity output with rules explanations per allowed/denied connections (default false)" synthesisDumpDirHelp = "run synthesis; specify directory path to store k8s synthesis results" - synthesizeAdminPoliciesHelp = "include admin network policies in policy synthesis (default false)" + synthesizeAdminPoliciesHelp = "synthesize category environment into admin network policies (which included deny, pass and priority) (default false)" outputFormatHelp = "output format; must be one of " outputFilterFlagHelp = "filter the analysis results by vm names, can specify more than one (example: \"vm1,vm2\")" quietHelp = "flag to run quietly, report only severe errors and result (default false)" From 264ad254116d6450308df6b989a9d1adf03da5c3 Mon Sep 17 00:00:00 2001 From: shirim Date: Sun, 23 Feb 2025 09:03:35 +0200 Subject: [PATCH 02/24] lint --- cmd/commands.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cmd/commands.go b/cmd/commands.go index e32c9d99..05027a06 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -50,15 +50,16 @@ const ( outputFileHelp = "file path to store analysis results" explainHelp = "flag to explain connectivity output with rules explanations per allowed/denied connections (default false)" synthesisDumpDirHelp = "run synthesis; specify directory path to store k8s synthesis results" - synthesizeAdminPoliciesHelp = "synthesize category environment into admin network policies (which included deny, pass and priority) (default false)" - outputFormatHelp = "output format; must be one of " - outputFilterFlagHelp = "filter the analysis results by vm names, can specify more than one (example: \"vm1,vm2\")" - quietHelp = "flag to run quietly, report only severe errors and result (default false)" - verboseHelp = "flag to run with more informative messages printed to log (default false)" - colorHelp = "flag to enable color output (default false)" - createDNSPolicyHelp = "flag to create a policy allowing access to target env dns pod" - synthHelp = "flag to run synthesis, even if synthesis-dump-dir is not specified" - disjointHintsHelp = "comma separated list of NSX groups/tags that are always disjoint in their VM members," + + synthesizeAdminPoliciesHelp = "synthesize category environment into admin network policies" + + " (which included deny, pass and priority) (default false)" + outputFormatHelp = "output format; must be one of " + outputFilterFlagHelp = "filter the analysis results by vm names, can specify more than one (example: \"vm1,vm2\")" + quietHelp = "flag to run quietly, report only severe errors and result (default false)" + verboseHelp = "flag to run with more informative messages printed to log (default false)" + colorHelp = "flag to enable color output (default false)" + createDNSPolicyHelp = "flag to create a policy allowing access to target env dns pod" + synthHelp = "flag to run synthesis, even if synthesis-dump-dir is not specified" + disjointHintsHelp = "comma separated list of NSX groups/tags that are always disjoint in their VM members," + " needed for an effective and sound synthesis process, can specify more than one hint" + " (example: \"--" + disjointHintsFlag + " frontend,backend --" + disjointHintsFlag + " app,web,db\")" ) From 61c327aea13205ba56132305b47100faf59d2c56 Mon Sep 17 00:00:00 2001 From: shirim Date: Sun, 23 Feb 2025 11:21:15 +0200 Subject: [PATCH 03/24] readme writing (cont) --- README_Synthesis.md | 61 +++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index f0de0064..63148541 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -14,21 +14,22 @@ Flags: ``` ## Overview -Synthesize a given NSX DFW configuration, specifically firewall rules, into an equivalent k8s network policy. +Synthesize a given NSX DFW policy into an equivalent k8s network policy. There are two main challenges here: -* *The flattening challenge*: translating allow/deny/pass with priorities into flat allow rules (which is what k8s network policies support) +* *The flattening challenge*: translating prioritize _allow/deny/pass_ into flat allow rules (which is what k8s network policies support) * *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. -This is important since once new e.g. VMs are added with the relevant tags/labels they should be granted the desired communication permissions and the -desired protection. +This is important since e.g. once a new VM as added with the relevant tags/labels it be granted the desired connectivity. ### The flattening challenge The translation of priortized allow/deny/pass rules into flat allow rules is exponential in the number of terms of the original rules (to be accurate, the number of allow rules generated for each original allow rule is exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: -1. Ask the user to provide the tool with lists - _hints_ - of disjoint tags/groups. In the future it is planned to "guess" these +1. Ask the user to provide the tool with _hints_ - lists of disjoint tags/groups. +E.g., tags _{frontend, backend}_ are disjoint. +In the future it is planned to "guess" these disjoint sets, and ask the user to approve them. 2. Apply various optimization to simplify the resulting rules and to rid redundant rules; the more accurate hints the -tool is provided, the more efficient the hints are +tool is provided, the more concise and readable rules it will synthesize. ### The policy preserving challenge To preserve the original intent of the policy, the synthesized policy refers, where possible, to permanent labeling such @@ -37,17 +38,45 @@ as tags - e.g. _front-end_ - and not to temporarily labeling such as _VM_ names. of the tag. ## Currently supported -Currently the tool supports groups defined by expressions over tags; nested expression are not supported at the moment. +Currently, the tool supports groups defined by expressions over tags; nested expression are not yet supported. If a group is defined by an expression that we do not yet support, then the synthesized policy will refer just to the group, and the relevant *VM*s will be granted labels of this group. In the following releases we will expend our expressions support. -## Debuging -The tool first translates the priortized allow/deny/pass rules into an abstract model that -contains the rules to be syntactically translated to k8s policies; if _synthesize-admin-policies_ is off then all rules must -be flat allow rules, and so all the rules in the abstract model are allow rules; -otherwise rules originating from _environment_ category are translated to admin policies and as such may also contain -priorties, allow, deny and pass; other rules are flat allow rules. Each rule is defined over -_src_, _dst_ and a _connection_. The rules are "Or"ed. -The _src_ and the _dst_ are _Conjunction_, and the _connection_ contains a protocol and potentially _src/dst_ ports. -The synthesis dump directory contains the abstract model, in addition to other debug data, such as the connectivity map. \ No newline at end of file +## Output +_k8s_resources_ dir under the dir specified in _synthesis-dump-dir_ contains the following files: +* **pods.yaml** the list of synthesized _VM's_ _pods_ with the labels of each pos +* **policies.yaml** the k8s policies + +The combination of the policies and the pods' labels: +1. Satisfies the snapshot of the connectivity at the time of the synthesis +2. Preserve the policy's intent, as expressed e.g. by *tags*, for future changes +(e.g. adding a _VM_ or changing its functionality and thus its labeling) + +## Debugging +The synthesize process is a complex one. Along it, in order to have the intent preserving synthesis as explained above, +we use a *symbolic* representation of the *rules*: each *symbolic rule* is a _priority_, an _action_, a +_src_, a _dst_ and a _connection_; The priority in a natural number; the action is _allow/deny/pass_; The _src_ and the _dst_ are _Conjunctions_ of simple expressions +(equal/not equal) over e.g. _tags_; the _connection_ is a protocol and potentially _src/dst_ min and max ports. + +The synthesis dump directory (specified in _synthesis-dump-dir_) contains (among others) the following files: +* under subdirectory **debug_dir** + * **config.txt** Contains the NSX config as being read by the tool; this includes _VMs_, _groups_ and _firewall rules_. + * **pre_processing.txt** Contains the translation of the firewall rule into _symbolic rules_ ; e.g., if a specific +src is a group which is an expression over tags, then this file will have this rule's _src_ defined over tags. + * **abstract_model.txt** The tool translates the *allow/deny/pass* rules from _pre_processing.txt_ into an abstract model that + contains the rules to be syntactically translated to _k8s policies_. If _synthesize-admin-policies_ is off then all rules must + be _flat allow rules_, and so all the rules in the abstract model are non-prioritized with action _allow_; + otherwise rules originating from the _environment_ category are translated to admin policies with + a priority and an allow/deny/pass action; rules that originate from the other categories are flat allow rules. + _abstract_model.txt_ contains these rules and a list of the groups, each groups with the expression that defined + it and the snapshot of the *VMs* in the group. + + The following log files contain warning messages and various debug printing of the different stages + of the synthesis, as following: + + * **runPreprocessing.log** Log of the stage in which the NSX rules are translated to symbolic rules. + * **runConvertToAbstract.log** Log of the stage in which the symbolic rules from the preprocessing stage +are translated to the abstract model's rules. + * **runK8SSynthesis.log** Log of the stage in which the k8s yaml pods and polices files are synthesized from +the abstract model. From eee48227a1d0d4dbd72b59a2e0bc9758c5e16ded Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Tue, 25 Feb 2025 16:56:16 +0200 Subject: [PATCH 04/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 63148541..bcf2866f 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -1,4 +1,4 @@ -# Synthesize k8s policy +# Synthesize k8s network policy resources from NSX DFW config Synthesize a given NSX DFW configuration into an equivalent k8s network policy. From 4182a89f89ba514258dd6766b2b457803eb529d8 Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:01:59 +0200 Subject: [PATCH 05/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index bcf2866f..1e0fae9a 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -16,7 +16,7 @@ Flags: ## Overview Synthesize a given NSX DFW policy into an equivalent k8s network policy. There are two main challenges here: -* *The flattening challenge*: translating prioritize _allow/deny/pass_ into flat allow rules (which is what k8s network policies support) +* *The flattening challenge*: translating prioritized set of rules with actions `allow/deny/jump-to-app` into a flat set of `allow` rules (which is what k8s network policies support). * *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. This is important since e.g. once a new VM as added with the relevant tags/labels it be granted the desired connectivity. From 11d0975f0e013604a0d397dc520329328cd22a59 Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:05:50 +0200 Subject: [PATCH 06/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 1e0fae9a..60d8610a 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -18,7 +18,7 @@ Synthesize a given NSX DFW policy into an equivalent k8s network policy. There are two main challenges here: * *The flattening challenge*: translating prioritized set of rules with actions `allow/deny/jump-to-app` into a flat set of `allow` rules (which is what k8s network policies support). * *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. -This is important since e.g. once a new VM as added with the relevant tags/labels it be granted the desired connectivity. +This is important since e.g. once a new VM is added with the relevant tags/labels in the target env, it will be granted the desired connectivity. ### The flattening challenge The translation of priortized allow/deny/pass rules into flat allow rules is exponential in the number of terms of the From d7c7447cf6ef241f85b7a8fb9d6f40c10e93bcf7 Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:06:43 +0200 Subject: [PATCH 07/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 60d8610a..724d1856 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -21,7 +21,7 @@ There are two main challenges here: This is important since e.g. once a new VM is added with the relevant tags/labels in the target env, it will be granted the desired connectivity. ### The flattening challenge -The translation of priortized allow/deny/pass rules into flat allow rules is exponential in the number of terms of the +The translation of priortized `allow,deny,jump-to-app` rules into flat `allow` rules is exponential in the number of terms of the original rules (to be accurate, the number of allow rules generated for each original allow rule is exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: 1. Ask the user to provide the tool with _hints_ - lists of disjoint tags/groups. From f051961557439563fd61306bf9eb6a9741534f69 Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:07:34 +0200 Subject: [PATCH 08/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 724d1856..3ca5e529 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -28,7 +28,7 @@ exponential in the number of term in this allow rule and in higher priority deny E.g., tags _{frontend, backend}_ are disjoint. In the future it is planned to "guess" these disjoint sets, and ask the user to approve them. -2. Apply various optimization to simplify the resulting rules and to rid redundant rules; the more accurate hints the +2. Apply various optimization to simplify the resulting rules and to delete redundant rules; the more accurate hints the tool is provided, the more concise and readable rules it will synthesize. ### The policy preserving challenge From cea0a22e5cea81b33760760d78ad646385586b41 Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:08:55 +0200 Subject: [PATCH 09/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 3ca5e529..568bad29 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -45,7 +45,8 @@ relevant *VM*s will be granted labels of this group. In the following releases w ## Output _k8s_resources_ dir under the dir specified in _synthesis-dump-dir_ contains the following files: -* **pods.yaml** the list of synthesized _VM's_ _pods_ with the labels of each pos +* **pods.yaml** the list pods (as place holder for VMs resources for now) with the relevant labels of each pod. +The labels are added based on original VMs' tags and groups in NSX env. * **policies.yaml** the k8s policies The combination of the policies and the pods' labels: From 1527f11e05512deb742e94f1783c3b9e4e29a40e Mon Sep 17 00:00:00 2001 From: shirim Date: Tue, 25 Feb 2025 17:09:50 +0200 Subject: [PATCH 10/24] CR --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 63148541..44e4e959 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -14,7 +14,7 @@ Flags: ``` ## Overview -Synthesize a given NSX DFW policy into an equivalent k8s network policy. +Synthesize a given NSX DFW policy into k8s network policy. There are two main challenges here: * *The flattening challenge*: translating prioritize _allow/deny/pass_ into flat allow rules (which is what k8s network policies support) * *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. From cbf73ce647a22bbd3302b4cc2a5b2a783ba33362 Mon Sep 17 00:00:00 2001 From: shirim Date: Wed, 26 Feb 2025 11:27:51 +0200 Subject: [PATCH 11/24] CR: use `aa` instead of _aa_ --- README_Synthesis.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 07192f82..7a6d6681 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -24,8 +24,8 @@ This is important since e.g. once a new VM is added with the relevant tags/label The translation of priortized `allow,deny,jump-to-app` rules into flat `allow` rules is exponential in the number of terms of the original rules (to be accurate, the number of allow rules generated for each original allow rule is exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: -1. Ask the user to provide the tool with _hints_ - lists of disjoint tags/groups. -E.g., tags _{frontend, backend}_ are disjoint. +1. Ask the user to provide the tool with `hints` - lists of disjoint tags/groups. +E.g., tags `{frontend, backend}` are disjoint. In the future it is planned to "guess" these disjoint sets, and ask the user to approve them. 2. Apply various optimization to simplify the resulting rules and to delete redundant rules; the more accurate hints the @@ -33,8 +33,8 @@ tool is provided, the more concise and readable rules it will synthesize. ### The policy preserving challenge To preserve the original intent of the policy, the synthesized policy refers, where possible, to permanent labeling such -as tags - e.g. _front-end_ - and not to temporarily labeling such as _VM_ names. E.g., a specific rule's _src_ is - defined to be group _aaa_ that is defined as _tag = backend_ and _tag != DB_ then the synthesized policy will refer to the value +as tags - e.g. `front-end` - and not to temporarily labeling such as `VM` names. E.g., a specific rule's `src` is + defined to be group `aaa` that is defined as `tag = backend` and `tag != DB` then the synthesized policy will refer to the value of the tag. ## Currently supported @@ -44,7 +44,7 @@ by an expression that we do not yet support, then the synthesized policy will re relevant *VM*s will be granted labels of this group. In the following releases we will expend our expressions support. ## Output -_k8s_resources_ dir under the dir specified in _synthesis-dump-dir_ contains the following files: +`k8s_resources` dir under the dir specified in `synthesis-dump-dir` contains the following files: * **pods.yaml** the list pods (as place holder for VMs resources for now) with the relevant labels of each pod. The labels are added based on original VMs' tags and groups in NSX env. * **policies.yaml** the k8s policies @@ -52,25 +52,25 @@ The labels are added based on original VMs' tags and groups in NSX env. The combination of the policies and the pods' labels: 1. Satisfies the snapshot of the connectivity at the time of the synthesis 2. Preserve the policy's intent, as expressed e.g. by *tags*, for future changes -(e.g. adding a _VM_ or changing its functionality and thus its labeling) +(e.g. adding a `VM` or changing its functionality and thus its labeling) ## Debugging The synthesize process is a complex one. Along it, in order to have the intent preserving synthesis as explained above, -we use a *symbolic* representation of the *rules*: each *symbolic rule* is a _priority_, an _action_, a -_src_, a _dst_ and a _connection_; The priority in a natural number; the action is _allow/deny/pass_; The _src_ and the _dst_ are _Conjunctions_ of simple expressions -(equal/not equal) over e.g. _tags_; the _connection_ is a protocol and potentially _src/dst_ min and max ports. +we use a *symbolic* representation of the *rules*: each *symbolic rule* is a `priority`, an `action`, a +`src`, a `dst` and a `connection`; The priority in a natural number; the action is `allow/deny/pass`; The `src` and the `dst` are `Conjunctions` of simple expressions +(equal/not equal) over e.g. `tags`; the `connection` is a protocol and potentially `src/dst` min and max ports. -The synthesis dump directory (specified in _synthesis-dump-dir_) contains (among others) the following files: +The synthesis dump directory (specified in `synthesis-dump-dir`) contains (among others) the following files: * under subdirectory **debug_dir** - * **config.txt** Contains the NSX config as being read by the tool; this includes _VMs_, _groups_ and _firewall rules_. - * **pre_processing.txt** Contains the translation of the firewall rule into _symbolic rules_ ; e.g., if a specific -src is a group which is an expression over tags, then this file will have this rule's _src_ defined over tags. - * **abstract_model.txt** The tool translates the *allow/deny/pass* rules from _pre_processing.txt_ into an abstract model that - contains the rules to be syntactically translated to _k8s policies_. If _synthesize-admin-policies_ is off then all rules must - be _flat allow rules_, and so all the rules in the abstract model are non-prioritized with action _allow_; - otherwise rules originating from the _environment_ category are translated to admin policies with + * **config.txt** Contains the NSX config as being read by the tool; this includes `VMs`, `groups` and `firewall rules`. + * **pre_processing.txt** Contains the translation of the firewall rule into `symbolic rules` ; e.g., if a specific +src is a group which is an expression over tags, then this file will have this rule's `src` defined over tags. + * **abstract_model.txt** The tool translates the *allow/deny/pass* rules from `pre_processing.txt` into an abstract model that + contains the rules to be syntactically translated to `k8s policies`. If `synthesize-admin-policies` is off then all rules must + be `flat allow rules`, and so all the rules in the abstract model are non-prioritized with action `allow`; + otherwise rules originating from the `environment` category are translated to admin policies with a priority and an allow/deny/pass action; rules that originate from the other categories are flat allow rules. - _abstract_model.txt_ contains these rules and a list of the groups, each groups with the expression that defined + `abstract_model.txt` contains these rules and a list of the groups, each groups with the expression that defined it and the snapshot of the *VMs* in the group. The following log files contain warning messages and various debug printing of the different stages From baa91e7823eab10943eef6b74fb5b43a986c334b Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Wed, 26 Feb 2025 11:32:51 +0200 Subject: [PATCH 12/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 7a6d6681..92d3a640 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -41,7 +41,7 @@ of the tag. Currently, the tool supports groups defined by expressions over tags; nested expression are not yet supported. If a group is defined by an expression that we do not yet support, then the synthesized policy will refer just to the group, and the -relevant *VM*s will be granted labels of this group. In the following releases we will expend our expressions support. +relevant *VM*s will be granted labels of this group. ## Output `k8s_resources` dir under the dir specified in `synthesis-dump-dir` contains the following files: From 331f6762bcfeaabc77b4feec7b2ecd8d69e6d068 Mon Sep 17 00:00:00 2001 From: shirim Date: Wed, 26 Feb 2025 12:44:11 +0200 Subject: [PATCH 13/24] CR --- README_Synthesis.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 92d3a640..14f3c602 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -14,7 +14,8 @@ Flags: ``` ## Overview -Synthesize a given NSX DFW policy into k8s network policy. +Synthesize a given `NSX DFW` policy into k8s network policy. +The result may not be totally equivalent, due to limitations of the target policy; more details regarding the k8s synthesis [here](#limitation). There are two main challenges here: * *The flattening challenge*: translating prioritized set of rules with actions `allow/deny/jump-to-app` into a flat set of `allow` rules (which is what k8s network policies support). * *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. @@ -44,8 +45,9 @@ by an expression that we do not yet support, then the synthesized policy will re relevant *VM*s will be granted labels of this group. ## Output -`k8s_resources` dir under the dir specified in `synthesis-dump-dir` contains the following files: -* **pods.yaml** the list pods (as place holder for VMs resources for now) with the relevant labels of each pod. +### Synthesized k8s resource +`k8s_resources` folder under the folder specified in `synthesis-dump-dir` contains the following files: +* **pods.yaml** the list pods (as placeholder for VMs resources for now) with the relevant labels of each pod. The labels are added based on original VMs' tags and groups in NSX env. * **policies.yaml** the k8s policies @@ -54,6 +56,12 @@ The combination of the policies and the pods' labels: 2. Preserve the policy's intent, as expressed e.g. by *tags*, for future changes (e.g. adding a `VM` or changing its functionality and thus its labeling) + + +#### limitations +There are differences in the expression power between `NSX DFW` to `Kubernetes Network Policies`; e.g. `ICMP protocols` +support depends on the network plugin and is not supported by our synthesis. + ## Debugging The synthesize process is a complex one. Along it, in order to have the intent preserving synthesis as explained above, we use a *symbolic* representation of the *rules*: each *symbolic rule* is a `priority`, an `action`, a From 7821a75be836831e385a5eda6d9a0bf5f22ae419 Mon Sep 17 00:00:00 2001 From: shirim Date: Wed, 26 Feb 2025 14:02:00 +0200 Subject: [PATCH 14/24] typo --- README_Synthesis.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 14f3c602..a263b93b 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -19,7 +19,8 @@ The result may not be totally equivalent, due to limitations of the target polic There are two main challenges here: * *The flattening challenge*: translating prioritized set of rules with actions `allow/deny/jump-to-app` into a flat set of `allow` rules (which is what k8s network policies support). * *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. -This is important since e.g. once a new VM is added with the relevant tags/labels in the target env, it will be granted the desired connectivity. +This is important since e.g. once a new VM is added with the relevant tags/labels in the target env, +it will be granted the desired connectivity. ### The flattening challenge The translation of priortized `allow,deny,jump-to-app` rules into flat `allow` rules is exponential in the number of terms of the From e4f2fe3ad04aeea06fcea622d9ed3641d7e7af2d Mon Sep 17 00:00:00 2001 From: ShiriMoran <139739065+ShiriMoran@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:02:37 +0200 Subject: [PATCH 15/24] Update README_Synthesis.md Co-authored-by: Adi Sosnovich <82078442+adisos@users.noreply.github.com> --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index a263b93b..24548e6f 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -18,7 +18,7 @@ Synthesize a given `NSX DFW` policy into k8s network policy. The result may not be totally equivalent, due to limitations of the target policy; more details regarding the k8s synthesis [here](#limitation). There are two main challenges here: * *The flattening challenge*: translating prioritized set of rules with actions `allow/deny/jump-to-app` into a flat set of `allow` rules (which is what k8s network policies support). -* *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just synthesis a snapshot. +* *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just generate any set of rules that preserves the connectivity between VMs given the current state of the configuration. For example, a DFW rule that uses an NSX group with no VMs at the moment of analysis and synthesis, may still be relevant to maintain in the conversion to network policies. This is important since e.g. once a new VM is added with the relevant tags/labels in the target env, it will be granted the desired connectivity. From 7494ae1d09a4729919e59764ec98cc627e292f51 Mon Sep 17 00:00:00 2001 From: shirim Date: Wed, 26 Feb 2025 14:25:24 +0200 Subject: [PATCH 16/24] readme --- README_Synthesis.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 24548e6f..d74b7e16 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -18,9 +18,9 @@ Synthesize a given `NSX DFW` policy into k8s network policy. The result may not be totally equivalent, due to limitations of the target policy; more details regarding the k8s synthesis [here](#limitation). There are two main challenges here: * *The flattening challenge*: translating prioritized set of rules with actions `allow/deny/jump-to-app` into a flat set of `allow` rules (which is what k8s network policies support). -* *The intent preserving challenge*: maintain the original semantic intent of the rules, and not just generate any set of rules that preserves the connectivity between VMs given the current state of the configuration. For example, a DFW rule that uses an NSX group with no VMs at the moment of analysis and synthesis, may still be relevant to maintain in the conversion to network policies. -This is important since e.g. once a new VM is added with the relevant tags/labels in the target env, -it will be granted the desired connectivity. +* *The intent preserving challenge*: maintain the original semantic intent of the rules +and not just generate a set of rules that preserves the connectivity between VMs given the current state of the configuration. + ### The flattening challenge The translation of priortized `allow,deny,jump-to-app` rules into flat `allow` rules is exponential in the number of terms of the @@ -34,10 +34,18 @@ disjoint sets, and ask the user to approve them. tool is provided, the more concise and readable rules it will synthesize. ### The policy preserving challenge -To preserve the original intent of the policy, the synthesized policy refers, where possible, to permanent labeling such -as tags - e.g. `front-end` - and not to temporarily labeling such as `VM` names. E.g., a specific rule's `src` is - defined to be group `aaa` that is defined as `tag = backend` and `tag != DB` then the synthesized policy will refer to the value -of the tag. +The synthesis maintains the original semantic intent of the rules +and not just generates a set of rules that preserves the connectivity between VMs given the current state of the configuration. +For example: +* When a `frontend` `VM` is added it should be granted the policies defined for the `frontend` +* A DFW rule that uses an NSX group with no VMs at the moment of analysis and synthesis, +will still be relevant to maintain in the conversion to network policies. + +To preserve the original intent of the policy, the synthesized policy refers to permanent labeling such +as tags - e.g. `front-end` - and not to temporal labeling such as `VM` names. E.g., given a specific rule's `src` + defined to be group `aaa` that is defined as `tag = backend` and `tag != DB` then the synthesized policy will maintain +this reference to the tag's value. +`todo: add tag -> labeling machanism` ## Currently supported Currently, the tool supports groups defined by expressions over tags; nested expression are not yet supported. From 9397633e0b2b6f3bc6c1f2e8463ec14622322878 Mon Sep 17 00:00:00 2001 From: shirim Date: Wed, 26 Feb 2025 14:37:34 +0200 Subject: [PATCH 17/24] term not defined --- README_Synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index d74b7e16..873cc5d5 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -23,7 +23,7 @@ and not just generate a set of rules that preserves the connectivity between VMs ### The flattening challenge -The translation of priortized `allow,deny,jump-to-app` rules into flat `allow` rules is exponential in the number of terms of the +The translation of priortized `allow,deny,jump-to-app` rules into flat `allow` rules is exponential in the size of the original rules (to be accurate, the number of allow rules generated for each original allow rule is exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: 1. Ask the user to provide the tool with `hints` - lists of disjoint tags/groups. From ffed4357c8ee7e47e3ed3f385544acda9f2ecb1f Mon Sep 17 00:00:00 2001 From: shirim Date: Wed, 26 Feb 2025 15:11:57 +0200 Subject: [PATCH 18/24] CR: breaking and clarifying too long sentence --- README_Synthesis.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 873cc5d5..e415c4bc 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -41,10 +41,12 @@ For example: * A DFW rule that uses an NSX group with no VMs at the moment of analysis and synthesis, will still be relevant to maintain in the conversion to network policies. -To preserve the original intent of the policy, the synthesized policy refers to permanent labeling such -as tags - e.g. `front-end` - and not to temporal labeling such as `VM` names. E.g., given a specific rule's `src` - defined to be group `aaa` that is defined as `tag = backend` and `tag != DB` then the synthesized policy will maintain -this reference to the tag's value. +To preserve the original intent of the policy, the synthesized policy prioritizes referencing non-ephemeral features. +E.g., it prefers referencing`frontend` label instead of referencing `VMs'` names. `VM`s may be deleted and added, while +the `frontend` label is always granted to any relevant `VM`. +Specifically, given a rule with `src`defined as group `aaa` which is defined as `tag = backend and tag != DB`, +the synthesized policy will reference the labels corresponding to the `backend` and `DB` values. + `todo: add tag -> labeling machanism` ## Currently supported From 8c8546a7213cc276fdd8b9d4fbccbc3e76cbf2df Mon Sep 17 00:00:00 2001 From: shirim Date: Wed, 26 Feb 2025 15:36:38 +0200 Subject: [PATCH 19/24] add example for supported and not supported expression --- README_Synthesis.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index e415c4bc..4a9adcad 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -50,10 +50,14 @@ the synthesized policy will reference the labels corresponding to the `backend` `todo: add tag -> labeling machanism` ## Currently supported -Currently, the tool supports groups defined by expressions over tags; nested expression are not yet supported. -If a group is defined -by an expression that we do not yet support, then the synthesized policy will refer just to the group, and the -relevant *VM*s will be granted labels of this group. +Currently, the tool supports groups defined by expressions over tags; `nested NSX expression` are not yet supported. +If a group is defined by an expression that we do not yet support, then the synthesized policy will refer just to the group, +and the relevant *VM*s will be granted labels of this group. + +For example, the expression `tag = backend and tag != DB` is supported, while the nested expression +`(tag = backend and tag != DB) or (tag = research) ` is not supported. For a group defined over the former expression, +the synthesis will reference labels corresponding to the above tags' values, while for a group defined over the latter +expression, the synthesis will reference a label corresponding to the group. ## Output ### Synthesized k8s resource From addf52c46b30e500d05a56819cf4059c58e4744d Mon Sep 17 00:00:00 2001 From: shirim Date: Thu, 27 Feb 2025 10:19:49 +0200 Subject: [PATCH 20/24] hints example with hints --- pkg/synthesis/synthesis_test.go | 8 +- .../abstract_models/ExampleHintsDisjoint.txt | 25 ++++ .../ExampleHintsDisjoint/pods.yaml | 59 ++++++++ .../ExampleHintsDisjoint/policies.yaml | 129 ++++++++++++++++++ .../pre_process/ExampleHintsDisjoint.txt | 19 +++ 5 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint.txt create mode 100644 pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/pods.yaml create mode 100644 pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/policies.yaml create mode 100644 pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint.txt diff --git a/pkg/synthesis/synthesis_test.go b/pkg/synthesis/synthesis_test.go index 47d95812..6b2ad730 100644 --- a/pkg/synthesis/synthesis_test.go +++ b/pkg/synthesis/synthesis_test.go @@ -104,6 +104,12 @@ var groupsByVmsTests = []synthesisTest{ synthesizeAdmin: false, noHint: true, }, + { + name: "ExampleHintsDisjoint", + exData: &data.ExampleHintsDisjoint, + synthesizeAdmin: false, + noHint: false, + }, { name: "ExampleHogwartsSimpler", exData: &data.ExampleHogwartsSimpler, @@ -456,7 +462,7 @@ const ( ) // to generate output results change runTestMode: -const runTestMode = OutputComparison +const runTestMode = OutputGeneration func compareOrRegenerateOutputDirPerTest(t *testing.T, actualDir, expectedDir, testName string) { actualFiles, err := os.ReadDir(actualDir) diff --git a/pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint.txt b/pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint.txt new file mode 100644 index 00000000..90080187 --- /dev/null +++ b/pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint.txt @@ -0,0 +1,25 @@ + +Abstract Model Details +======================= + +Groups' definition +~~~~~~~~~~~~~~~~~~ +Group |Expression |VM +Dumbledore1 | |Dumbledore1 +Dumbledore2 | |Dumbledore2 +Gryffindor | |Gryffindor +Hufflepuff | |Hufflepuff +Slytherin | |Slytherin + +Allow Only Rules +~~~~~~~~~~~~~~~~~ +inbound rules +Original allow rule priority |Rule id |Src |Dst |Connection +0 |9200 |(group = Dumbledore1) |(group != Slytherin) |All Connections +1 |9201 |(group = Dumbledore2) |(group != Gryffindor) |All Connections + +outbound rules +Original allow rule priority |Rule id |Src |Dst |Connection +0 |9200 |(group = Dumbledore1) |(group != Slytherin) |All Connections +1 |9201 |(group = Dumbledore2) |(group != Gryffindor) |All Connections + diff --git a/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/pods.yaml b/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/pods.yaml new file mode 100644 index 00000000..94d55be7 --- /dev/null +++ b/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/pods.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + group__Slytherin: "true" + name: Slytherin + namespace: default +spec: + containers: null +status: {} +--- +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + group__Hufflepuff: "true" + name: Hufflepuff + namespace: default +spec: + containers: null +status: {} +--- +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + group__Gryffindor: "true" + name: Gryffindor + namespace: default +spec: + containers: null +status: {} +--- +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + group__Dumbledore1: "true" + name: Dumbledore1 + namespace: default +spec: + containers: null +status: {} +--- +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + group__Dumbledore2: "true" + name: Dumbledore2 + namespace: default +spec: + containers: null +status: {} diff --git a/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/policies.yaml b/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/policies.yaml new file mode 100644 index 00000000..93bb3d4d --- /dev/null +++ b/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint/policies.yaml @@ -0,0 +1,129 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + description: Network Policy To Allow Access To DNS Server + nsx-id: none + creationTimestamp: null + name: dns-policy + namespace: default +spec: + egress: + - ports: + - port: 53 + protocol: UDP + to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + podSelector: {} + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + description: 'src: (group = Dumbledore1) dst: (group != Slytherin) conn: All Connections' + nsx-id: "9200" + creationTimestamp: null + name: policy-1 + namespace: default +spec: + egress: + - to: + - podSelector: + matchExpressions: + - key: group__Slytherin + operator: DoesNotExist + podSelector: + matchExpressions: + - key: group__Dumbledore1 + operator: Exists + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + description: 'src: (group = Dumbledore1) dst: (group != Slytherin) conn: All Connections' + nsx-id: "9200" + creationTimestamp: null + name: policy-2 + namespace: default +spec: + ingress: + - from: + - podSelector: + matchExpressions: + - key: group__Dumbledore1 + operator: Exists + podSelector: + matchExpressions: + - key: group__Slytherin + operator: DoesNotExist + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + description: 'src: (group = Dumbledore2) dst: (group != Gryffindor) conn: All Connections' + nsx-id: "9201" + creationTimestamp: null + name: policy-3 + namespace: default +spec: + egress: + - to: + - podSelector: + matchExpressions: + - key: group__Gryffindor + operator: DoesNotExist + podSelector: + matchExpressions: + - key: group__Dumbledore2 + operator: Exists + policyTypes: + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + description: 'src: (group = Dumbledore2) dst: (group != Gryffindor) conn: All Connections' + nsx-id: "9201" + creationTimestamp: null + name: policy-4 + namespace: default +spec: + ingress: + - from: + - podSelector: + matchExpressions: + - key: group__Dumbledore2 + operator: Exists + podSelector: + matchExpressions: + - key: group__Gryffindor + operator: DoesNotExist + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + description: Default Deny Network Policy + nsx-id: "10230" + creationTimestamp: null + name: default-deny + namespace: default +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress diff --git a/pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint.txt b/pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint.txt new file mode 100644 index 00000000..c0b96911 --- /dev/null +++ b/pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint.txt @@ -0,0 +1,19 @@ +category: Application +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +symbolic inbound rules: +Priority |Rule Id |Action |Src |Dst |Connection +0 |9198 |deny |(group = Dumbledore1) |(group = Slytherin) |All Connections +1 |9199 |deny |(group = Dumbledore2) |(group = Gryffindor) |All Connections +2 |9200 |allow |(group = Dumbledore1) |(*) |All Connections +3 |9201 |allow |(group = Dumbledore2) |(*) |All Connections +4 |10230 |deny |(*) |(*) |All Connections + + +symbolic outbound rules: +Priority |Rule Id |Action |Src |Dst |Connection +0 |9198 |deny |(group = Dumbledore1) |(group = Slytherin) |All Connections +1 |9199 |deny |(group = Dumbledore2) |(group = Gryffindor) |All Connections +2 |9200 |allow |(group = Dumbledore1) |(*) |All Connections +3 |9201 |allow |(group = Dumbledore2) |(*) |All Connections +4 |10230 |deny |(*) |(*) |All Connections + From 355a44d1af0d31c6c9f62c18361ebfc4ea902b03 Mon Sep 17 00:00:00 2001 From: shirim Date: Thu, 27 Feb 2025 10:27:05 +0200 Subject: [PATCH 21/24] renamed to standard --- pkg/synthesis/synthesis_test.go | 4 ++-- ...oint_NoHint.txt => ExampleHintsDisjoint_NoHint_NoHint.txt} | 0 .../pods.yaml | 0 .../policies.yaml | 0 ...oint_NoHint.txt => ExampleHintsDisjoint_NoHint_NoHint.txt} | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename pkg/synthesis/tests_expected_output/abstract_models/{ExampleHintsDisjoint_NoHint.txt => ExampleHintsDisjoint_NoHint_NoHint.txt} (100%) rename pkg/synthesis/tests_expected_output/k8s_resources/{ExampleHintsDisjoint_NoHint => ExampleHintsDisjoint_NoHint_NoHint}/pods.yaml (100%) rename pkg/synthesis/tests_expected_output/k8s_resources/{ExampleHintsDisjoint_NoHint => ExampleHintsDisjoint_NoHint_NoHint}/policies.yaml (100%) rename pkg/synthesis/tests_expected_output/pre_process/{ExampleHintsDisjoint_NoHint.txt => ExampleHintsDisjoint_NoHint_NoHint.txt} (100%) diff --git a/pkg/synthesis/synthesis_test.go b/pkg/synthesis/synthesis_test.go index 6b2ad730..6a216b22 100644 --- a/pkg/synthesis/synthesis_test.go +++ b/pkg/synthesis/synthesis_test.go @@ -99,7 +99,7 @@ var groupsByVmsTests = []synthesisTest{ noHint: true, }, { - name: "ExampleHintsDisjoint", + name: "ExampleHintsDisjoint_NoHint", exData: &data.ExampleHintsDisjoint, synthesizeAdmin: false, noHint: true, @@ -462,7 +462,7 @@ const ( ) // to generate output results change runTestMode: -const runTestMode = OutputGeneration +const runTestMode = OutputComparison func compareOrRegenerateOutputDirPerTest(t *testing.T, actualDir, expectedDir, testName string) { actualFiles, err := os.ReadDir(actualDir) diff --git a/pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint_NoHint.txt b/pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint_NoHint_NoHint.txt similarity index 100% rename from pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint_NoHint.txt rename to pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint_NoHint_NoHint.txt diff --git a/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint/pods.yaml b/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint_NoHint/pods.yaml similarity index 100% rename from pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint/pods.yaml rename to pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint_NoHint/pods.yaml diff --git a/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint/policies.yaml b/pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint_NoHint/policies.yaml similarity index 100% rename from pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint/policies.yaml rename to pkg/synthesis/tests_expected_output/k8s_resources/ExampleHintsDisjoint_NoHint_NoHint/policies.yaml diff --git a/pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint_NoHint.txt b/pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint_NoHint_NoHint.txt similarity index 100% rename from pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint_NoHint.txt rename to pkg/synthesis/tests_expected_output/pre_process/ExampleHintsDisjoint_NoHint_NoHint.txt From a25b8531b4f187c57856314af16a6eec9b7af2d0 Mon Sep 17 00:00:00 2001 From: shirim Date: Thu, 27 Feb 2025 10:40:08 +0200 Subject: [PATCH 22/24] added text regarding hints --- README_Synthesis.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README_Synthesis.md b/README_Synthesis.md index 4a9adcad..99d9777f 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -10,7 +10,7 @@ Flags: -- synthesize-admin-policies flag to synthesize category environment into admin network policies (which included deny, pass and priority) (default false) -- disjoint-hint comma separated list of NSX groups/tags that are always disjoint in their VM members, needed for an effective and sound synthesis process, can specify more than one hint - (example: \"--" + disjointHintsFlag + " frontend,backend --" + disjointHintsFlag + " app,web,db\") + (example: --disjoint-hint frontend,backend --disjoint-hint app,web,db) ``` ## Overview @@ -28,8 +28,9 @@ original rules (to be accurate, the number of allow rules generated for each ori exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: 1. Ask the user to provide the tool with `hints` - lists of disjoint tags/groups. E.g., tags `{frontend, backend}` are disjoint. -In the future it is planned to "guess" these -disjoint sets, and ask the user to approve them. +In the future it is planned to "guess" these disjoint sets, and ask the user to approve them. +Consider the example `ExampleHintsDisjoint` in `./pkg/data/examples`. +When run without Hints the result contains 2. Apply various optimization to simplify the resulting rules and to delete redundant rules; the more accurate hints the tool is provided, the more concise and readable rules it will synthesize. From fe038b869250e74f89931054e840c400b29e4f34 Mon Sep 17 00:00:00 2001 From: shirim Date: Thu, 27 Feb 2025 10:59:38 +0200 Subject: [PATCH 23/24] CR: added hints examples --- README_Synthesis.md | 7 +++-- pkg/data/exampleHint.go | 64 ++++++++++++++++++++++++++++++++++++++++ pkg/data/examples.go | 65 ----------------------------------------- 3 files changed, 68 insertions(+), 68 deletions(-) create mode 100644 pkg/data/exampleHint.go diff --git a/README_Synthesis.md b/README_Synthesis.md index 99d9777f..bf529ca4 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -28,9 +28,10 @@ original rules (to be accurate, the number of allow rules generated for each ori exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: 1. Ask the user to provide the tool with `hints` - lists of disjoint tags/groups. E.g., tags `{frontend, backend}` are disjoint. -In the future it is planned to "guess" these disjoint sets, and ask the user to approve them. -Consider the example `ExampleHintsDisjoint` in `./pkg/data/examples`. -When run without Hints the result contains +In the future it is planned to "guess" these disjoint sets, and ask the user to approve them. E.g., +for [this example](pkg/data/exampleHint.go) there are two related files: [flat allow rules without hints](pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint_NoHint_NoHint.txt) +contains the flat allow rules when executed without hints; and [flat allow rules with hints](pkg/synthesis/tests_expected_output/abstract_models/ExampleHintsDisjoint.txt) +contains the flat allow when executed with `--disjoint-hint sly, huf, gry, dum1, dum2` 2. Apply various optimization to simplify the resulting rules and to delete redundant rules; the more accurate hints the tool is provided, the more concise and readable rules it will synthesize. diff --git a/pkg/data/exampleHint.go b/pkg/data/exampleHint.go new file mode 100644 index 00000000..ba671dad --- /dev/null +++ b/pkg/data/exampleHint.go @@ -0,0 +1,64 @@ +package data + +// ExampleHintsDisjoint for testing the hint of disjoint groups/tags and relevant optimization +// Dumbledore1 can talk to all but Slytherin +// Dumbledore2 can talk to all but Gryffindor +var ExampleHintsDisjoint = Example{ + Name: "ExampleHintsDisjoint", + VMs: []string{sly, huf, gry, dum1, dum2}, + GroupsByVMs: map[string][]string{ + sly: {sly}, + huf: {huf}, + gry: {gry}, + dum1: {dum1}, + dum2: {dum2}, + }, + Policies: []Category{ + { + Name: "From-Dumbledore-connection", + CategoryType: application, + Rules: []Rule{ + { + Name: "Dumb1-Not-Sly", + ID: newRuleID, + Source: dum1, + Dest: sly, + Services: []string{anyStr}, + Action: Drop, + }, + { + Name: "Dumb2-Not-Gryf", + ID: newRuleID + 1, + Source: dum2, + Dest: gry, + Services: []string{anyStr}, + Action: Drop, + }, + { + Name: "Dumb1-To-All", + ID: newRuleID + 2, + Source: dum1, + Dest: anyStr, + Services: []string{anyStr}, + Action: Allow, + }, + { + Name: "Dumb2-To-All", + ID: newRuleID + 3, + Source: dum2, + Dest: anyStr, + Services: []string{anyStr}, + Action: Allow, + }, + }, + }, + { + Name: defaultL3, + CategoryType: application, + Rules: []Rule{ + DefaultDenyRule(denyRuleIDEnv), + }, + }, + }, + DisjointGroupsTags: disjointHouses2Dum, +} diff --git a/pkg/data/examples.go b/pkg/data/examples.go index 79982420..f64f93da 100644 --- a/pkg/data/examples.go +++ b/pkg/data/examples.go @@ -716,71 +716,6 @@ var ExampleDenyPassSimple = Example{ /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// ExampleHintsDisjoint for testing the hint of disjoint groups/tags and relevant optimization -// Dumbledore1 can talk to all but Slytherin -// Dumbledore2 can talk to all but Gryffindor -var ExampleHintsDisjoint = Example{ - Name: "ExampleHintsDisjoint", - VMs: []string{sly, huf, gry, dum1, dum2}, - GroupsByVMs: map[string][]string{ - sly: {sly}, - huf: {huf}, - gry: {gry}, - dum1: {dum1}, - dum2: {dum2}, - }, - Policies: []Category{ - { - Name: "From-Dumbledore-connection", - CategoryType: application, - Rules: []Rule{ - { - Name: "Dumb1-Not-Sly", - ID: newRuleID, - Source: dum1, - Dest: sly, - Services: []string{anyStr}, - Action: Drop, - }, - { - Name: "Dumb2-Not-Gryf", - ID: newRuleID + 1, - Source: dum2, - Dest: gry, - Services: []string{anyStr}, - Action: Drop, - }, - { - Name: "Dumb1-To-All", - ID: newRuleID + 2, - Source: dum1, - Dest: anyStr, - Services: []string{anyStr}, - Action: Allow, - }, - { - Name: "Dumb2-To-All", - ID: newRuleID + 3, - Source: dum2, - Dest: anyStr, - Services: []string{anyStr}, - Action: Allow, - }, - }, - }, - { - Name: defaultL3, - CategoryType: application, - Rules: []Rule{ - DefaultDenyRule(denyRuleIDEnv), - }, - }, - }, - DisjointGroupsTags: disjointHouses2Dum, -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /* ExampleHogwarts with macro and micro segmentation From c542f5d587c8ba8a3ef38929d84ce9aa2af1b109 Mon Sep 17 00:00:00 2001 From: shirim Date: Thu, 27 Feb 2025 11:51:27 +0200 Subject: [PATCH 24/24] CR: added admin policies examples --- README_Synthesis.md | 16 ++++ pkg/data/exampleHogwarts.go | 164 ++++++++++++++++++++++++++++++++++++ pkg/data/examples.go | 160 ----------------------------------- 3 files changed, 180 insertions(+), 160 deletions(-) create mode 100644 pkg/data/exampleHogwarts.go diff --git a/README_Synthesis.md b/README_Synthesis.md index bf529ca4..319a4287 100644 --- a/README_Synthesis.md +++ b/README_Synthesis.md @@ -23,6 +23,22 @@ and not just generate a set of rules that preserves the connectivity between VMs ### The flattening challenge +There are two modes of policies synthesis, depending on the value of `synthesize-admin-policies`; when +it is not active then priortized `allow, deny, jump-to-app` rules from all `NSX categories` should be synthesized to +`k8s network policy`, namely, to `flat allow rules`; when it is activated then rules from `NSX category environment` should be synthesized to +`admin network policy` which rules have `allow, deny, pass` and priority; other categories should be synthesized as +before to `k8s network policy`. + +For example, for [this example](pkg/data/exampleHogwarts.go) there are two related files: +[no admin policies](pkg/synthesis/tests_expected_output/abstract_models/ExampleHogwarts.txt), which is the result of execution +without `-- synthesize-admin-policies` and contains translation of all rules to flat allow rules; and +[with admin policies](pkg/synthesis/tests_expected_output/abstract_models/ExampleHogwartsAdmin_AdminPoliciesEnabled.txt), +result of execution with `-- synthesize-admin-policies`, contains the translation when `NSX category env` +is synthesized to admin polices that can use `deny/pass/allow` and priorities. Full synthesis results for this example can be found +[here](pkg/synthesis/tests_expected_output/k8s_resources/ExampleHogwarts) +for non-admin polices and [here](pkg/synthesis/tests_expected_output/k8s_resources/ExampleHogwartsAdmin_AdminPoliciesEnabled) +for admin policies. + The translation of priortized `allow,deny,jump-to-app` rules into flat `allow` rules is exponential in the size of the original rules (to be accurate, the number of allow rules generated for each original allow rule is exponential in the number of term in this allow rule and in higher priority deny and pass rules). To tackle this we: diff --git a/pkg/data/exampleHogwarts.go b/pkg/data/exampleHogwarts.go new file mode 100644 index 00000000..42dc2373 --- /dev/null +++ b/pkg/data/exampleHogwarts.go @@ -0,0 +1,164 @@ +package data + +import ( + "github.com/np-guard/models/pkg/netp" + "github.com/np-guard/models/pkg/netset" + nsx "github.com/np-guard/vmware-analyzer/pkg/analyzer/generated" +) + +/* +ExampleHogwarts with macro and micro segmentation + +Slytherin House {Vms : S1, S2, S3} +Hufflepuff House {VMs: H1, H2, H3} +Gryffindor House {VMs: G1, G2, G3} +Dumbledore {VMs: D1, D2} + Web {S1, H1, G1} + APP {S2, H2, G2} + DB {S3, H3, G3} + + +Macro Segmentation +- Houses (tenants / apps) must not communicate with each other +- each house must be able to communicate to Dumbledore (shared services) +- [Dumbledore must be able to communicate only to Gryffindor house (ML server / other special use case server, etc )] removed +- Within each house (tenants/apps) tiers must be able to communicate with each other + +Macro Segmentation - the starting point to the land of zero trust + +micro segmentation +- within each house (tenants/apps) tiers must have granular firewall policies + - anyone can access WEB servers + - only Web server can access App server over a whitelisted port + - only App Server can access DB Server over a whitelisted port + + +*/ + +var ExampleHogwarts = Example{ + Name: "ExampleHogwarts", + VMs: []string{slyWeb, slyApp, slyDB, hufWeb, hufApp, hufDB, + gryWeb, gryApp, gryDB, dum1, dum2}, + GroupsByVMs: map[string][]string{ + sly: {slyWeb, slyApp, slyDB}, + huf: {hufWeb, hufApp, hufDB}, + gry: {gryWeb, gryApp, gryDB}, + dum: {dum1, dum2}, + web: {slyWeb, gryWeb, hufWeb}, + app: {slyApp, gryApp, hufApp}, + db: {slyDB, gryDB, hufDB}, + }, + Policies: []Category{ + { + Name: "Gryffindor-to-Gryffindor-allow", + CategoryType: environment, + Rules: []Rule{ + { + Name: "allow-Gryffindor-to-Gryffindor", + ID: 10218, + Source: gry, + Dest: gry, + Action: JumpToApp, + Conn: netset.AllTCPTransport(), + }, + }, + }, + { + Name: "Hufflepuff-to-Hufflepuff-allow", + CategoryType: environment, + Rules: []Rule{ + { + Name: "allow-Hufflepuff-to-Hufflepuff", + ID: 10219, + Source: huf, + Dest: huf, + Action: JumpToApp, + //nolint:mnd // these are the port numbers for the test + Conn: netset.NewUDPTransport(netp.MinPort, netp.MinPort, 300, 320), + Direction: string(nsx.RuleDirectionIN), + }, + }, + }, + { + Name: "Slytherin-to-Slytherin-allow", + CategoryType: environment, + Rules: []Rule{ + { + Name: "allow-Slytherin-to-Slytherin", + ID: 10220, + Source: sly, + Dest: sly, + Services: []string{anyStr}, + Action: JumpToApp, + }, + }, + }, + { + Name: "Dumbledore-connection", + CategoryType: environment, + Rules: []Rule{ + { + Name: "allow-Dumbledore-to-all", + ID: 10221, + Source: dum, + Dest: gry, + Services: []string{anyStr}, + Action: JumpToApp, + }, + { + Name: "default-deny-env", + ID: 10300, + Source: anyStr, + Dest: anyStr, + Services: []string{anyStr}, + Action: Drop, + }, + }, + }, + + { + Name: "Intra-App-Policy", + CategoryType: application, + Rules: []Rule{ + { + Name: "Client-Access", + ID: 10400, + Source: anyStr, + Dest: web, + Services: []string{anyStr}, + Action: Allow, + }, + { + Name: "Web-To-App-Access", + ID: 10401, + Source: web, + Dest: app, + Services: []string{anyStr}, + Action: Allow, + }, + { + Name: "App-To-DB-Access", + ID: 10405, + Source: app, + Dest: db, + Services: []string{anyStr}, + Action: Allow, + }, + }, + }, + { + Name: defaultL3, + CategoryType: application, + Rules: []Rule{ + DefaultDenyRule(denyRuleIDEnv), + }, + }, + }, + DisjointGroupsTags: [][]string{ + {sly, huf, gry, dum}, + {web, app, db}, + {web, dum}, + {app, dum}, + {db, dum}, + }, +} diff --git a/pkg/data/examples.go b/pkg/data/examples.go index f64f93da..2869997b 100644 --- a/pkg/data/examples.go +++ b/pkg/data/examples.go @@ -1,7 +1,6 @@ package data import ( - "github.com/np-guard/models/pkg/netp" "github.com/np-guard/models/pkg/netset" nsx "github.com/np-guard/vmware-analyzer/pkg/analyzer/generated" ) @@ -716,165 +715,6 @@ var ExampleDenyPassSimple = Example{ /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -/* -ExampleHogwarts with macro and micro segmentation - -Slytherin House {Vms : S1, S2, S3} -Hufflepuff House {VMs: H1, H2, H3} -Gryffindor House {VMs: G1, G2, G3} -Dumbledore {VMs: D1, D2} - Web {S1, H1, G1} - APP {S2, H2, G2} - DB {S3, H3, G3} - - -Macro Segmentation -- Houses (tenants / apps) must not communicate with each other -- each house must be able to communicate to Dumbledore (shared services) -- [Dumbledore must be able to communicate only to Gryffindor house (ML server / other special use case server, etc )] removed -- Within each house (tenants/apps) tiers must be able to communicate with each other - -Macro Segmentation - the starting point to the land of zero trust - -micro segmentation -- within each house (tenants/apps) tiers must have granular firewall policies - - anyone can access WEB servers - - only Web server can access App server over a whitelisted port - - only App Server can access DB Server over a whitelisted port - - -*/ - -var ExampleHogwarts = Example{ - Name: "ExampleHogwarts", - VMs: []string{slyWeb, slyApp, slyDB, hufWeb, hufApp, hufDB, - gryWeb, gryApp, gryDB, dum1, dum2}, - GroupsByVMs: map[string][]string{ - sly: {slyWeb, slyApp, slyDB}, - huf: {hufWeb, hufApp, hufDB}, - gry: {gryWeb, gryApp, gryDB}, - dum: {dum1, dum2}, - web: {slyWeb, gryWeb, hufWeb}, - app: {slyApp, gryApp, hufApp}, - db: {slyDB, gryDB, hufDB}, - }, - Policies: []Category{ - { - Name: "Gryffindor-to-Gryffindor-allow", - CategoryType: environment, - Rules: []Rule{ - { - Name: "allow-Gryffindor-to-Gryffindor", - ID: 10218, - Source: gry, - Dest: gry, - Action: JumpToApp, - Conn: netset.AllTCPTransport(), - }, - }, - }, - { - Name: "Hufflepuff-to-Hufflepuff-allow", - CategoryType: environment, - Rules: []Rule{ - { - Name: "allow-Hufflepuff-to-Hufflepuff", - ID: 10219, - Source: huf, - Dest: huf, - Action: JumpToApp, - //nolint:mnd // these are the port numbers for the test - Conn: netset.NewUDPTransport(netp.MinPort, netp.MinPort, 300, 320), - Direction: string(nsx.RuleDirectionIN), - }, - }, - }, - { - Name: "Slytherin-to-Slytherin-allow", - CategoryType: environment, - Rules: []Rule{ - { - Name: "allow-Slytherin-to-Slytherin", - ID: 10220, - Source: sly, - Dest: sly, - Services: []string{anyStr}, - Action: JumpToApp, - }, - }, - }, - { - Name: "Dumbledore-connection", - CategoryType: environment, - Rules: []Rule{ - { - Name: "allow-Dumbledore-to-all", - ID: 10221, - Source: dum, - Dest: gry, - Services: []string{anyStr}, - Action: JumpToApp, - }, - { - Name: "default-deny-env", - ID: 10300, - Source: anyStr, - Dest: anyStr, - Services: []string{anyStr}, - Action: Drop, - }, - }, - }, - - { - Name: "Intra-App-Policy", - CategoryType: application, - Rules: []Rule{ - { - Name: "Client-Access", - ID: 10400, - Source: anyStr, - Dest: web, - Services: []string{anyStr}, - Action: Allow, - }, - { - Name: "Web-To-App-Access", - ID: 10401, - Source: web, - Dest: app, - Services: []string{anyStr}, - Action: Allow, - }, - { - Name: "App-To-DB-Access", - ID: 10405, - Source: app, - Dest: db, - Services: []string{anyStr}, - Action: Allow, - }, - }, - }, - { - Name: defaultL3, - CategoryType: application, - Rules: []Rule{ - DefaultDenyRule(denyRuleIDEnv), - }, - }, - }, - DisjointGroupsTags: [][]string{ - {sly, huf, gry, dum}, - {web, app, db}, - {web, dum}, - {app, dum}, - {db, dum}, - }, -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////////////// - var disjointHousesAndFunctionality = [][]string{ {sly, huf, gry, dum}, {web, app, db}}