From 80c44adc91f3f32b78510db53dc1c1cd0c3e2e9b Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Wed, 4 Dec 2024 16:05:18 +0800 Subject: [PATCH 1/4] Reopen e2e traffic test --- go.mod | 7 +- go.sum | 77 ++------------ test/e2e/framework.go | 146 ++++++++++++++------------- test/e2e/nsx_security_policy_test.go | 124 ++++++++++------------- 4 files changed, 138 insertions(+), 216 deletions(-) diff --git a/go.mod b/go.mod index b779fad8a..ccf1e77e8 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( k8s.io/apimachinery v0.31.2 k8s.io/client-go v0.31.2 k8s.io/code-generator v0.31.0 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.19.0 ) @@ -64,13 +65,16 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -85,7 +89,7 @@ require ( golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect @@ -99,7 +103,6 @@ require ( k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index e1bb35210..d3c1ed43d 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,16 @@ -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg= github.com/agiledragon/gomonkey/v2 v2.11.0 h1:5oxSgA+tC1xuGsrIorR+sYiziYltmJyEZ9qA25b6l5U= github.com/agiledragon/gomonkey/v2 v2.11.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= -github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -28,25 +19,20 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gibson042/canonicaljson-go v1.0.3 h1:EAyF8L74AWabkyUmrvEFHEt/AGFQeD6RfwbAuf0j1bI= github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -70,9 +56,6 @@ github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -87,25 +70,15 @@ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -120,7 +93,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -129,7 +102,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= @@ -137,8 +110,6 @@ github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/openlyinc/pointy v1.1.2 h1:LywVV2BWC5Sp5v7FoP4bUD+2Yn5k0VNeRbU5vq9jUMY= github.com/openlyinc/pointy v1.1.2/go.mod h1:w2Sytx+0FVuMKn37xpXIAyBNhFNBIJGR/v2m7ik1WtM= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -160,11 +131,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -178,7 +146,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vmware-tanzu/vm-operator/api v1.8.2 h1:7cZHVusqAmAMFWvsiU7X5xontxdjasknI/sVfe0p0Z4= github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wfrj+0nH2zSEc9Q7AzWJ54= github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I= @@ -194,29 +161,9 @@ github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-202411180704 github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20241118070432-460aadb3b866/go.mod h1:ugk9I4YM62SSAox57l5NAVBCRIkPQ1RNLb3URxyTADc= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU= -go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI= -go.etcd.io/etcd/client/v2 v2.305.13/go.mod h1:iQnL7fepbiomdXMb3om1rHq96htNNGv2sJkEcZGDRRg= -go.etcd.io/etcd/client/v3 v3.5.14/go.mod h1:k3XfdV/VIHy/97rqWjoUzrj9tk7GgJGH9J8L4dNXmAk= -go.etcd.io/etcd/pkg/v3 v3.5.13/go.mod h1:N+4PLrp7agI/Viy+dUYpX7iRtSPvKq+w8Y14d1vX+m0= -go.etcd.io/etcd/raft/v3 v3.5.13/go.mod h1:uUFibGLn2Ksm2URMxN1fICGhk8Wu96EfDQyuLhAcAmw= -go.etcd.io/etcd/server/v3 v3.5.13/go.mod h1:K/8nbsGupHqmr5MkgaZpLlH1QdX1pcNQLAkODy44XcQ= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -260,9 +207,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= @@ -284,14 +230,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -303,7 +243,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -316,22 +255,18 @@ k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24 k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk= k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= k8s.io/code-generator v0.31.0 h1:w607nrMi1KeDKB3/F/J4lIoOgAwc+gV9ZKew4XRfMp8= k8s.io/code-generator v0.31.0/go.mod h1:84y4w3es8rOJOUUP1rLsIiGlO1JuEaPFXQPA9e/K6U0= -k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo= k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.31.0/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/test/e2e/framework.go b/test/e2e/framework.go index e6ab820f9..9eb430b44 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -8,6 +8,7 @@ import ( "net/url" "os/exec" "regexp" + "strconv" "strings" "time" @@ -15,10 +16,13 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" "github.com/vmware-tanzu/nsx-operator/pkg/logger" @@ -83,6 +87,12 @@ type TestData struct { var testData *TestData +type PodIPs struct { + ipv4 *net.IP + ipv6 *net.IP + ipStrings []string +} + func initProvider() error { providerFactory := map[string]func(string) (providers.ProviderInterface, error){ "remote": providers.NewRemoteProvider, @@ -332,8 +342,6 @@ func (data *TestData) deploymentWaitForNames(timeout time.Duration, namespace, d return podNames, nil } -// Temporarily disable traffic check -/* // podWaitFor polls the K8s apiServer until the specified Pod is found (in the test Namespace) and // the condition predicate is met (or until the provided timeout expires). func (data *TestData) podWaitFor(timeout time.Duration, name, namespace string, condition PodCondition) (*corev1.Pod, error) { @@ -398,37 +406,37 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin } // deploymentWaitForIPsOrNames polls the K8s apiServer until the specified Pod in deployment has an IP address -func (data *TestData) deploymentWaitForIPsOrNames(timeout time.Duration, namespace, deployment string) ([]string, []string, error) { - podIPStrings := sets.NewString() - var podNames []string - opt := metav1.ListOptions{ - LabelSelector: "deployment=" + deployment, - } - err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - if pods, err := data.clientset.CoreV1().Pods(namespace).List(context.TODO(), opt); err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, fmt.Errorf("error when getting Pod %v", err) - } else { - for _, p := range pods.Items { - if p.Status.Phase != corev1.PodRunning { - return false, nil - } else if p.Status.PodIP == "" { - return false, nil - } else { - podIPStrings.Insert(p.Status.PodIP) - podNames = append(podNames, p.Name) - } - } - return true, nil - } - }) - if err != nil { - return nil, nil, err - } - return podIPStrings.List(), podNames, nil -} +//func (data *TestData) deploymentWaitForIPsOrNames(timeout time.Duration, namespace, deployment string) ([]string, []string, error) { +// podIPStrings := sets.NewString() +// var podNames []string +// opt := metav1.ListOptions{ +// LabelSelector: "deployment=" + deployment, +// } +// err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { +// if pods, err := data.clientset.CoreV1().Pods(namespace).List(context.TODO(), opt); err != nil { +// if errors.IsNotFound(err) { +// return false, nil +// } +// return false, fmt.Errorf("error when getting Pod %v", err) +// } else { +// for _, p := range pods.Items { +// if p.Status.Phase != corev1.PodRunning { +// return false, nil +// } else if p.Status.PodIP == "" { +// return false, nil +// } else { +// podIPStrings.Insert(p.Status.PodIP) +// podNames = append(podNames, p.Name) +// } +// } +// return true, nil +// } +// }) +// if err != nil { +// return nil, nil, err +// } +// return podIPStrings.List(), podNames, nil +//} func parsePodIPs(podIPStrings sets.Set[string]) (*PodIPs, error) { ips := new(PodIPs) @@ -510,20 +518,19 @@ func (data *TestData) runPingCommandFromPod(namespace string, podName string, ta return nil } -func (data *TestData) runNetcatCommandFromPod(namespace string, podName string, containerName string, server string, port int) error { - cmd := []string{ - "/bin/sh", - "-c", - fmt.Sprintf("for i in $(seq 1 5); do nc -w 4 %s %d && exit 0 || sleep 1; done; exit 1", - server, port), - } - _, _, err := data.runCommandFromPod(namespace, podName, containerName, cmd) - if err != nil { - return err - } - return nil -} -*/ +//func (data *TestData) runNetcatCommandFromPod(namespace string, podName string, containerName string, server string, port int) error { +// cmd := []string{ +// "/bin/sh", +// "-c", +// fmt.Sprintf("for i in $(seq 1 5); do nc -w 4 %s %d && exit 0 || sleep 1; done; exit 1", +// server, port), +// } +// _, _, err := data.runCommandFromPod(namespace, podName, containerName, cmd) +// if err != nil { +// return err +// } +// return nil +//} func applyYAML(filename string, ns string) error { cmd := fmt.Sprintf("kubectl apply -f %s -n %s", filename, ns) @@ -547,30 +554,27 @@ func applyYAML(filename string, ns string) error { return nil } -// Temporarily disable traffic check -/* -func runCommand(cmd string) (string, error) { - err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (bool, error) { - var stdout, stderr bytes.Buffer - command := exec.Command("bash", "-c", cmd) - log.Info("Running command %s", cmd) - command.Stdout = &stdout - command.Stderr = &stderr - err := command.Run() - if err != nil { - log.Info("Error when running command %s: %v", cmd, err) - return false, nil - } - outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) - log.Info("Command %s returned with output: '%s' and error: '%s'", cmd, outStr, errStr) - if errStr != "" { - return false, nil - } - return true, nil - }) - return "", err -} -*/ +//func runCommand(cmd string) (string, error) { +// err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (bool, error) { +// var stdout, stderr bytes.Buffer +// command := exec.Command("bash", "-c", cmd) +// log.Info("Running command %s", cmd) +// command.Stdout = &stdout +// command.Stderr = &stderr +// err := command.Run() +// if err != nil { +// log.Info("Error when running command %s: %v", cmd, err) +// return false, nil +// } +// outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) +// log.Info("Command %s returned with output: '%s' and error: '%s'", cmd, outStr, errStr) +// if errStr != "" { +// return false, nil +// } +// return true, nil +// }) +// return "", err +//} func deleteYAML(filename string, ns string) error { cmd := fmt.Sprintf("kubectl delete -f %s -n %s", filename, ns) diff --git a/test/e2e/nsx_security_policy_test.go b/test/e2e/nsx_security_policy_test.go index 129b9718f..a9c8a7b19 100644 --- a/test/e2e/nsx_security_policy_test.go +++ b/test/e2e/nsx_security_policy_test.go @@ -51,28 +51,23 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { setupTest(t, ns) defer teardownTest(t, ns, defaultTimeout) - // Temporarily disable traffic check - /* - // Create pods - busyboxPath, _ := filepath.Abs("./manifest/testSecurityPolicy/busybox.yaml") - _ = applyYAML(busyboxPath, ns) - netcatPath, _ := filepath.Abs("./manifest/testSecurityPolicy/netcat-pod.yaml") - _ = applyYAML(netcatPath, ns) - - busybox := "busybox" - ncPod := "nc-pod - // Wait for pods - ps, err := testData.podWaitForIPs(defaultTimeout, busybox, ns) - t.Logf("Pods are %v", ps) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", busybox) - iPs, err := testData.podWaitForIPs(defaultTimeout, ncPod, ns) - t.Logf("Pods are %v", iPs) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", ncPod) - - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Create pods + busyboxPath, _ := filepath.Abs("./manifest/testSecurityPolicy/busybox.yaml") + _ = applyYAML(busyboxPath, ns) + netcatPath, _ := filepath.Abs("./manifest/testSecurityPolicy/netcat-pod.yaml") + _ = applyYAML(netcatPath, ns) + + busybox := "busybox" + ncPod := "nc-pod" + // Wait for pods + _, err = testData.podWaitForIPs(defaultTimeout, busybox, ns) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", busybox) + iPs, err := testData.podWaitForIPs(defaultTimeout, ncPod, ns) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", ncPod) + + // Ping from pod + err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) + assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) // Create security policy nsIsolationPath, _ := filepath.Abs("./manifest/testSecurityPolicy/ns-isolation-policy.yaml") require.NoError(t, applyYAML(nsIsolationPath, ns)) @@ -84,12 +79,9 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName0, true)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName1, true)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assertNotNil(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) + require.NoError(t, err, "Error when running ping command from test Pod %s", busybox) // Delete security policy _ = deleteYAML(nsIsolationPath, ns) @@ -111,12 +103,9 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName0, false)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName1, false)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) + assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) } // TestSecurityPolicyAddDeleteRule verifies that when adding or deleting rule, the security policy will be updated. @@ -194,28 +183,25 @@ func testSecurityPolicyMatchExpression(t *testing.T) { require.NoError(t, applyYAML(podPath, ns)) defer deleteYAML(podPath, "") - // Temporarily disable traffic check - // clientA := "client-a" - // clientB := "client-b" - // podA := "pod-a" - /* - // Wait for pods - ps, err := testData.podWaitForIPs(defaultTimeout, clientA, ns) - t.Logf("Pods are %v", ps) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientA) - psb, err := testData.podWaitForIPs(defaultTimeout, clientB, ns) - t.Logf("Pods are %v", psb) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientB) - iPs, err := testData.podWaitForIPs(defaultTimeout, podA, ns) - t.Logf("Pods are %v", iPs) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", podA) - - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) - */ + clientA := "client-a" + clientB := "client-b" + podA := "pod-a" + // Wait for pods + ps, err := testData.podWaitForIPs(defaultTimeout, clientA, ns) + log.Info("Pods are %v", ps) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientA) + psb, err := testData.podWaitForIPs(defaultTimeout, clientB, ns) + log.Info("Pods are %v", psb) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientB) + iPs, err := testData.podWaitForIPs(defaultTimeout, podA, ns) + log.Info("Pods are %v", iPs) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", podA) + + // Ping from pod + err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) + err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) // Create security policy nsIsolationPath, _ := filepath.Abs("./manifest/testSecurityPolicy/match-expression.yaml") @@ -227,14 +213,11 @@ func testSecurityPolicyMatchExpression(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeSecurityPolicy, securityPolicyName, true)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName, true)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NotNilf(t, err, "Error when running ping command from Pod %s", clientB) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) + err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) + assert.NotNilf(t, err, "Error when running ping command from Pod %s", clientB) // Delete security policy _ = deleteYAML(nsIsolationPath, ns) @@ -256,14 +239,11 @@ func testSecurityPolicyMatchExpression(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeSecurityPolicy, securityPolicyName, false)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName, false)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) + err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) } // TestSecurityPolicyNamedPortWithoutPod verifies that the traffic of security policy when named port applied. From 42f3e2b9fa46ebfdfad2e56ec9cb2058562d9dd6 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Wed, 4 Dec 2024 16:05:18 +0800 Subject: [PATCH 2/4] Reopen e2e traffic test --- go.mod | 5 +- test/e2e/framework.go | 146 ++++++++++++++------------- test/e2e/nsx_security_policy_test.go | 124 ++++++++++------------- 3 files changed, 131 insertions(+), 144 deletions(-) diff --git a/go.mod b/go.mod index b537d43d4..ccf1e77e8 100644 --- a/go.mod +++ b/go.mod @@ -65,13 +65,16 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -86,7 +89,7 @@ require ( golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 252ca3bc0..a0f3a61d8 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -8,6 +8,7 @@ import ( "net/url" "os/exec" "regexp" + "strconv" "strings" "time" @@ -15,10 +16,13 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" "github.com/vmware-tanzu/nsx-operator/pkg/logger" @@ -85,6 +89,12 @@ type TestData struct { var testData *TestData +type PodIPs struct { + ipv4 *net.IP + ipv6 *net.IP + ipStrings []string +} + func initProvider() error { providerFactory := map[string]func(string) (providers.ProviderInterface, error){ "remote": providers.NewRemoteProvider, @@ -334,8 +344,6 @@ func (data *TestData) deploymentWaitForNames(timeout time.Duration, namespace, d return podNames, nil } -// Temporarily disable traffic check -/* // podWaitFor polls the K8s apiServer until the specified Pod is found (in the test Namespace) and // the condition predicate is met (or until the provided timeout expires). func (data *TestData) podWaitFor(timeout time.Duration, name, namespace string, condition PodCondition) (*corev1.Pod, error) { @@ -400,37 +408,37 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin } // deploymentWaitForIPsOrNames polls the K8s apiServer until the specified Pod in deployment has an IP address -func (data *TestData) deploymentWaitForIPsOrNames(timeout time.Duration, namespace, deployment string) ([]string, []string, error) { - podIPStrings := sets.NewString() - var podNames []string - opt := metav1.ListOptions{ - LabelSelector: "deployment=" + deployment, - } - err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - if pods, err := data.clientset.CoreV1().Pods(namespace).List(context.TODO(), opt); err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, fmt.Errorf("error when getting Pod %v", err) - } else { - for _, p := range pods.Items { - if p.Status.Phase != corev1.PodRunning { - return false, nil - } else if p.Status.PodIP == "" { - return false, nil - } else { - podIPStrings.Insert(p.Status.PodIP) - podNames = append(podNames, p.Name) - } - } - return true, nil - } - }) - if err != nil { - return nil, nil, err - } - return podIPStrings.List(), podNames, nil -} +//func (data *TestData) deploymentWaitForIPsOrNames(timeout time.Duration, namespace, deployment string) ([]string, []string, error) { +// podIPStrings := sets.NewString() +// var podNames []string +// opt := metav1.ListOptions{ +// LabelSelector: "deployment=" + deployment, +// } +// err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { +// if pods, err := data.clientset.CoreV1().Pods(namespace).List(context.TODO(), opt); err != nil { +// if errors.IsNotFound(err) { +// return false, nil +// } +// return false, fmt.Errorf("error when getting Pod %v", err) +// } else { +// for _, p := range pods.Items { +// if p.Status.Phase != corev1.PodRunning { +// return false, nil +// } else if p.Status.PodIP == "" { +// return false, nil +// } else { +// podIPStrings.Insert(p.Status.PodIP) +// podNames = append(podNames, p.Name) +// } +// } +// return true, nil +// } +// }) +// if err != nil { +// return nil, nil, err +// } +// return podIPStrings.List(), podNames, nil +//} func parsePodIPs(podIPStrings sets.Set[string]) (*PodIPs, error) { ips := new(PodIPs) @@ -512,20 +520,19 @@ func (data *TestData) runPingCommandFromPod(namespace string, podName string, ta return nil } -func (data *TestData) runNetcatCommandFromPod(namespace string, podName string, containerName string, server string, port int) error { - cmd := []string{ - "/bin/sh", - "-c", - fmt.Sprintf("for i in $(seq 1 5); do nc -w 4 %s %d && exit 0 || sleep 1; done; exit 1", - server, port), - } - _, _, err := data.runCommandFromPod(namespace, podName, containerName, cmd) - if err != nil { - return err - } - return nil -} -*/ +//func (data *TestData) runNetcatCommandFromPod(namespace string, podName string, containerName string, server string, port int) error { +// cmd := []string{ +// "/bin/sh", +// "-c", +// fmt.Sprintf("for i in $(seq 1 5); do nc -w 4 %s %d && exit 0 || sleep 1; done; exit 1", +// server, port), +// } +// _, _, err := data.runCommandFromPod(namespace, podName, containerName, cmd) +// if err != nil { +// return err +// } +// return nil +//} func applyYAML(filename string, ns string) error { cmd := fmt.Sprintf("kubectl apply -f %s -n %s", filename, ns) @@ -549,30 +556,27 @@ func applyYAML(filename string, ns string) error { return nil } -// Temporarily disable traffic check -/* -func runCommand(cmd string) (string, error) { - err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (bool, error) { - var stdout, stderr bytes.Buffer - command := exec.Command("bash", "-c", cmd) - log.Info("Running command %s", cmd) - command.Stdout = &stdout - command.Stderr = &stderr - err := command.Run() - if err != nil { - log.Info("Error when running command %s: %v", cmd, err) - return false, nil - } - outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) - log.Info("Command %s returned with output: '%s' and error: '%s'", cmd, outStr, errStr) - if errStr != "" { - return false, nil - } - return true, nil - }) - return "", err -} -*/ +//func runCommand(cmd string) (string, error) { +// err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (bool, error) { +// var stdout, stderr bytes.Buffer +// command := exec.Command("bash", "-c", cmd) +// log.Info("Running command %s", cmd) +// command.Stdout = &stdout +// command.Stderr = &stderr +// err := command.Run() +// if err != nil { +// log.Info("Error when running command %s: %v", cmd, err) +// return false, nil +// } +// outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) +// log.Info("Command %s returned with output: '%s' and error: '%s'", cmd, outStr, errStr) +// if errStr != "" { +// return false, nil +// } +// return true, nil +// }) +// return "", err +//} func deleteYAML(filename string, ns string) error { cmd := fmt.Sprintf("kubectl delete -f %s -n %s", filename, ns) diff --git a/test/e2e/nsx_security_policy_test.go b/test/e2e/nsx_security_policy_test.go index 129b9718f..a9c8a7b19 100644 --- a/test/e2e/nsx_security_policy_test.go +++ b/test/e2e/nsx_security_policy_test.go @@ -51,28 +51,23 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { setupTest(t, ns) defer teardownTest(t, ns, defaultTimeout) - // Temporarily disable traffic check - /* - // Create pods - busyboxPath, _ := filepath.Abs("./manifest/testSecurityPolicy/busybox.yaml") - _ = applyYAML(busyboxPath, ns) - netcatPath, _ := filepath.Abs("./manifest/testSecurityPolicy/netcat-pod.yaml") - _ = applyYAML(netcatPath, ns) - - busybox := "busybox" - ncPod := "nc-pod - // Wait for pods - ps, err := testData.podWaitForIPs(defaultTimeout, busybox, ns) - t.Logf("Pods are %v", ps) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", busybox) - iPs, err := testData.podWaitForIPs(defaultTimeout, ncPod, ns) - t.Logf("Pods are %v", iPs) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", ncPod) - - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Create pods + busyboxPath, _ := filepath.Abs("./manifest/testSecurityPolicy/busybox.yaml") + _ = applyYAML(busyboxPath, ns) + netcatPath, _ := filepath.Abs("./manifest/testSecurityPolicy/netcat-pod.yaml") + _ = applyYAML(netcatPath, ns) + + busybox := "busybox" + ncPod := "nc-pod" + // Wait for pods + _, err = testData.podWaitForIPs(defaultTimeout, busybox, ns) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", busybox) + iPs, err := testData.podWaitForIPs(defaultTimeout, ncPod, ns) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", ncPod) + + // Ping from pod + err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) + assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) // Create security policy nsIsolationPath, _ := filepath.Abs("./manifest/testSecurityPolicy/ns-isolation-policy.yaml") require.NoError(t, applyYAML(nsIsolationPath, ns)) @@ -84,12 +79,9 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName0, true)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName1, true)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assertNotNil(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) + require.NoError(t, err, "Error when running ping command from test Pod %s", busybox) // Delete security policy _ = deleteYAML(nsIsolationPath, ns) @@ -111,12 +103,9 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName0, false)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName1, false)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) + assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) } // TestSecurityPolicyAddDeleteRule verifies that when adding or deleting rule, the security policy will be updated. @@ -194,28 +183,25 @@ func testSecurityPolicyMatchExpression(t *testing.T) { require.NoError(t, applyYAML(podPath, ns)) defer deleteYAML(podPath, "") - // Temporarily disable traffic check - // clientA := "client-a" - // clientB := "client-b" - // podA := "pod-a" - /* - // Wait for pods - ps, err := testData.podWaitForIPs(defaultTimeout, clientA, ns) - t.Logf("Pods are %v", ps) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientA) - psb, err := testData.podWaitForIPs(defaultTimeout, clientB, ns) - t.Logf("Pods are %v", psb) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientB) - iPs, err := testData.podWaitForIPs(defaultTimeout, podA, ns) - t.Logf("Pods are %v", iPs) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", podA) - - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) - */ + clientA := "client-a" + clientB := "client-b" + podA := "pod-a" + // Wait for pods + ps, err := testData.podWaitForIPs(defaultTimeout, clientA, ns) + log.Info("Pods are %v", ps) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientA) + psb, err := testData.podWaitForIPs(defaultTimeout, clientB, ns) + log.Info("Pods are %v", psb) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientB) + iPs, err := testData.podWaitForIPs(defaultTimeout, podA, ns) + log.Info("Pods are %v", iPs) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", podA) + + // Ping from pod + err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) + err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) // Create security policy nsIsolationPath, _ := filepath.Abs("./manifest/testSecurityPolicy/match-expression.yaml") @@ -227,14 +213,11 @@ func testSecurityPolicyMatchExpression(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeSecurityPolicy, securityPolicyName, true)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName, true)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NotNilf(t, err, "Error when running ping command from Pod %s", clientB) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) + err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) + assert.NotNilf(t, err, "Error when running ping command from Pod %s", clientB) // Delete security policy _ = deleteYAML(nsIsolationPath, ns) @@ -256,14 +239,11 @@ func testSecurityPolicyMatchExpression(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeSecurityPolicy, securityPolicyName, false)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName, false)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) - */ + // Ping from pod + err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) + err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) + assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) } // TestSecurityPolicyNamedPortWithoutPod verifies that the traffic of security policy when named port applied. From 88ad0eeeaeefcbd231d832ca2944656a486f8fe9 Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Tue, 26 Nov 2024 16:21:47 +0800 Subject: [PATCH 3/4] [e2e] Add test cases for pre-created VPC This change is to add a new end-to-end test for the pre-created VPC, it includes these steps, 1. Create a VPC/LBS/Attachment on NSX 2. Create a Namespace on vCenter or K8s, which depends on if the setup is a wcp testbed or not. Note, it planned to run the test on wcp testbed, please provide a valid pair of vc-user and vc-password which has the permission to create instance on vCenter 3. Create LoadBalancer typed Service inside the Namespace 4. Create server Pod for the LoadBalancer typed Service 5. Create client Pod in the same Namespace 6. Verified the traffic from the client Pod to the LoadBalancer's external IP 7. Delete LoadBalancer typed Service and verified that the realted NSX resources are removed as expected 8. Delete the Namespace from vCenter or K8s 9. Delete the VPC from NSX --- cover-e2e.out | 1 + test/e2e/framework.go | 135 ++++++++++- test/e2e/main_test.go | 2 +- test/e2e/nsxclient.go | 7 +- test/e2e/precreated_vpc_test.go | 413 ++++++++++++++++++++++++++++++++ test/e2e/vclient.go | 261 ++++++++++++++++++++ 6 files changed, 802 insertions(+), 17 deletions(-) create mode 100644 cover-e2e.out create mode 100644 test/e2e/precreated_vpc_test.go create mode 100644 test/e2e/vclient.go diff --git a/cover-e2e.out b/cover-e2e.out new file mode 100644 index 000000000..5f02b1119 --- /dev/null +++ b/cover-e2e.out @@ -0,0 +1 @@ +mode: set diff --git a/test/e2e/framework.go b/test/e2e/framework.go index a0f3a61d8..bac758a47 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -16,6 +16,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -23,10 +24,11 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" - - "github.com/vmware-tanzu/nsx-operator/pkg/logger" + "k8s.io/utils/ptr" "github.com/vmware-tanzu/nsx-operator/pkg/client/clientset/versioned" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" "github.com/vmware-tanzu/nsx-operator/test/e2e/providers" @@ -85,6 +87,7 @@ type TestData struct { clientset clientset.Interface crdClientset versioned.Interface nsxClient *NSXClient + vcClient *vcClient } var testData *TestData @@ -111,21 +114,29 @@ func initProvider() error { return nil } -func NewTestData(nsxConfig string) error { +func NewTestData(nsxConfig string, vcUser string, vcPassword string) error { testData = &TestData{} err := testData.createClients() if err != nil { return err } - err = testData.createNSXClients(nsxConfig) + config.UpdateConfigFilePath(nsxConfig) + cf, err := config.NewNSXOperatorConfigFromFile() + if err != nil { + return err + } + err = testData.createNSXClients(cf) if err != nil { return err } + if vcUser != "" && vcPassword != "" { + testData.vcClient = newVcClient(cf.VCEndPoint, cf.HttpsPort, vcUser, vcPassword) + } return nil } -func (data *TestData) createNSXClients(nsxConfig string) error { - nsxClient, err := NewNSXClient(nsxConfig) +func (data *TestData) createNSXClients(cf *config.NSXOperatorConfig) error { + nsxClient, err := NewNSXClient(cf) if err != nil { return err } @@ -267,12 +278,15 @@ func collectClusterInfo() error { } // createNamespace creates the provided namespace. -func (data *TestData) createNamespace(namespace string) error { +func (data *TestData) createNamespace(namespace string, mutators ...func(ns *corev1.Namespace)) error { ns := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, }, } + for _, mutator := range mutators { + mutator(&ns) + } if ns, err := data.clientset.CoreV1().Namespaces().Create(context.TODO(), &ns, metav1.CreateOptions{}); err != nil { // Ignore error if the namespace already exists if !errors.IsAlreadyExists(err) { @@ -372,6 +386,7 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin return pod.Status.Phase == corev1.PodRunning, nil }) if err != nil { + log.Error(err, "Failed to wait for Pod becoming RUNNING phase", "Pod", name) return nil, err } // According to the K8s API documentation (https://godoc.org/k8s.io/api/core/v1#PodStatus), @@ -389,6 +404,7 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin } ips, err := parsePodIPs(podIPStrings) if err != nil { + log.Error(err, "Failed to parse Pod's IP", "Pod", name) return nil, err } @@ -472,7 +488,7 @@ func parsePodIPs(podIPStrings sets.Set[string]) (*PodIPs, error) { // stdout and stderr as strings. An error either indicates that the command couldn't be run or that // the command returned a non-zero error code. func (data *TestData) runCommandFromPod(namespace string, podName string, containerName string, cmd []string) (stdout string, stderr string, err error) { - log.Info("Running '%s' in Pod '%s/%s' container '%s'", strings.Join(cmd, " "), namespace, podName, containerName) + log.Info("Running command in Pod's container", "Namespace", namespace, "Pod", podName, "Container", containerName, "Command", cmd) request := data.clientset.CoreV1().RESTClient().Post(). Namespace(namespace). Resource("pods"). @@ -495,11 +511,11 @@ func (data *TestData) runCommandFromPod(namespace string, podName string, contai Stdout: &stdoutB, Stderr: &stderrB, }); err != nil { - log.Info("Error when running command '%s' in Pod '%s/%s' container '%s': %v", strings.Join(cmd, " "), namespace, podName, containerName, err) + log.Error(err, "Failed to run command in Pod's container", "Namespace", namespace, "Pod", podName, "Container", containerName, "Command", cmd) return stdoutB.String(), stderrB.String(), err } outStr, errStr := stdoutB.String(), stderrB.String() - log.Info("Command '%s' in Pod '%s/%s' container '%s' returned with output: '%s' and error: '%s'", strings.Join(cmd, " "), namespace, podName, containerName, outStr, errStr) + log.Info("Successfully run command in Pod's container", "Namespace", namespace, "Pod", podName, "Container", containerName, "Command", cmd, "stdOut", outStr, "stdErr", errStr) return stdoutB.String(), stderrB.String(), nil } @@ -694,3 +710,102 @@ func (data *TestData) waitForResourceExistByPath(pathPolicy string, shouldExist }) return err } + +func (data *TestData) createService(namespace, serviceName string, port, targetPort int32, protocol corev1.Protocol, selector map[string]string, + serviceType corev1.ServiceType, mutators ...func(service *corev1.Service)) (*corev1.Service, error) { + ipFamilies := []corev1.IPFamily{corev1.IPv4Protocol} + + service := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: namespace, + Labels: map[string]string{ + "nsx-op-e2e": serviceName, + "app": serviceName, + }, + }, + Spec: corev1.ServiceSpec{ + SessionAffinity: corev1.ServiceAffinityNone, + Ports: []corev1.ServicePort{{ + Port: port, + TargetPort: intstr.FromInt32(targetPort), + Protocol: protocol, + }}, + Type: serviceType, + Selector: selector, + IPFamilies: ipFamilies, + }, + } + for _, mutator := range mutators { + mutator(&service) + } + return data.clientset.CoreV1().Services(namespace).Create(context.TODO(), &service, metav1.CreateOptions{}) +} + +func (data *TestData) createPod(namespace, podName, containerName, image string, protocol corev1.Protocol, containerPort int32, + mutators ...func(pod *corev1.Pod)) (*corev1.Pod, error) { + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + Annotations: map[string]string{}, + Labels: map[string]string{ + "nsx-op-e2e": podName, + "app": podName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: containerName, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: []corev1.ContainerPort{ + { + Protocol: protocol, + ContainerPort: containerPort, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + HostNetwork: false, + // Set it to 1s for immediate shutdown to reduce test run time and to avoid affecting subsequent tests. + TerminationGracePeriodSeconds: ptr.To[int64](1), + }, + } + for _, mutator := range mutators { + mutator(&pod) + } + return data.clientset.CoreV1().Pods(namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) +} + +func (data *TestData) serviceWaitFor(readyTime time.Duration, namespace string, name string, conditionFunc func(svc *corev1.Service) (bool, error)) (*corev1.Service, error) { + err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, readyTime, false, func(ctx context.Context) (bool, error) { + if svc, err := data.clientset.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}); err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("error when getting Service '%s/%s': %v", namespace, name, err) + } else { + return conditionFunc(svc) + } + }) + if err != nil { + return nil, err + } + return data.clientset.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) +} + +func (data *TestData) deleteService(nsName string, svcName string) error { + ctx := context.TODO() + err := data.clientset.CoreV1().Services(nsName).Delete(ctx, svcName, metav1.DeleteOptions{}) + if err != nil { + log.Error(err, "Failed to delete Service", "namespace", nsName, "name", svcName) + } + return err +} + +func (data *TestData) useWCPSetup() bool { + return data.vcClient != nil +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 5fe2fcf83..fd900c5cc 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -39,7 +39,7 @@ func testMain(m *testing.M) int { log.Info("Creating clientSets") - if err := NewTestData(testOptions.operatorConfigPath); err != nil { + if err := NewTestData(testOptions.operatorConfigPath, testOptions.vcUser, testOptions.vcPassword); err != nil { log.Error(err, "Error when creating client") return 1 } diff --git a/test/e2e/nsxclient.go b/test/e2e/nsxclient.go index d96592bbe..31d92bfc1 100644 --- a/test/e2e/nsxclient.go +++ b/test/e2e/nsxclient.go @@ -14,13 +14,8 @@ type NSXClient struct { *nsx.Client } -func NewNSXClient(configFile string) (*NSXClient, error) { +func NewNSXClient(cf *config.NSXOperatorConfig) (*NSXClient, error) { // nsxClient is used to interact with NSX API. - config.UpdateConfigFilePath(configFile) - cf, err := config.NewNSXOperatorConfigFromFile() - if err != nil { - return nil, err - } client := nsx.GetClient(cf) if client == nil { return nil, fmt.Errorf("failed to get nsx client") diff --git a/test/e2e/precreated_vpc_test.go b/test/e2e/precreated_vpc_test.go new file mode 100644 index 000000000..4d9a8a0f7 --- /dev/null +++ b/test/e2e/precreated_vpc_test.go @@ -0,0 +1,413 @@ +package e2e + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "regexp" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" + pkgutil "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +const ( + preVPCLabelKey = "prevpc-test" + podImage = "nsx-ujo-docker-local.artifactory.eng.vmware.com/nsx-ujo/http_https_echo" + containerName = "netexec-container" + + lbServicePort = int32(8080) + podPort = int32(80) + resourceReadyTime = 2 * time.Minute + nsDeleteTime = 2 * time.Minute + + nsPrivilegedLabel = "pod-security.kubernetes.io/enforce" +) + +var ( + projectPathFormat = "/orgs/%s/projects/%s" + vpcPathFormat = "/orgs/%s/projects/%s/vpcs/%s" + defaultConnectivityProfileFormat = "/orgs/%s/projects/%s/vpc-connectivity-profiles/default" +) + +func TestPreCreatedVPC(t *testing.T) { + orgID, projectID, vpcID := setupVPC(t) + nsName := "test-prevpc" + projectPath := fmt.Sprintf(projectPathFormat, orgID, projectID) + profilePath := fmt.Sprintf(defaultConnectivityProfileFormat, orgID, projectID) + preCreatedVPCPath := fmt.Sprintf(vpcPathFormat, orgID, projectID, vpcID) + log.Info("Created VPC on NSX", "path", preCreatedVPCPath) + defer func() { + log.Info("Deleting the created VPC from NSX", "path", preCreatedVPCPath) + ctx := context.Background() + if pollErr := wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (done bool, err error) { + if err := testData.nsxClient.VPCClient.Delete(orgID, projectID, vpcID, common.Bool(true)); err != nil { + return false, nil + } + log.Info("The pre-created VPC is successfully deleted", "path", preCreatedVPCPath) + return true, nil + }); pollErr != nil { + log.Error(pollErr, "Failed to delete the pre-created VPC within 5m after the test", "path", preCreatedVPCPath) + } + }() + + // Test: create NetworkConfig and NS using the pre-created VPC + useVCAPI := testData.useWCPSetup() + if useVCAPI { + err := testData.vcClient.startSession() + require.NoError(t, err, "A new VC session should be created for test") + defer func() { + testData.vcClient.closeSession() + }() + } + + err := createVPCNamespace(nsName, projectPath, profilePath, preCreatedVPCPath, nil, useVCAPI) + require.NoError(t, err, "VPCNetworkConfiguration and Namespace should be created") + log.Info("Created test Namespace", "Namespace", nsName) + + defer func() { + deleteVPCNamespace(nsName, useVCAPI) + _, err = testData.nsxClient.VPCClient.Get(orgID, projectID, vpcID) + require.NoError(t, err, "Pre-Created VPC should exist after the K8s Namespace is deleted") + }() + // Wait until the created NetworkInfo is ready. + getNetworkInfoWithPrivateIPs(t, nsName, nsName) + log.Info("New Namespace's networkInfo is ready", "Namespace", nsName) + + // Test create LB Service inside the NS + podName := "prevpc-service-pod" + svcName := "prevpc-loadbalancer" + err = createLBService(nsName, svcName, podName) + require.NoError(t, err, "K8s LoadBalancer typed Service should be created") + log.Info("Created LoadBalancer Service in the Namespace", "Namespace", nsName, "Service", svcName) + + // Wait until Pod has allocated IP + _, err = testData.podWaitForIPs(resourceReadyTime, podName, nsName) + require.NoErrorf(t, err, "Pod '%s/%s' is not ready within time %s", nsName, podName, resourceReadyTime.String()) + log.Info("Server Pod for the LoadBalancer Service in the Namespace is ready", "Namespace", nsName, "Service", svcName, "Pod", podName) + + // Wait until LoadBalancer Service has external IP. + svc, err := testData.serviceWaitFor(resourceReadyTime, nsName, svcName, func(svc *corev1.Service) (bool, error) { + lbStatuses := svc.Status.LoadBalancer.Ingress + if len(lbStatuses) == 0 { + return false, nil + } + lbStatus := lbStatuses[0] + if lbStatus.IP == "" { + return false, nil + } + return true, nil + }) + require.NoErrorf(t, err, "K8s LoadBalancer typed Service should get an external IP within time %s", resourceReadyTime) + svcUID := svc.UID + lbIP := svc.Status.LoadBalancer.Ingress[0].IP + log.Info("Load Balancer Service has been assigned with external IP", "Namespace", nsName, "Service", svcName, "ExternalIP", lbIP) + + // Create client Pod inside the NS + clientPodName := "prevpc-client-pod" + _, err = testData.createPod(nsName, clientPodName, containerName, podImage, corev1.ProtocolTCP, podPort) + require.NoErrorf(t, err, "Client Pod '%s/%s' should be created", nsName, clientPodName) + _, err = testData.podWaitForIPs(resourceReadyTime, clientPodName, nsName) + require.NoErrorf(t, err, "Client Pod '%s/%s' is not ready within time %s", nsName, clientPodName, resourceReadyTime.String()) + log.Info("Client Pod in the Namespace is ready", "Namespace", nsName, "Service", svcName, "Pod", clientPodName) + + // Test traffic from client Pod to LB Service + url := fmt.Sprintf("http://%s:%d", lbIP, lbServicePort) + cmd := []string{ + "curl", "-s", "-o", `/dev/null`, "-w", `"%{http_code}"`, url, + } + stdOut, _, err := testData.runCommandFromPod(nsName, clientPodName, containerName, cmd) + require.NoError(t, err, "LoadBalancer traffic should work") + statusCode := strings.Trim(stdOut, `"`) + require.NoError(t, err) + assert.Equal(t, "200", statusCode) + log.Info("Verified traffic from client Pod to the LoadBalancer Service") + + // Test NSX LB VS should be removed after K8s LB Service is deleted + err = testData.deleteService(nsName, svcName) + require.NoError(t, err, "Service should be deleted") + log.Info("Deleted the LoadBalancer Service") + err = testData.waitForLBVSDeletion(resourceReadyTime, string(svcUID)) + require.NoErrorf(t, err, "NSX resources should be removed after K8s LoadBalancer Service is deleted") + log.Info("NSX resources for the LoadBalancer Service are removed") +} + +func deleteVPCNamespace(nsName string, usingVCAPI bool) { + if usingVCAPI { + if err := testData.vcClient.deleteNamespace(nsName); err != nil { + log.Error(err, "Failed to delete Namespace on VCenter", "namespace", nsName) + } + return + } + + vpcConfigName := fmt.Sprintf("%s-vpcconfig-%s", nsName, getRandomString()) + deleteVPCNamespaceOnK8s(nsName, vpcConfigName) +} + +func deleteVPCNamespaceOnK8s(nsName string, vpcConfigName string) { + ctx := context.Background() + if err := testData.deleteNamespace(nsName, nsDeleteTime); err != nil { + log.Error(err, "Failed to delete VPCNetworkConfiguration", "name", vpcConfigName) + } + if err := testData.crdClientset.CrdV1alpha1().VPCNetworkConfigurations().Delete(ctx, vpcConfigName, metav1.DeleteOptions{}); err != nil { + log.Error(err, "Failed to delete VPCNetworkConfiguration %s", "name", vpcConfigName) + } +} + +func setupVPC(tb testing.TB) (string, string, string) { + systemVPC, err := testData.waitForSystemNetworkConfigReady(5 * time.Minute) + require.NoError(tb, err) + + nsxProjectPath := systemVPC.Spec.NSXProject + reExp := regexp.MustCompile(`/orgs/([^/]+)/projects/([^/]+)([/\S+]*)`) + matches := reExp.FindStringSubmatch(nsxProjectPath) + orgID, projectID := matches[1], matches[2] + systemVPCStatus := systemVPC.Status.VPCs[0] + useNSXLB := systemVPCStatus.NSXLoadBalancerPath != "" + + vpcID := fmt.Sprintf("testvpc-%s", getRandomString()) + if err := testData.createVPC(orgID, projectID, vpcID, []string{customizedPrivateCIDR1}, useNSXLB); err != nil { + tb.Fatalf("Unable to create a VPC on NSX: %v", err) + } + return orgID, projectID, vpcID +} + +func createVPCNamespace(nsName, projectPath, profilePath, vpcPath string, privateIPs []string, usingVCAPI bool) error { + if usingVCAPI { + return testData.createPreVPCNamespaceByVCenter(nsName, vpcPath) + } + + vpcConfigName := fmt.Sprintf("%s-vpcconfig-%s", nsName, getRandomString()) + return createVPCNamespaceOnK8s(nsName, vpcConfigName, projectPath, profilePath, vpcPath, privateIPs) +} + +func createVPCNamespaceOnK8s(nsName, vpcConfigName, projectPath, profilePath, vpcPath string, privateIPs []string) error { + ctx := context.Background() + vpcNetConfig := &v1alpha1.VPCNetworkConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: vpcConfigName, + }, + Spec: v1alpha1.VPCNetworkConfigurationSpec{ + NSXProject: projectPath, + VPCConnectivityProfile: profilePath, + VPC: vpcPath, + PrivateIPs: privateIPs, + }, + } + + if _, err := testData.crdClientset.CrdV1alpha1().VPCNetworkConfigurations().Create(ctx, vpcNetConfig, metav1.CreateOptions{}); err != nil { + log.Error(err, "Failed to create VPCNetworkConfiguration", "name", vpcNetConfig) + return err + } + + if err := testData.createNamespace(nsName, func(ns *corev1.Namespace) { + if ns.Annotations == nil { + ns.Annotations = map[string]string{} + } + ns.Annotations[common.AnnotationVPCNetworkConfig] = vpcConfigName + if ns.Labels == nil { + ns.Labels = map[string]string{} + } + ns.Labels[nsPrivilegedLabel] = "privileged" + }); err != nil { + // Clean up the created VPCNetworkConfiguration. + log.Error(err, "Failed to create Namespace", "name", nsName) + if delErr := testData.crdClientset.CrdV1alpha1().VPCNetworkConfigurations().Delete(ctx, vpcConfigName, metav1.DeleteOptions{}); delErr != nil { + log.Error(err, "Failed to delete VPCNetworkConfiguration", "name", vpcConfigName) + } + return err + } + return nil +} + +func createLBService(nsName, svcName, podName string) error { + podLabels := map[string]string{ + preVPCLabelKey: svcName, + } + if _, err := testData.createPod(nsName, podName, containerName, podImage, corev1.ProtocolTCP, podPort, func(pod *corev1.Pod) { + for k, v := range podLabels { + pod.Labels[k] = v + } + }); err != nil { + log.Error(err, "Failed to create Pod", "namespace", nsName, "name", podName) + return err + } + if _, err := testData.createService(nsName, svcName, lbServicePort, podPort, corev1.ProtocolTCP, podLabels, corev1.ServiceTypeLoadBalancer); err != nil { + log.Error(err, "Failed to create LoadBalancer Service", "namespace", nsName, "name", svcName) + return err + } + return nil +} + +func getRandomString() string { + timestamp := time.Now().UnixNano() + hash := sha256.Sum256([]byte(fmt.Sprintf("%d", timestamp))) + return hex.EncodeToString(hash[:])[:8] +} + +func (data *TestData) createVPC(orgID, projectID, vpcID string, privateIPs []string, useNSXLB bool) error { + createdVPC := &model.Vpc{ + Id: common.String(vpcID), + DisplayName: common.String("e2e-test-pre-vpc"), + IpAddressType: common.String("IPV4"), + PrivateIps: privateIPs, + ResourceType: common.String(common.ResourceTypeVpc), + } + vpcPath := fmt.Sprintf(vpcPathFormat, orgID, projectID, vpcID) + var lbsPath string + var createdLBS *model.LBService + if !useNSXLB { + loadBalancerVPCEndpointEnabled := true + createdVPC.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled} + } else { + lbsPath = fmt.Sprintf("%s/vpc-lbs/default", vpcPath) + createdLBS = &model.LBService{ + Id: common.String("default"), + ConnectivityPath: common.String(vpcPath), + Size: common.String(model.LBService_SIZE_SMALL), + ResourceType: common.String(common.ResourceTypeLBService), + } + } + attachmentPath := fmt.Sprintf("%s/attachments/default", vpcPath) + attachment := &model.VpcAttachment{ + Id: common.String("default"), + VpcConnectivityProfile: common.String(fmt.Sprintf(defaultConnectivityProfileFormat, orgID, projectID)), + } + svc := &vpc.VPCService{} + orgRoot, err := svc.WrapHierarchyVPC(orgID, projectID, createdVPC, createdLBS, attachment) + if err != nil { + log.Error(err, "Failed to build HAPI request for VPC related resources") + return err + } + enforceRevisionCheckParam := false + if err := data.nsxClient.OrgRootClient.Patch(*orgRoot, &enforceRevisionCheckParam); err != nil { + return err + } + + log.Info("Successfully requested VPC on NSX", "path", vpcPath) + realizeService := realizestate.InitializeRealizeState(common.Service{NSXClient: data.nsxClient.Client}) + if pollErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (done bool, err error) { + if err = realizeService.CheckRealizeState(pkgutil.NSXTDefaultRetry, vpcPath); err != nil { + log.Error(err, "NSX VPC is not yet realized", "path", vpcPath) + return false, nil + } + if lbsPath != "" { + if err := realizeService.CheckRealizeState(pkgutil.NSXTLBVSDefaultRetry, lbsPath); err != nil { + log.Error(err, "NSX LBS is not yet realized", "path", lbsPath) + return false, nil + } + } + if err = realizeService.CheckRealizeState(pkgutil.NSXTDefaultRetry, attachmentPath); err != nil { + log.Error(err, "VPC attachment is not yet realized", "path", attachmentPath) + return false, nil + } + return true, nil + }); pollErr != nil { + log.Error(pollErr, "Failed to realize VPC and related resources within 2m") + data.nsxClient.VPCClient.Delete(orgID, projectID, vpcID, common.Bool(true)) + if err := data.nsxClient.VPCClient.Delete(orgID, projectID, vpcID, common.Bool(true)); err != nil { + log.Error(err, "Failed to recursively delete NSX VPC", "path", fmt.Sprintf("/orgs/%s/projects/%s/vpcs/%s", orgID, projectID, vpcID)) + } + return pollErr + } + return nil +} + +func (data *TestData) waitForSystemNetworkConfigReady(timeout time.Duration) (*v1alpha1.VPCNetworkConfiguration, error) { + var systemConfig *v1alpha1.VPCNetworkConfiguration + if pollErr := wait.PollUntilContextTimeout(context.Background(), 20*time.Second, timeout, true, func(ctx context.Context) (done bool, err error) { + systemConfig, err = data.crdClientset.CrdV1alpha1().VPCNetworkConfigurations().Get(ctx, "system", metav1.GetOptions{}) + if err != nil { + return false, nil + } + if len(systemConfig.Status.VPCs) == 0 { + return false, nil + } + systemVPC := systemConfig.Status.VPCs[0] + if systemVPC.VPCPath == "" { + return false, nil + } + return true, nil + }); pollErr != nil { + log.Error(pollErr, "Failed to wait for system VPCNetworkConfiguration to be ready", "timeout", timeout.String()) + return nil, pollErr + } + return systemConfig, nil +} + +func (data *TestData) waitForLBVSDeletion(timeout time.Duration, svcID string) error { + lbServiceTags := []string{"ncp/service_uid", svcID} + if pollErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, timeout, true, func(ctx context.Context) (done bool, err error) { + // Check NSX VirtualServer deletion. + vsResults, err := data.queryResource("LBVirtualServer", lbServiceTags) + if err != nil { + return false, err + } + if len(vsResults.Results) > 0 { + return false, nil + } + // Check NSX LBPool deletion. + lbPoolResults, err := data.queryResource("LBPool", lbServiceTags) + if err != nil { + return false, err + } + if len(lbPoolResults.Results) > 0 { + return false, nil + } + // Check NSX IP Allocation deletion. + ipAllocationResults, err := data.queryResource("VpcIpAddressAllocation", lbServiceTags) + if err != nil { + return false, err + } + if len(ipAllocationResults.Results) > 0 { + return false, nil + } + return true, nil + }); pollErr != nil { + log.Error(pollErr, "Failed to delete LoadBalancer Service related resources on NSX") + return pollErr + } + return nil +} + +func (data *TestData) createPreVPCNamespaceByVCenter(nsName, vpcPath string) error { + svID, err := data.vcClient.getSupervisorID() + if err != nil { + return fmt.Errorf("failed to get a valid supervisor ID: %v", err) + } + err = data.vcClient.createNamespaceWithPreCreatedVPC(nsName, vpcPath, svID) + if err != nil { + return fmt.Errorf("failed to create Namespace on VCenter: %v", err) + } + ctx := context.Background() + getErr := wait.PollUntilContextTimeout(ctx, 2*time.Second, 10*time.Second, false, func(ctx context.Context) (done bool, err error) { + _, statusCode, err := data.vcClient.getNamespaceInfoByName(nsName) + if statusCode == http.StatusNotFound { + return false, nil + } + if err != nil { + return true, err + } + return true, nil + }) + if getErr != nil { + data.vcClient.deleteNamespace(nsName) + return fmt.Errorf("failed to create Namespace on VCenter, delete it: %v", err) + } + + return nil +} diff --git a/test/e2e/vclient.go b/test/e2e/vclient.go new file mode 100644 index 000000000..82f15bc0e --- /dev/null +++ b/test/e2e/vclient.go @@ -0,0 +1,261 @@ +package e2e + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sync" + "time" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" +) + +type vcClient struct { + url *url.URL + httpClient *http.Client + sessionMutex sync.Mutex + sessionKey string +} + +type supervisorInfo struct { + Name string `json:"name"` + ConfigStatus string `json:"config_status"` + K8sStatus string `json:"kubernetes_status"` +} + +type supervisorSummary struct { + ID string `json:"supervisor"` + Info supervisorInfo `json:"info"` +} + +type InstancesIpv4Cidr struct { + Address string `json:"address"` + Prefix int64 `json:"prefix"` +} + +type InstancesVpcConfig struct { + PrivateCidrs []InstancesIpv4Cidr `json:"private_cidrs"` +} + +type InstancesVpcNetworkInfo struct { + VpcConfig InstancesVpcConfig `json:"vpc_config,omitempty"` + Vpc string `json:"vpc,omitempty"` + DefaultSubnetSize int64 `json:"default_subnet_size"` +} + +type InstancesNetworkConfigInfo struct { + NetworkProvider string `json:"network_provider"` + VpcNetwork InstancesVpcNetworkInfo `json:"vpc_network"` +} + +type VCNamespaceCreateSpec struct { + Supervisor string `json:"supervisor"` + Namespace string `json:"namespace"` + NetworkSpec InstancesNetworkConfigInfo `json:"network_spec"` +} + +type VCNamespaceGetInfo struct { + Supervisor string `json:"supervisor"` + NetworkSpec InstancesNetworkConfigInfo `json:"network_spec"` +} + +var ( + sessionURLPath = "/api/session" +) + +func newVcClient(hostname string, port int, userName, password string) *vcClient { + httpClient := createHttpClient() + baseurl := fmt.Sprintf("https://%s:%d/", hostname, port) + vcurl, _ := url.Parse(baseurl) + + vcurl.User = url.UserPassword(userName, password) + return &vcClient{ + url: vcurl, + httpClient: httpClient, + } +} + +func createHttpClient() *http.Client { + tlsConfig := &tls.Config{InsecureSkipVerify: true} // #nosec G402: ignore insecure options + transport := &http.Transport{ + TLSClientConfig: tlsConfig, + } + return &http.Client{Transport: transport, Timeout: time.Minute} +} + +func (c *vcClient) startSession() error { + c.sessionMutex.Lock() + defer c.sessionMutex.Unlock() + if c.sessionKey == "" { + url := fmt.Sprintf("%s://%s%s", c.url.Scheme, c.url.Host, sessionURLPath) + request, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return err + } + username := c.url.User.Username() + password, _ := c.url.User.Password() + request.SetBasicAuth(username, password) + + var sessionData string + if _, err = c.handleRequest(request, &sessionData); err != nil { + return err + } + + c.sessionKey = sessionData + } + return nil +} + +func (c *vcClient) closeSession() error { + c.sessionMutex.Lock() + defer c.sessionMutex.Unlock() + if c.sessionKey == "" { + return nil + } + request, err := c.prepareRequest(http.MethodDelete, sessionURLPath, nil) + if err != nil { + return err + } + + if _, err = c.handleRequest(request, nil); err != nil { + return err + } + + c.sessionKey = "" + return nil +} + +func (c *vcClient) getSupervisorID() (string, error) { + urlPath := "/api/vcenter/namespace-management/supervisors/summaries" + request, err := c.prepareRequest(http.MethodGet, urlPath, nil) + if err != nil { + return "", err + } + var response struct { + Items []supervisorSummary `json:"items"` + } + if _, err = c.handleRequest(request, &response); err != nil { + return "", err + } + + for _, sv := range response.Items { + if sv.Info.ConfigStatus == "RUNNING" { + return sv.ID, nil + } + } + return "", fmt.Errorf("no valid supervisor found on vCenter") +} + +func (c *vcClient) createNamespaceWithPreCreatedVPC(namespace string, vpcPath string, supervisorID string) error { + urlPath := "/api/vcenter/namespaces/instances/v2" + vcNamespace := createVCNamespaceSpec(namespace, supervisorID, vpcPath) + data, err := json.Marshal(vcNamespace) + if err != nil { + return fmt.Errorf("unable convert vcNamespace object to json bytes: %v", err) + } + request, err := c.prepareRequest(http.MethodPost, urlPath, data) + if err != nil { + return fmt.Errorf("failed to parepare http request with vcNamespace data: %v", err) + } + if _, err = c.handleRequest(request, nil); err != nil { + return err + } + return nil +} + +func (c *vcClient) getNamespaceInfoByName(namespace string) (*VCNamespaceGetInfo, int, error) { + urlPath := fmt.Sprintf("/api/vcenter/namespaces/instances/v2/%s", namespace) + request, err := c.prepareRequest(http.MethodGet, urlPath, nil) + if err != nil { + return nil, 0, fmt.Errorf("failed to prepare http request with vcNamespace get: %v", err) + } + result := &VCNamespaceGetInfo{} + statusCode, err := c.handleRequest(request, result) + if err != nil { + return nil, statusCode, err + } + return result, statusCode, nil +} + +func (c *vcClient) deleteNamespace(namespace string) error { + urlPath := fmt.Sprintf("/api/vcenter/namespaces/instances/%s", namespace) + request, err := c.prepareRequest(http.MethodDelete, urlPath, nil) + if err != nil { + return fmt.Errorf("failed to parepare http request with vcNamespace deletion: %v", err) + } + if _, err = c.handleRequest(request, nil); err != nil { + return err + } + return nil +} + +func createVCNamespaceSpec(namespace string, svID string, vpcPath string) *VCNamespaceCreateSpec { + return &VCNamespaceCreateSpec{ + Supervisor: svID, + Namespace: namespace, + NetworkSpec: InstancesNetworkConfigInfo{ + NetworkProvider: "NSX_VPC", + VpcNetwork: InstancesVpcNetworkInfo{ + Vpc: vpcPath, + DefaultSubnetSize: 16, + }, + }, + } +} + +func (c *vcClient) prepareRequest(method string, urlPath string, data []byte) (*http.Request, error) { + url := fmt.Sprintf("%s://%s%s", c.url.Scheme, c.url.Host, urlPath) + req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("vmware-api-session-id", c.sessionKey) + return req, nil +} + +func (c *vcClient) handleRequest(request *http.Request, responseData interface{}) (int, error) { + response, err := c.httpClient.Do(request) + if err != nil { + log.Error(err, "Failed to do HTTP request") + return 0, err + } + return handleHTTPResponse(response, responseData) +} + +func handleHTTPResponse(response *http.Response, result interface{}) (int, error) { + statusCode := response.StatusCode + if statusCode == http.StatusNoContent { + return statusCode, nil + } + + if statusCode >= http.StatusOK && statusCode < http.StatusMultipleChoices { + if result == nil { + return statusCode, nil + } + body, err := io.ReadAll(response.Body) + defer response.Body.Close() + + if err != nil { + return statusCode, err + } + if err = json.Unmarshal(body, result); err != nil { + return statusCode, err + } + return statusCode, nil + } + + var err error + if statusCode == http.StatusNotFound { + err = util.HttpNotFoundError + } else if statusCode == http.StatusBadRequest { + err = util.HttpBadRequest + } else { + err = fmt.Errorf("HTTP response with errorcode %d", response.StatusCode) + } + return statusCode, err +} From 7342ec8e90e6bf982e9545791a9cdaaeda7a92f5 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Mon, 16 Dec 2024 17:26:13 +0800 Subject: [PATCH 4/4] Create namespace on VC side for security policy e2e test --- test/e2e/framework.go | 59 ++++++++++++++++++++++++++++ test/e2e/nsx_security_policy_test.go | 4 +- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index bac758a47..de592e801 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -3,8 +3,10 @@ package e2e import ( "bytes" "context" + "encoding/json" "fmt" "net" + "net/http" "net/url" "os/exec" "regexp" @@ -300,6 +302,63 @@ func (data *TestData) createNamespace(namespace string, mutators ...func(ns *cor return nil } +// createVCNamespace creates a VC namespace with the provided namespace. +func (data *TestData) createVCNamespace(namespace string) error { + svID, err := data.vcClient.getSupervisorID() + vcNamespace := &VCNamespaceCreateSpec{ + Supervisor: svID, + Namespace: namespace, + } + dataJson, err := json.Marshal(vcNamespace) + if err != nil { + return fmt.Errorf("unable convert vcNamespace object to json bytes: %v", err) + } + urlPath := "/api/vcenter/namespaces/instances/v2" + request, err := data.vcClient.prepareRequest(http.MethodPost, urlPath, dataJson) + if err != nil { + return fmt.Errorf("failed to parepare http request with vcNamespace data: %v", err) + } + if _, err = data.vcClient.handleRequest(request, nil); err != nil { + return err + } + // wait for the namespace on k8s running + err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (done bool, err error) { + ns, err := data.clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + if err != nil { + return false, err + } + if ns.Status.Phase == corev1.NamespaceActive { + return true, nil + } + return false, nil + }) + return err +} + +// deleteVCNamespace deletes the provided VC namespace and waits for deletion to actually complete. +func (data *TestData) deleteVCNamespace(namespace string) error { + urlPath := "/api/vcenter/namespaces/instances/v2/" + namespace + request, err := data.vcClient.prepareRequest(http.MethodDelete, urlPath, nil) + if err != nil { + return fmt.Errorf("failed to parepare http request with vcNamespace deletion: %v", err) + } + if _, err = data.vcClient.handleRequest(request, nil); err != nil { + return err + } + // wait for the namespace on k8s terminating + err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (done bool, err error) { + ns, err := data.clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + if err != nil { + return false, err + } + if ns.Status.Phase == corev1.NamespaceTerminating { + return true, nil + } + return false, nil + }) + return err +} + // deleteNamespace deletes the provided namespace and waits for deletion to actually complete. func (data *TestData) deleteNamespace(namespace string, timeout time.Duration) error { var gracePeriodSeconds int64 diff --git a/test/e2e/nsx_security_policy_test.go b/test/e2e/nsx_security_policy_test.go index a9c8a7b19..da56c020c 100644 --- a/test/e2e/nsx_security_policy_test.go +++ b/test/e2e/nsx_security_policy_test.go @@ -48,8 +48,8 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { ruleName0 := "all_ingress_isolation" ruleName1 := "all_egress_isolation" var err error - setupTest(t, ns) - defer teardownTest(t, ns, defaultTimeout) + testData.createVCNamespace(ns) + defer testData.deleteVCNamespace(ns) // Create pods busyboxPath, _ := filepath.Abs("./manifest/testSecurityPolicy/busybox.yaml")