From 0a436b78b32432150edd082ee37e078e127fe896 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Tue, 30 Apr 2024 02:45:15 +0400 Subject: [PATCH 1/5] systest: add QUIC support This adds a job that runs systests using QUIC instead of TCP --- .github/workflows/systest.yml | 7 +++++ systest/Makefile | 2 ++ systest/cluster/discover.go | 1 + systest/cluster/nodes.go | 49 +++++++++++++++++++++++++++------- systest/testcontext/context.go | 5 ++++ 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/.github/workflows/systest.yml b/.github/workflows/systest.yml index 125540740d..79ee32ea63 100644 --- a/.github/workflows/systest.yml +++ b/.github/workflows/systest.yml @@ -54,6 +54,12 @@ jobs: systest: runs-on: ubuntu-latest + strategy: + # fail-fast: true + matrix: + protocol: + - tcp + - quic if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }} needs: - filter-changes @@ -127,6 +133,7 @@ jobs: level: ${{ inputs.log_level }} clusters: 4 norbac: 1 + quic: ${{ matrix.protocol == 'tcp' && 'false' || 'true' }} run: make -C systest run test_name=${{ inputs.test_name }} - name: Delete pod diff --git a/systest/Makefile b/systest/Makefile index 2ad1a9e641..df0818d02f 100644 --- a/systest/Makefile +++ b/systest/Makefile @@ -23,6 +23,7 @@ node_selector ?= namespace ?= label ?= count ?= 1 +quic ?= false configname ?= $(test_job_name) smesher_config ?= parameters/fastnet/smesher.json @@ -63,6 +64,7 @@ template: @echo post-init-image=$(post_init_image) >> $(tmpfile) @echo keep=$(keep) >> $(tmpfile) @echo testid=$(test_id) >> $(tmpfile) + @echo quic=$(quic) >> $(tmpfile) .PHONY: config config: template diff --git a/systest/cluster/discover.go b/systest/cluster/discover.go index 965f7008dd..03f8541916 100644 --- a/systest/cluster/discover.go +++ b/systest/cluster/discover.go @@ -26,6 +26,7 @@ func discoverNodes(ctx *testcontext.Context, kind string) ([]*NodeClient, error) P2P: 7513, GRPC_PUB: 9092, GRPC_PRIV: 9093, + QUIC: ctx.QUIC, }, }) } diff --git a/systest/cluster/nodes.go b/systest/cluster/nodes.go index 813e3a5ead..0c2845316e 100644 --- a/systest/cluster/nodes.go +++ b/systest/cluster/nodes.go @@ -129,11 +129,16 @@ const ( type Node struct { Name string P2P, GRPC_PUB, GRPC_PRIV uint16 + QUIC bool } // P2PEndpoint returns full p2p endpoint, including identity. func p2pEndpoint(n Node, ip, id string) string { - return fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", ip, n.P2P, id) + if n.QUIC { + return fmt.Sprintf("/ip4/%s/udp/%d/quic-v1/p2p/%s", ip, n.P2P, id) + } else { + return fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", ip, n.P2P, id) + } } // NodeClient is a Node with attached grpc connection. @@ -368,14 +373,21 @@ func deployPoetD(ctx *testcontext.Context, id string, flags ...DeploymentFlag) ( func deployBootnodeSvc(ctx *testcontext.Context, id string) error { labels := nodeLabels(bootnodeApp, id) + p2pProto := apiv1.ProtocolTCP + if ctx.QUIC { + p2pProto = apiv1.ProtocolUDP + } svc := corev1.Service(id, ctx.Namespace). WithLabels(labels). WithSpec(corev1.ServiceSpec(). WithSelector(labels). WithPorts( - corev1.ServicePort().WithName("grpc-pub").WithPort(9092).WithProtocol("TCP"), - corev1.ServicePort().WithName("grpc-priv").WithPort(9093).WithProtocol("TCP"), - corev1.ServicePort().WithName("p2p").WithPort(7513).WithProtocol("TCP"), + corev1.ServicePort().WithName("grpc-pub").WithPort(9092). + WithProtocol(apiv1.ProtocolTCP), + corev1.ServicePort().WithName("grpc-priv").WithPort(9093). + WithProtocol(apiv1.ProtocolTCP), + corev1.ServicePort().WithName("p2p").WithPort(7513). + WithProtocol(p2pProto), ). WithClusterIP("None"), ) @@ -388,15 +400,23 @@ func deployBootnodeSvc(ctx *testcontext.Context, id string) error { func deployNodeSvc(ctx *testcontext.Context, id string) error { labels := nodeLabels(smesherApp, id) + p2pProto := apiv1.ProtocolTCP + if ctx.QUIC { + p2pProto = apiv1.ProtocolUDP + } svc := corev1.Service(id, ctx.Namespace). WithLabels(labels). WithSpec(corev1.ServiceSpec(). WithSelector(labels). WithPorts( - corev1.ServicePort().WithName("p2p").WithPort(7513).WithProtocol("TCP"), - corev1.ServicePort().WithName("grpc-pub").WithPort(9092).WithProtocol("TCP"), - corev1.ServicePort().WithName("grpc-priv").WithPort(9093).WithProtocol("TCP"), - corev1.ServicePort().WithName("grpc-post").WithPort(9094).WithProtocol("TCP"), + corev1.ServicePort().WithName("p2p").WithPort(7513). + WithProtocol(p2pProto), + corev1.ServicePort().WithName("grpc-pub").WithPort(9092). + WithProtocol(apiv1.ProtocolTCP), + corev1.ServicePort().WithName("grpc-priv").WithPort(9093). + WithProtocol(apiv1.ProtocolTCP), + corev1.ServicePort().WithName("grpc-post").WithPort(9094). + WithProtocol(apiv1.ProtocolTCP), ). WithClusterIP("None"), ) @@ -581,6 +601,7 @@ func deployNodes(ctx *testcontext.Context, kind string, from, to int, opts ...De P2P: 7513, GRPC_PUB: 9092, GRPC_PRIV: 9093, + QUIC: ctx.QUIC, }, } return nil @@ -654,6 +675,7 @@ func deployRemoteNodes(ctx *testcontext.Context, from, to int, P2P: 7513, GRPC_PUB: 9092, GRPC_PRIV: 9093, + QUIC: ctx.QUIC, }, } return nil @@ -726,6 +748,15 @@ func deployNode(ctx *testcontext.Context, id string, labels map[string]string, f for _, flag := range flags { cmd = append(cmd, flag.Flag()) } + p2pProto := apiv1.ProtocolTCP + if ctx.QUIC { + p2pProto = apiv1.ProtocolUDP + cmd = append(cmd, + "--listen", "/ip4/0.0.0.0/udp/7513/quic-v1", + "--enable-tcp-transport=false", + "--enable-quic-transport=true", + ) + } deployment := appsv1.Deployment(id, ctx.Namespace). WithLabels(labels). WithSpec(appsv1.DeploymentSpec(). @@ -757,7 +788,7 @@ func deployNode(ctx *testcontext.Context, id string, labels map[string]string, f WithImage(ctx.Image). WithImagePullPolicy(apiv1.PullIfNotPresent). WithPorts( - corev1.ContainerPort().WithContainerPort(7513).WithName("p2p"), + corev1.ContainerPort().WithContainerPort(7513).WithName("p2p").WithProtocol(p2pProto), corev1.ContainerPort().WithContainerPort(9092).WithName("grpc-pub"), corev1.ContainerPort().WithContainerPort(9093).WithName("grpc-priv"), corev1.ContainerPort().WithContainerPort(9094).WithName("grpc-post"), diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go index 4d7e047975..9b326fae56 100644 --- a/systest/testcontext/context.go +++ b/systest/testcontext/context.go @@ -132,6 +132,9 @@ var ( LayersPerEpoch = parameters.Int( ParamLayersPerEpoch, "number of layers in an epoch", 4, ) + quic = parameters.Bool( + "quic", "if true, QUIC is used for the tests instead of TCP", + ) ) const nsfile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" @@ -175,6 +178,7 @@ type Context struct { Class string } NodeSelector map[string]string + QUIC bool Log *zap.SugaredLogger } @@ -356,6 +360,7 @@ func New(t *testing.T, opts ...Opt) *Context { PostServiceImage: postServiceImage.Get(p), PostInitImage: postInitImage.Get(p), NodeSelector: nodeSelector.Get(p), + QUIC: quic.Get(p), Log: logger.Sugar().Named(t.Name()), } cctx.Storage.Class = class From a4b8d0b7a9609a16ffd81426248b8467b4d2cb72 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Tue, 30 Apr 2024 17:59:59 +0400 Subject: [PATCH 2/5] systest: fix TestPostMalfeasanceProof in QUIC mode --- systest/tests/distributed_post_verification_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/systest/tests/distributed_post_verification_test.go b/systest/tests/distributed_post_verification_test.go index 271fbd3103..eaa5d1554b 100644 --- a/systest/tests/distributed_post_verification_test.go +++ b/systest/tests/distributed_post_verification_test.go @@ -81,6 +81,7 @@ func TestPostMalfeasanceProof(t *testing.T) { cfg.P2P.PrivateNetwork = true cfg.Bootstrap.URL = cluster.BootstrapperGlobalEndpoint(ctx.Namespace, 0) cfg.P2P.MinPeers = 2 + cfg.P2P.EnableQUICTransport = ctx.QUIC ctx.Log.Debugw("Prepared config", "cfg", cfg) goldenATXID := cl.GoldenATX() From 5ff97bf788be9de93debb700c7c709f82e8817a4 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 1 May 2024 02:41:10 +0400 Subject: [PATCH 3/5] systest: fix test_id conflict between the jobs --- systest/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/systest/Makefile b/systest/Makefile index df0818d02f..af731520d6 100644 --- a/systest/Makefile +++ b/systest/Makefile @@ -10,8 +10,14 @@ post_service_image ?= $(org)/post-service:v0.7.6 post_init_image ?= $(org)/postcli:v0.12.5 smesher_image ?= $(org)/go-spacemesh-dev:$(version_info) bs_image ?= $(org)/go-spacemesh-dev-bs:$(version_info) +quic ?= false +ifeq ($(quic),true) +test_id ?= systest-$(version_info)-quic +test_job_name ?= systest-$(version_info)-$(date)-quic +else test_id ?= systest-$(version_info) test_job_name ?= systest-$(version_info)-$(date) +endif keep ?= false clusters ?= 1 size ?= 10 @@ -23,7 +29,6 @@ node_selector ?= namespace ?= label ?= count ?= 1 -quic ?= false configname ?= $(test_job_name) smesher_config ?= parameters/fastnet/smesher.json From 11220618efc772b477f1690f6f7d11d83fc39c63 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 1 May 2024 04:11:40 +0400 Subject: [PATCH 4/5] Fix cleanup --- .github/workflows/systest.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/systest.yml b/.github/workflows/systest.yml index 79ee32ea63..ef4e5c716b 100644 --- a/.github/workflows/systest.yml +++ b/.github/workflows/systest.yml @@ -124,7 +124,7 @@ jobs: - name: Run tests timeout-minutes: 60 env: - test_id: systest-${{ steps.vars.outputs.sha_short }} + test_id: systest-${{ steps.vars.outputs.sha_short }}${{ matrix.protocol == 'quic' && '-quic' || '' }} label: sanity storage: premium-rwo=10Gi node_selector: cloud.google.com/gke-nodepool=gha @@ -139,7 +139,8 @@ jobs: - name: Delete pod if: always() env: - test_id: systest-${{ steps.vars.outputs.sha_short }} + test_id: systest-${{ steps.vars.outputs.sha_short }}${{ matrix.protocol == 'quic' && '-quic' || '' }} + quic: ${{ matrix.protocol == 'tcp' && 'false' || 'true' }} run: make -C systest clean systest-status: From d8024a1663d165297411839be48a0ad147a01e8e Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Thu, 2 May 2024 20:35:16 +0400 Subject: [PATCH 5/5] systest: enable fail-fast for the matrix --- .github/workflows/systest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/systest.yml b/.github/workflows/systest.yml index ef4e5c716b..90b3638559 100644 --- a/.github/workflows/systest.yml +++ b/.github/workflows/systest.yml @@ -55,7 +55,7 @@ jobs: systest: runs-on: ubuntu-latest strategy: - # fail-fast: true + fail-fast: true matrix: protocol: - tcp