Merge pull request #1293 from gotd/teat/connclose-reproduce

test: add connection close repro attempt

cmd/connclose/.gitignore

@@ -0,0 +1,2 @@
+connclose
+secret.yml

cmd/connclose/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:latest
+COPY connclose /usr/sbin/connclose
+ENTRYPOINT ["/usr/sbin/connclose"]

cmd/connclose/Makefile

@@ -0,0 +1,42 @@
+all: connclose
+
+connclose:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o connclose .
+.PHONY: connclose
+
+image: connclose
+	docker build -t connclose:latest .
+
+image-load:
+	kind load docker-image connclose:latest
+
+deploy:
+	kubectl apply -f secret.yml -f deployment.yml -f network.yml -f network.allow.yml
+
+start:
+	bash start.sh
+
+down:
+	kind delete cluster
+
+restart:
+	kubectl rollout restart deployment bot
+
+update: image image-load restart
+
+allow:
+	kubectl apply -f network.allow.yml
+
+deny:
+	kubectl apply -f network.deny.yml
+
+wait:
+	kubectl rollout status deployment bot
+
+logs:
+	kubectl logs -l app=bot
+
+follow:
+	kubectl logs -f -l app=bot
+
+up: start image image-load deploy wait

cmd/connclose/README.md

@@ -0,0 +1,56 @@
+This project is an attempt to reproduce connection timeout that causes gotd to hang.
+
+> [!WARNING]
+> Currently I'm unable to reproduce the issue.
+
+Referenced issue: https://github.com/gotd/td/issues/1030
+
+Dependencies:
+- docker
+- kind
+- helm
+- cilium cli
+- go
+
+## Preparation
+
+```bash
+cp secret.example.yml secret.yml
+```
+
+Edit `secret.yml` and put there your bot and application credentials.
+
+## Running
+
+Start cluster:
+
+```bash
+make up
+```
+
+Deny connections to telegram:
+```bash
+make deny
+```
+
+Update binary
+```bash
+make update
+```
+
+Restore connections
+```bash
+make allow
+```
+
+## Logs
+
+```bash
+make logs
+```
+
+## Cleanup
+
+```bash
+make down
+```

cmd/connclose/deployment.yml

@@ -0,0 +1,65 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bot
+  labels:
+    app.kubernetes.io/name: bot
+spec:
+  strategy:
+    type: Recreate
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: bot
+  template:
+    metadata:
+      labels:
+        app: bot
+        app.kubernetes.io/name: bot
+    spec:
+      containers:
+        - name: bot
+          image: connclose:latest
+          imagePullPolicy: Never
+          resources:
+            requests:
+              cpu: 500m
+              memory: 128M
+            limits:
+              cpu: 1000m
+              memory: 256M
+          env:
+            - name: GOMEMLIMIT
+              value: "128MiB"
+            - name: GOMAXPROCS
+              value: "2"
+            - name: OTEL_METRICS_EXPORTER
+              value: "prometheus"
+            - name: OTEL_EXPORTER_PROMETHEUS_PORT
+              value: "8090"
+            - name: OTEL_EXPORTER_PROMETHEUS_HOST
+              value: "0.0.0.0"
+            - name: PPROF_ADDR
+              value: "0.0.0.0:8090"
+            - name: OTEL_RESOURCE_ATTRIBUTES
+              value: "service.name=connclose"
+            - name: OTEL_LOG_LEVEL
+              value: "DEBUG"
+            - name: OTEL_TRACES_EXPORTER
+              value: "none"
+            - name: BOT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: config
+                  key: BOT_TOKEN
+            - name: APP_ID
+              valueFrom:
+                secretKeyRef:
+                  name: config
+                  key: APP_ID
+            - name: APP_HASH
+              valueFrom:
+                secretKeyRef:
+                  name: config
+                  key: APP_HASH

cmd/connclose/kind.yml

@@ -0,0 +1,7 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
+networking:
+  disableDefaultCNI: true

cmd/connclose/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/gotd/td/telegram"
+	"github.com/gotd/td/telegram/message"
+	"github.com/gotd/td/tg"
+)
+
+func run(ctx context.Context) error {
+	logger, _ := zap.NewDevelopment(zap.IncreaseLevel(zapcore.DebugLevel))
+	defer func() { _ = logger.Sync() }()
+
+	dispatcher := tg.NewUpdateDispatcher()
+	return telegram.BotFromEnvironment(ctx, telegram.Options{
+		Logger:        logger,
+		UpdateHandler: dispatcher,
+	}, func(ctx context.Context, client *telegram.Client) error {
+		sender := message.NewSender(tg.NewClient(client))
+		dispatcher.OnNewMessage(func(ctx context.Context, entities tg.Entities, u *tg.UpdateNewMessage) error {
+			m, ok := u.Message.(*tg.Message)
+			if !ok || m.Out {
+				return nil
+			}
+
+			_, err := sender.Reply(entities, u).Text(ctx, m.Message)
+			return err
+		})
+		return nil
+	}, telegram.RunUntilCanceled)
+}
+
+func main() {
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	if err := run(ctx); err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "%+v\n", err)
+		os.Exit(2)
+	}
+}

cmd/connclose/network.allow.yml

@@ -0,0 +1,31 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: telegram
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: bot
+  egress:
+    - # telegram datacenters list
+      # generated by cmd/dcsidr
+      toCIDRSet:
+        - cidr: 149.154.175.59/32
+        - cidr: 149.154.175.53/32
+        - cidr: 2001:b28:f23d:f001::a/128
+        - cidr: 149.154.167.50/32
+        - cidr: 149.154.167.51/32
+        - cidr: 149.154.167.151/32
+        - cidr: 2001:67c:4e8:f002::a/128
+        - cidr: 2001:67c:4e8:f002::b/128
+        - cidr: 149.154.175.100/32
+        - cidr: 2001:b28:f23d:f003::a/128
+        - cidr: 149.154.167.91/32
+        - cidr: 2001:67c:4e8:f004::a/128
+        - cidr: 149.154.166.120/32
+        - cidr: 2001:67c:4e8:f004::b/128
+        - cidr: 2001:b28:f23f:f005::a/128
+        - cidr: 91.108.56.173/32
+      toPorts:
+        - ports:
+            - port: "443"

cmd/connclose/network.deny.yml

@@ -0,0 +1,10 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: telegram
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: bot
+  # None.
+  egress: []

cmd/connclose/network.yml

@@ -0,0 +1,20 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: dns
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: bot
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+          rules:
+            dns:
+              - matchPattern: "*"

cmd/connclose/secret.example.yml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: config
+stringData:
+  BOT_TOKEN: "1111111111:AAG1SomeBotTokenFromBotFather-1337a"
+  APP_ID: "0123456"
+  APP_HASH: "0123456789abcdef0123456789abcdef"

cmd/connclose/start.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -ex
+
+kind create cluster --config kind.yml
+docker pull quay.io/cilium/cilium:v1.14.6
+kind load docker-image quay.io/cilium/cilium:v1.14.6
+
+helm install cilium cilium/cilium --version 1.14.6 \
+   --namespace kube-system \
+   --set image.pullPolicy=IfNotPresent \
+   --set ipam.mode=kubernetes
+
+cilium status --wait

cmd/dscidr/main.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"net/netip"
+	"os"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/gotd/td/telegram/dcs"
+)
+
+type CIDRElement struct {
+	CIDR string `json:"cidr" yaml:"cidr"`
+}
+
+type Ports struct {
+	Ports []Port `json:"ports" yaml:"ports"`
+}
+
+type Port struct {
+	Port string `json:"port" yaml:"port"`
+}
+
+type Element struct {
+	ToCIDRSet []CIDRElement `json:"toCIDRSet" yaml:"toCIDRSet"`
+	ToPorts   []Ports       `json:"toPorts" yaml:"toPorts"`
+}
+
+/*
+
+- toCIDRSet:
+   - cidr: 192.169.0.1/32
+  toPorts:
+   - ports:
+	   - port: "443"
+*/
+
+func main() {
+	list := dcs.Prod()
+	var out []CIDRElement
+	met := make(map[string]struct{})
+	for _, v := range list.Options {
+		ip, err := netip.ParseAddr(v.IPAddress)
+		if err != nil {
+			panic(err)
+		}
+		if v.Port != 443 {
+			panic("port != 443")
+		}
+		prefix := netip.PrefixFrom(ip, ip.BitLen())
+		cidr := prefix.String()
+		if _, ok := met[cidr]; ok {
+			continue
+		}
+		met[cidr] = struct{}{}
+		out = append(out, CIDRElement{CIDR: cidr})
+	}
+	fmt.Println("# telegram datacenters list")
+	fmt.Println("# generated by cmd/dcsidr")
+	e := yaml.NewEncoder(os.Stdout)
+	e.SetIndent(2)
+	if err := e.Encode(Element{
+		ToCIDRSet: out,
+		ToPorts: []Ports{
+			{
+				Ports: []Port{
+					{Port: "443"},
+				},
+			},
+		},
+	}); err != nil {
+		panic(err)
+	}
+}