From 62b0df8ed92b6edab4a1a89cdb02b8cdfea31e27 Mon Sep 17 00:00:00 2001 From: Michael Washer Date: Mon, 20 Sep 2021 15:46:46 +1000 Subject: [PATCH] Add a NodeSnifferService to perform tcpdump in default netnamespace Add NodeSnifferService to collect tcpdump on a Node Add PrivilegedPodConfig to define values for CreatePrivilegedPod() --- kube/kubernetes_api_service.go | 98 ++++++++++------- pkg/cmd/sniff.go | 17 +-- pkg/config/settings.go | 12 +- pkg/service/sniffer/node_sniffer_service.go | 104 ++++++++++++++++++ .../sniffer/privileged_pod_sniffer_service.go | 22 ++-- 5 files changed, 198 insertions(+), 55 deletions(-) create mode 100644 pkg/service/sniffer/node_sniffer_service.go diff --git a/kube/kubernetes_api_service.go b/kube/kubernetes_api_service.go index 4d6d27b..3202c94 100644 --- a/kube/kubernetes_api_service.go +++ b/kube/kubernetes_api_service.go @@ -18,12 +18,24 @@ import ( "k8s.io/client-go/rest" ) +type PrivilegedPodConfig struct { + NodeName string + ContainerName string + Image string + SocketPath string + Timeout time.Duration +} + +func NewCreatePodConfig() *PrivilegedPodConfig { + return &PrivilegedPodConfig{Timeout: 10 * time.Minute} +} + type KubernetesApiService interface { ExecuteCommand(podName string, containerName string, command []string, stdOut io.Writer) (int, error) DeletePod(podName string) error - CreatePrivilegedPod(nodeName string, containerName string, image string, socketPath string, timeout time.Duration) (*corev1.Pod, error) + CreatePrivilegedPod(config *PrivilegedPodConfig) (*corev1.Pod, error) UploadFile(localPath string, remotePath string, podName string, containerName string) error } @@ -104,16 +116,19 @@ func (k *KubernetesApiServiceImpl) DeletePod(podName string) error { return err } -func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containerName string, image string, socketPath string, timeout time.Duration) (*corev1.Pod, error) { - log.Debugf("creating privileged pod on remote node") +func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(config *PrivilegedPodConfig) (*corev1.Pod, error) { + log.Info("creating privileged pod on remote node") + log.Debugf("creating privileged pod with the following options: { %v }", config) - isSupported, err := k.IsSupportedContainerRuntime(nodeName) + hostNetwork := true + + isSupported, err := k.IsSupportedContainerRuntime(config.NodeName) if err != nil { return nil, err } if !isSupported { - return nil, errors.Errorf("Container runtime on node %s isn't supported. Supported container runtimes are: %v", nodeName, runtime.SupportedContainerRuntimes) + return nil, errors.Errorf("Container runtime on node %s isn't supported. Supported container runtimes are: %v", config.NodeName, runtime.SupportedContainerRuntimes) } typeMetadata := v1.TypeMeta{ @@ -129,12 +144,11 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe }, } + // Add Storage / Mounts + hostPathType := corev1.HostPathSocket + directoryType := corev1.HostPathDirectory + volumeMounts := []corev1.VolumeMount{ - { - Name: "container-socket", - ReadOnly: true, - MountPath: socketPath, - }, { Name: "host", ReadOnly: false, @@ -142,10 +156,39 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe }, } + volumes := []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + Type: &directoryType, + }, + }, + }, + } + + if config.SocketPath != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "container-socket", + ReadOnly: true, + MountPath: config.SocketPath, + }) + volumes = append(volumes, corev1.Volume{ + Name: "container-socket", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: config.SocketPath, + Type: &hostPathType, + }, + }, + }) + } + privileged := true privilegedContainer := corev1.Container{ - Name: containerName, - Image: image, + Name: config.ContainerName, + Image: config.Image, SecurityContext: &corev1.SecurityContext{ Privileged: &privileged, @@ -155,34 +198,13 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe VolumeMounts: volumeMounts, } - hostPathType := corev1.HostPathSocket - directoryType := corev1.HostPathDirectory - podSpecs := corev1.PodSpec{ - NodeName: nodeName, + NodeName: config.NodeName, RestartPolicy: corev1.RestartPolicyNever, HostPID: true, Containers: []corev1.Container{privilegedContainer}, - Volumes: []corev1.Volume{ - { - Name: "host", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/", - Type: &directoryType, - }, - }, - }, - { - Name: "container-socket", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: socketPath, - Type: &hostPathType, - }, - }, - }, - }, + Volumes: volumes, + HostNetwork: hostNetwork, } pod := corev1.Pod{ @@ -214,8 +236,8 @@ func (k *KubernetesApiServiceImpl) CreatePrivilegedPod(nodeName string, containe log.Info("waiting for pod successful startup") - if !utils.RunWhileFalse(verifyPodState, timeout, 1*time.Second) { - return nil, errors.Errorf("failed to create pod within timeout (%s)", timeout) + if !utils.RunWhileFalse(verifyPodState, config.Timeout, 1*time.Second) { + return nil, errors.Errorf("failed to create pod within timeout (%s)", config.Timeout) } return createdPod, nil diff --git a/pkg/cmd/sniff.go b/pkg/cmd/sniff.go index 5910124..ccf0532 100644 --- a/pkg/cmd/sniff.go +++ b/pkg/cmd/sniff.go @@ -20,16 +20,15 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" - _ "k8s.io/client-go/plugin/pkg/client/auth/azure" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" ) var ( @@ -174,7 +173,7 @@ func (o *Ksniff) Complete(cmd *cobra.Command, args []string) error { o.settings.UseDefaultImage = !cmd.Flag("image").Changed o.settings.UseDefaultTCPDumpImage = !cmd.Flag("tcpdump-image").Changed o.settings.UseDefaultSocketPath = !cmd.Flag("socket").Changed - + var err error if o.settings.UserSpecifiedVerboseMode { @@ -270,7 +269,7 @@ func (o *Ksniff) Validate() error { log.Infof("using tcpdump path at: '%s'", o.settings.UserSpecifiedLocalTcpdumpPath) } - pod, err := o.clientset.CoreV1().Pods(o.resultingContext.Namespace).Get(context.TODO(), o.settings.UserSpecifiedPodName, v1.GetOptions{}) + pod, err := o.clientset.CoreV1().Pods(o.resultingContext.Namespace).Get(context.TODO(), o.settings.UserSpecifiedPodName, metav1.GetOptions{}) if err != nil { return err } @@ -303,6 +302,10 @@ func (o *Ksniff) Validate() error { log.Info("sniffing method: privileged pod") bridge := runtime.NewContainerRuntimeBridge(o.settings.DetectedContainerRuntime) o.snifferService = sniffer.NewPrivilegedPodRemoteSniffingService(o.settings, kubernetesApiService, bridge) + } else if o.settings.UserSpecifiedNodeMode { + log.Info("sniffing method: node") + o.snifferService = sniffer.NewNodeSnifferService(o.settings, kubernetesApiService) + } else { log.Info("sniffing method: upload static tcpdump") o.snifferService = sniffer.NewUploadTcpdumpRemoteSniffingService(o.settings, kubernetesApiService) diff --git a/pkg/config/settings.go b/pkg/config/settings.go index 1ce0310..f3f556a 100644 --- a/pkg/config/settings.go +++ b/pkg/config/settings.go @@ -1,8 +1,8 @@ package config import ( - "k8s.io/cli-runtime/pkg/genericclioptions" "time" + "k8s.io/cli-runtime/pkg/genericclioptions" ) type KsniffSettings struct { @@ -17,6 +17,8 @@ type KsniffSettings struct { UserSpecifiedRemoteTcpdumpPath string UserSpecifiedVerboseMode bool UserSpecifiedPrivilegedMode bool + UserSpecifiedNodeMode bool + UserSpecifiedNodeName string UserSpecifiedImage string DetectedPodNodeName string DetectedContainerId string @@ -30,6 +32,14 @@ type KsniffSettings struct { UseDefaultSocketPath bool } +type NodeSnifferServiceConfig struct { + Image string + UserSpecifiedInterface string + UserSpecifiedFilter string + NodeName string + UserSpecifiedPodCreateTimeout time.Duration +} + func NewKsniffSettings(streams genericclioptions.IOStreams) *KsniffSettings { return &KsniffSettings{} } diff --git a/pkg/service/sniffer/node_sniffer_service.go b/pkg/service/sniffer/node_sniffer_service.go new file mode 100644 index 0000000..08becd7 --- /dev/null +++ b/pkg/service/sniffer/node_sniffer_service.go @@ -0,0 +1,104 @@ +package sniffer + +import ( + "io" + + "ksniff/kube" + "ksniff/pkg/config" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" +) + +var defaultInterface = "any" + +type NodeSnifferService struct { + *config.NodeSnifferServiceConfig + privilegedPod *v1.Pod + privilegedContainerName string + targetInterface string + // TODO Replace Node Name + nodeName string + kubernetesApiService kube.KubernetesApiService +} + +func NewNodeSnifferService(options *config.KsniffSettings, service kube.KubernetesApiService) SnifferService { + nodeSnifferService := &NodeSnifferService{ + NodeSnifferServiceConfig: &config.NodeSnifferServiceConfig{ + Image: options.UserSpecifiedImage, + UserSpecifiedInterface: options.UserSpecifiedInterface, + UserSpecifiedFilter: options.UserSpecifiedFilter, + NodeName: options.UserSpecifiedNodeName, + UserSpecifiedPodCreateTimeout: options.UserSpecifiedPodCreateTimeout, + }, + privilegedContainerName: "node-sniff", + kubernetesApiService: service, + nodeName: options.DetectedPodNodeName, + targetInterface: defaultInterface, + } + + if options.UseDefaultImage { + nodeSnifferService.Image = "maintained/tcpdump" + } + + return nodeSnifferService +} + +func (nss *NodeSnifferService) Setup() error { + var err error + // TODO Create a Nodesniffer Object + log.Infof("creating privileged pod on node: '%s'", nss.nodeName) + log.Debugf("initiating sniff on node with option: '%v'", nss) + + podConfig := kube.PrivilegedPodConfig{ + // TODO Replace DetectedPodNodeName with PodName + NodeName: nss.nodeName, + ContainerName: nss.privilegedContainerName, + Image: nss.Image, + Timeout: nss.UserSpecifiedPodCreateTimeout, + } + + nss.privilegedPod, err = nss.kubernetesApiService.CreatePrivilegedPod(&podConfig) + if err != nil { + log.WithError(err).Errorf("failed to create privileged pod on node: '%s'", nss.nodeName) + return err + } + + log.Infof("pod: '%s' created successfully on node: '%s'", nss.privilegedPod.Name, nss.nodeName) + + return nil +} + +func (nss *NodeSnifferService) Cleanup() error { + log.Infof("removing pod: '%s'", nss.privilegedPod.Name) + + err := nss.kubernetesApiService.DeletePod(nss.privilegedPod.Name) + if err != nil { + log.WithError(err).Errorf("failed to remove pod: '%s", nss.privilegedPod.Name) + return err + } + + log.Infof("pod: '%s' removed successfully", nss.privilegedPod.Name) + + return nil +} + +func buildTcpdumpCommand(netInterface string, filter string, tcpdumpImage string) []string { + return []string{"tcpdump", "-i", netInterface, "-U", "-w", "-", filter} +} + +func (nss *NodeSnifferService) Start(stdOut io.Writer) error { + log.Info("starting remote sniffing using privileged pod") + + command := buildTcpdumpCommand(nss.targetInterface, nss.UserSpecifiedFilter, nss.Image) + + exitCode, err := nss.kubernetesApiService.ExecuteCommand(nss.privilegedPod.Name, nss.privilegedContainerName, command, stdOut) + if err != nil { + log.WithError(err).Errorf("failed to start sniffing using privileged pod, exit code: '%d'", exitCode) + return err + } + + log.Info("remote sniffing using privileged pod completed") + + return nil +} diff --git a/pkg/service/sniffer/privileged_pod_sniffer_service.go b/pkg/service/sniffer/privileged_pod_sniffer_service.go index d44833c..3a2954c 100644 --- a/pkg/service/sniffer/privileged_pod_sniffer_service.go +++ b/pkg/service/sniffer/privileged_pod_sniffer_service.go @@ -4,11 +4,12 @@ import ( "bytes" "io" - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" "ksniff/kube" "ksniff/pkg/config" "ksniff/pkg/service/sniffer/runtime" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" ) type PrivilegedPodSnifferService struct { @@ -41,13 +42,16 @@ func (p *PrivilegedPodSnifferService) Setup() error { p.settings.SocketPath = p.runtimeBridge.GetDefaultSocketPath() } - p.privilegedPod, err = p.kubernetesApiService.CreatePrivilegedPod( - p.settings.DetectedPodNodeName, - p.privilegedContainerName, - p.settings.Image, - p.settings.SocketPath, - p.settings.UserSpecifiedPodCreateTimeout, - ) + podConfig := kube.PrivilegedPodConfig{ + NodeName: p.settings.DetectedPodNodeName, + ContainerName: p.privilegedContainerName, + Image: p.settings.Image, + SocketPath: p.settings.SocketPath, + Timeout: p.settings.UserSpecifiedPodCreateTimeout, + } + + p.privilegedPod, err = p.kubernetesApiService.CreatePrivilegedPod(&podConfig) + if err != nil { log.WithError(err).Errorf("failed to create privileged pod on node: '%s'", p.settings.DetectedPodNodeName) return err