Skip to content

Commit

Permalink
SecureComms: E2e test with KBS
Browse files Browse the repository at this point in the history
Install KBS and test SecureComms with KBS
Based on confidential-containers#2072 which should be merged first

Signed-off-by: David Hadas <[email protected]>
  • Loading branch information
davidhadas authored and davidhIBM committed Oct 8, 2024
1 parent 04ba307 commit e3fe1cc
Show file tree
Hide file tree
Showing 9 changed files with 207 additions and 55 deletions.
19 changes: 16 additions & 3 deletions src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ var logger = log.New(log.Writer(), "[forwarder] ", log.LstdFlags|log.Lmsgprefix)
type Config struct {
tlsConfig *tlsutil.TLSConfig
daemonConfig daemon.Config
configPath string
apfConfig daemon.ApfConfig
daemonConfigPath string
apfConfigPath string
listenAddr string
kataAgentSocketPath string
podNamespace string
Expand All @@ -52,6 +54,8 @@ func load(path string, obj interface{}) error {
return fmt.Errorf("failed to decode a Agent Protocol Forwarder config file file: %s: %w", path, err)
}

logger.Printf("succesful loading config from %s\n", path)

return nil
}

Expand All @@ -68,7 +72,8 @@ func (cfg *Config) Setup() (cmd.Starter, error) {

cmd.Parse(programName, os.Args, func(flags *flag.FlagSet) {
flags.BoolVar(&showVersion, "version", false, "Show version")
flags.StringVar(&cfg.configPath, "config", daemon.DefaultConfigPath, "Path to a daemon config file")
flags.StringVar(&cfg.daemonConfigPath, "config", daemon.DefaultDaemonConfigPath, "Path to a daemon config file")
flags.StringVar(&cfg.apfConfigPath, "apf-config", daemon.DefaultAPFConfigPath, "Path to APF config file")
flags.StringVar(&cfg.listenAddr, "listen", daemon.DefaultListenAddr, "Listen address")
flags.StringVar(&cfg.kataAgentSocketPath, "kata-agent-socket", daemon.DefaultKataAgentSocketPath, "Path to a kata agent socket")
flags.StringVar(&cfg.podNamespace, "pod-namespace", daemon.DefaultPodNamespace, "Path to the network namespace where the pod runs")
Expand All @@ -89,10 +94,18 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
cmd.Exit(0)
}

if err := load(cfg.configPath, &cfg.daemonConfig); err != nil {
if err := load(cfg.daemonConfigPath, &cfg.daemonConfig); err != nil {
return nil, err
}

if err := load(cfg.apfConfigPath, &cfg.apfConfig); err != nil {
return nil, err
}

if cfg.apfConfig.SecureComms {
secureComms = true
}

if secureComms {
ppssh.Singleton()
host, port, err := net.SplitHostPort(cfg.listenAddr)
Expand Down
2 changes: 1 addition & 1 deletion src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.kbs-operator-system:8080", "Address of a KBS Service for Secure-Comms")
flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a KBS Service for Secure-Comms")
flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection")

flags.StringVar(&cfg.networkConfig.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider")
Expand Down
85 changes: 72 additions & 13 deletions src/cloud-api-adaptor/docs/SecureComms.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th

See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details.

## Setup
## Setup for CoCo with Trustee

### Deploy CAA
Use any of the option for installing CAA depending on the cloud driver used.
Expand All @@ -37,49 +37,106 @@ Deploy Trustee-Operator by following instructions at [trustee Operator Getting S

Make sure to uncomment the secret generation as recommended for both public and private key (`kbs-auth-public-key` and `kbs-client` secrets).

Copy the kbs-client secret from the `kbs-operator-system` namespace to the `confidential-containers-system` ns. This can be done using:
```sh
kubectl get secrets -n trustee-operator-system
NAME TYPE DATA AGE
kbs-auth-public-key Opaque 1 28h
kbs-client Opaque 1 28h
```

Copy the kbs-client secret from the `trustee-operator-system` namespace to the `confidential-containers-system` ns. This can be done using:

```sh
kubectl get secret kbs-client -n kbs-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f -
kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f -
```

For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is:

```sh
kubectl -n kbs-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash
sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego
kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego

kubectl -n kbs-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
```

### Build a podvm that enforces Secure-Comms
### Build a podvm that enforces Secure-Comms (Optional)

This stage is optional, it should be used in following two cases:
1. Constructing a podvm that enforces Secure Comms to always be activated
2. Configuring the podvm for Secure-Comms Inbounds and outbounds

An alternative to this stage is to use InitData to enforce Secure Comms to be activated. From security standpoint, this alternative is helpful in production when InitData is measured and attested together with the podvm measurement.

Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include:
```sh
ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
```

You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below.
You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

For example:
```sh
ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
```

Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).


### Activate CAA Secure-Comms feature
Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace:
Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.

```sh
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
```

Set InitData to point KBC services to IP address 127.0.0.1
```sh
cat <<EOF > /tmp/initdata.txt
algorithm = "sha384"
version = "0.1.0"
[data]
"aa.toml" = '''
[token_configs]
[token_configs.coco_as]
url = 'http://127.0.0.1:8080'
[token_configs.kbs]
url = 'http://127.0.0.1:8080'
'''
"apf.json" = '''
{
sc: true
}
"cdh.toml" = '''
socket = 'unix:///run/confidential-containers/cdh.sock'
credentials = []
[kbc]
name = 'cc_kbc'
url = 'http://127.0.0.1:8080'
'''
EOF
export INITDATA=`base64 -w 0 /tmp/initdata.txt`
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^ INITDATA: .*/ INITDATA: '$INITDATA'/'|kubectl apply -f -

```

You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
```sh
apiVersion: v1
data:
...
SECURE_COMMS: "true"
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
...
```

You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. See more details regarding Inbounds and Outbounds below.

You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point.


### Adding named tunnels to the SSH channel

## Adding named tunnels to the SSH channel
Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created.

|---------Tunnel----------|
Expand All @@ -88,9 +145,11 @@ Client->Inbound----------->Outbound->Server

Inbounds and Outbounds take the form of a comma separated inbound/outbound tags such that Inbounds are formed as "InboundTag1,InboundTag2,InboundTag3,..." and Outbounds are formed as "OutboundTag1,OutboundTag2,outboundTag3,..."

Each Inbound tag is structured as `Phase:Name:Port` where:

Each Inbound tag is structured as `Phase:Name:Namespace:Port` or `Phase:Name:Port` where:
- Phase can be 'KUBERNETES_PHASE' to represent an outbound available during the Kubernetes phase, 'ATTESTATION_PHASE' to represent an outbound available during the Attestation phase, or 'BOTH_PHASES' to represent an outbound available during both phases.
- Name is the name of the tunnel
- Namespace (if available) is a linux network namespace where the local service should be available.
- Port is the local service port being opened to serve as ingress of the tunnel.

Each outbound tag is structured as `Phase:Name:Host:Port` or `Phase:Name:Port` where:
Expand Down
2 changes: 2 additions & 0 deletions src/cloud-api-adaptor/libvirt/config_libvirt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -122,3 +122,5 @@ KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml)
KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml)
[ -z ${KBS_IMAGE} ] || echo "KBS_IMAGE=\"${KBS_IMAGE}\"" >> libvirt.properties
[ -z ${KBS_IMAGE_TAG} ] || echo "KBS_IMAGE_TAG=\"${KBS_IMAGE_TAG}\"" >> libvirt.properties
echo "SECURE_COMMS=\"true\"" >> libvirt.properties
echo "SECURE_COMMS_KBS_ADDR=\"kbs.coco-tenant\"" >> libvirt.properties
2 changes: 1 addition & 1 deletion src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
Content: agentConfig,
},
{
Path: forwarder.DefaultConfigPath,
Path: forwarder.DefaultDaemonConfigPath,
Content: string(daemonJSON),
},
},
Expand Down
9 changes: 8 additions & 1 deletion src/cloud-api-adaptor/pkg/forwarder/forwarder.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,15 @@ const (
DefaultListenHost = "0.0.0.0"
DefaultListenPort = "15150"
DefaultListenAddr = DefaultListenHost + ":" + DefaultListenPort
DefaultConfigPath = "/run/peerpod/daemon.json"
DefaultDaemonConfigPath = "/run/peerpod/daemon.json"
DefaultAPFConfigPath = "/run/peerpod/apf.json"
DefaultPodNetworkSpecPath = "/run/peerpod/podnetwork.json"
DefaultKataAgentSocketPath = "/run/kata-containers/agent.sock"
DefaultPodNamespace = "/run/netns/podns"
AgentURLPath = "/agent"
)

// Unmeasured Configuration via User Data
type Config struct {
PodNetwork *tunneler.Config `json:"pod-network"`
PodNamespace string `json:"pod-namespace"`
Expand All @@ -44,6 +46,11 @@ type Config struct {
TLSClientCA string `json:"tls-client-ca,omitempty"`
}

// Measured Configuration via InitData
type ApfConfig struct {
SecureComms bool `json:"sc,omitempty"`
}

type Daemon interface {
Start(ctx context.Context) error
Shutdown() error
Expand Down
4 changes: 2 additions & 2 deletions src/cloud-api-adaptor/pkg/userdata/provision.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ const (
)

var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath}
var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath}
var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultDaemonConfigPath, cloud.AuthFilePath, cloud.InitdataPath}
var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath, forwarder.DefaultAPFConfigPath}

type Config struct {
fetchTimeout int
Expand Down
87 changes: 53 additions & 34 deletions src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,18 @@ import (

// LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
type LibvirtProvisioner struct {
conn *libvirt.Connect // Libvirt connection
network string // Network name
ssh_key_file string // SSH key file used to connect to Libvirt
storage string // Storage pool name
uri string // Libvirt URI
wd string // libvirt's directory path on this repository
volumeName string // Podvm volume name
clusterName string // Cluster name
kbs_image string // KBS Service OCI Image URL
kbs_image_tag string // KBS Service OCI Image Tag
conn *libvirt.Connect // Libvirt connection
network string // Network name
ssh_key_file string // SSH key file used to connect to Libvirt
storage string // Storage pool name
uri string // Libvirt URI
wd string // libvirt's directory path on this repository
volumeName string // Podvm volume name
clusterName string // Cluster name
kbs_image string // KBS Service OCI Image URL
kbs_image_tag string // KBS Service OCI Image Tag
secure_comms string // Activate CAA SECURE_COMMS
secure_comms_kbs_addr string // KBS URL or "false"
}

// LibvirtInstallOverlay implements the InstallOverlay interface
Expand Down Expand Up @@ -92,18 +94,30 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
kbs_image_tag = properties["KBS_IMAGE_TAG"]
}

secure_comms := "false"
if properties["SECURE_COMMS"] != "" {
secure_comms = properties["SECURE_COMMS"]
}

secure_comms_kbs_addr := ""
if properties["SECURE_COMMS_KBS_ADDR"] != "" {
secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"]
}

// TODO: Check network and storage are not nil?
return &LibvirtProvisioner{
conn: conn,
network: network,
ssh_key_file: ssh_key_file,
storage: storage,
uri: uri,
wd: wd,
volumeName: vol_name,
clusterName: clusterName,
kbs_image: kbs_image,
kbs_image_tag: kbs_image_tag,
conn: conn,
network: network,
ssh_key_file: ssh_key_file,
storage: storage,
uri: uri,
wd: wd,
volumeName: vol_name,
clusterName: clusterName,
kbs_image: kbs_image,
kbs_image_tag: kbs_image_tag,
secure_comms: secure_comms,
secure_comms_kbs_addr: secure_comms_kbs_addr,
}, nil
}

Expand Down Expand Up @@ -203,13 +217,15 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config)

func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
return map[string]string{
"network": l.network,
"podvm_volume": l.volumeName,
"ssh_key_file": l.ssh_key_file,
"storage": l.storage,
"uri": l.uri,
"KBS_IMAGE": l.kbs_image,
"KBS_IMAGE_TAG": l.kbs_image_tag,
"network": l.network,
"podvm_volume": l.volumeName,
"ssh_key_file": l.ssh_key_file,
"storage": l.storage,
"uri": l.uri,
"KBS_IMAGE": l.kbs_image,
"KBS_IMAGE_TAG": l.kbs_image_tag,
"SECURE_COMMS": l.secure_comms,
"SECURE_COMMS_KBS_ADDR": l.secure_comms_kbs_addr,
}
}

Expand Down Expand Up @@ -312,17 +328,20 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,

// Mapping the internal properties to ConfigMapGenerator properties and their default values.
mapProps := map[string][2]string{
"network": {"default", "LIBVIRT_NET"},
"storage": {"default", "LIBVIRT_POOL"},
"pause_image": {"", "PAUSE_IMAGE"},
"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
"uri": {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
"vxlan_port": {"", "VXLAN_PORT"},
"INITDATA": {"", "INITDATA"},
"network": {"default", "LIBVIRT_NET"},
"storage": {"default", "LIBVIRT_POOL"},
"pause_image": {"", "PAUSE_IMAGE"},
"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
"uri": {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
"vxlan_port": {"", "VXLAN_PORT"},
"INITDATA": {"", "INITDATA"},
"SECURE_COMMS": {"", "SECURE_COMMS"},
"SECURE_COMMS_KBS_ADDR": {"", "SECURE_COMMS_KBS_ADDR"},
}

for k, v := range mapProps {
if properties[k] != v[0] {
fmt.Printf("TESTONLY setting %s: %s\n", v[1], properties[k])
if err = lio.Overlay.SetKustomizeConfigMapGeneratorLiteral("peer-pods-cm",
v[1], properties[k]); err != nil {
return err
Expand Down
Loading

0 comments on commit e3fe1cc

Please sign in to comment.