Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SecureComms: Add Support for activating using InitData #2072

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

SecureComms: Add Support for activating using InitData #2072

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ var logger = log.New(log.Writer(), "[forwarder] ", log.LstdFlags|log.Lmsgprefix)
type Config struct {
tlsConfig *tlsutil.TLSConfig
daemonConfig daemon.Config
configPath string
apfConfig daemon.ApfConfig
daemonConfigPath string
apfConfigPath string
listenAddr string
kataAgentSocketPath string
podNamespace string
Expand All @@ -52,6 +54,8 @@ func load(path string, obj interface{}) error {
return fmt.Errorf("failed to decode a Agent Protocol Forwarder config file file: %s: %w", path, err)
}

logger.Printf("Succesfully loading config from %s\n", path)

return nil
}

Expand All @@ -68,7 +72,8 @@ func (cfg *Config) Setup() (cmd.Starter, error) {

cmd.Parse(programName, os.Args, func(flags *flag.FlagSet) {
flags.BoolVar(&showVersion, "version", false, "Show version")
flags.StringVar(&cfg.configPath, "config", daemon.DefaultConfigPath, "Path to a daemon config file")
flags.StringVar(&cfg.daemonConfigPath, "config", daemon.DefaultDaemonConfigPath, "Path to a daemon config file")
flags.StringVar(&cfg.apfConfigPath, "apf-config", daemon.DefaultAPFConfigPath, "Path to APF config file")
flags.StringVar(&cfg.listenAddr, "listen", daemon.DefaultListenAddr, "Listen address")
flags.StringVar(&cfg.kataAgentSocketPath, "kata-agent-socket", daemon.DefaultKataAgentSocketPath, "Path to a kata agent socket")
flags.StringVar(&cfg.podNamespace, "pod-namespace", daemon.DefaultPodNamespace, "Path to the network namespace where the pod runs")
Expand All @@ -89,10 +94,18 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
cmd.Exit(0)
}

if err := load(cfg.configPath, &cfg.daemonConfig); err != nil {
if err := load(cfg.daemonConfigPath, &cfg.daemonConfig); err != nil {
return nil, err
}

if err := load(cfg.apfConfigPath, &cfg.apfConfig); err != nil {
return nil, err
}

if cfg.apfConfig.SecureComms {
secureComms = true
}

if secureComms {
ppssh.Singleton()
host, port, err := net.SplitHostPort(cfg.listenAddr)
Expand Down
72 changes: 60 additions & 12 deletions src/cloud-api-adaptor/docs/SecureComms.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,28 +58,58 @@ kubectl -n trustee-operator-system exec deployment/trustee-deployment --containe
kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
```

### Build a podvm that enforces Secure-Comms
### Activate the Secure-Comms feature
Activate Secure-Comms at the CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.

Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include:
```sh
ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
```

You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
Next, we need to signal the Peer Pod to also activate the Secure-Comms feature. This can be done using an `apf.json` file included in the InitData. InitData must be measured and attested together with the podvm measurement to be trusted. In use cases where the InitData is not trusted, [use a podvm image with Secure-Comms enabled by default as shown below.](#build-a-podvm-that-enforces-secure-comms-optional)

For example:

Set InitData to point AA and KBC services to IP address 127.0.0.1 and activate Secure-Comms at the Peer Pod:
```sh
ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:podns:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
```
cat <<EOF > /tmp/initdata.txt
algorithm = "sha384"
version = "0.1.0"

Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).
[data]
"aa.toml" = '''
[token_configs]
[token_configs.coco_as]
url = 'http://127.0.0.1:8080'

[token_configs.kbs]
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As KBS endpoint is different when enabling SecureComms, shall we also add link from ./initdata.md to this file?

url = 'http://127.0.0.1:8080'
'''
"apf.json" = '''
{
secure-comms: true
}
davidhadas marked this conversation as resolved.
Show resolved Hide resolved
'''
"cdh.toml" = '''
socket = 'unix:///run/confidential-containers/cdh.sock'
credentials = []
[kbc]
name = 'cc_kbc'
url = 'http://127.0.0.1:8080'
'''
EOF
export INITDATA=`base64 -w 0 /tmp/initdata.txt`
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^ INITDATA: .*/ INITDATA: '$INITDATA'/'|kubectl apply -f -

### Activate CAA Secure-Comms feature
Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.
```
On the Worker Node side, you may include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
```sh
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
apiVersion: v1
data:
...
SECURE_COMMS: "true"
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
...
```

Set InitData to point KBC services to IP address 127.0.0.1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this have a heading like Build a podvm that enforces Secure-Comms (Optional) section does as it's an alternative to it?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some expansion of the explanation of what this is doing and why would be nice.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will be adapting the documentation based on this comment. Please reconsider the next version.

Expand Down Expand Up @@ -116,7 +146,7 @@ apiVersion: v1
data:
...
SECURE_COMMS: "true"
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:podns:149.81.64.62:7777"
...
```

Expand Down Expand Up @@ -153,6 +183,24 @@ Each outbound tag is structured as `Phase:Name:Host:Port` or `Phase:Name:Port` w

For example, an outbound tag such as `KUBERNETES_PHASE:ABC:myhost.com:1234` means that during the `Kubernetes phase`, an output of a tunnel named `ABC` is registered, such that information from a client connecting to ABC Inbound will be tunneled and forwarded to `myhost.com` port `1234`).

### Build a podvm that enforces Secure-Comms (Optional)

This stage is optional, it can be used to activate Secure Comms in the podvm for cases in which InitData is not trusted. InitData must be measured and attested together with the podvm measurement to be trusted. The below procedure can be used in cases where such measurement is not used. In such cases, you may force the a podvm to activate Secure-Comms using a flag as described below.

Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include:
```sh
ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
```

You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

For example:
```sh
ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:podns:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
```

Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).

## Testing

Testing securecomms as a standalone can be done by using:
Expand Down
2 changes: 1 addition & 1 deletion src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
Content: agentConfig,
},
{
Path: forwarder.DefaultConfigPath,
Path: forwarder.DefaultDaemonConfigPath,
Content: string(daemonJSON),
},
},
Expand Down
9 changes: 8 additions & 1 deletion src/cloud-api-adaptor/pkg/forwarder/forwarder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,15 @@ const (
DefaultListenHost = "0.0.0.0"
DefaultListenPort = "15150"
DefaultListenAddr = DefaultListenHost + ":" + DefaultListenPort
DefaultConfigPath = "/run/peerpod/daemon.json"
DefaultDaemonConfigPath = "/run/peerpod/daemon.json"
DefaultAPFConfigPath = "/run/peerpod/apf.json"
DefaultPodNetworkSpecPath = "/run/peerpod/podnetwork.json"
DefaultKataAgentSocketPath = "/run/kata-containers/agent.sock"
DefaultPodNamespace = "/run/netns/podns"
AgentURLPath = "/agent"
)

// Unmeasured Configuration via User Data
type Config struct {
PodNetwork *tunneler.Config `json:"pod-network"`
PodNamespace string `json:"pod-namespace"`
Expand All @@ -44,6 +46,11 @@ type Config struct {
TLSClientCA string `json:"tls-client-ca,omitempty"`
}

// Measured Configuration via InitData
type ApfConfig struct {
SecureComms bool `json:"secure-comms,omitempty"`
}

type Daemon interface {
Start(ctx context.Context) error
Shutdown() error
Expand Down
4 changes: 2 additions & 2 deletions src/cloud-api-adaptor/pkg/userdata/provision.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ const (
)

var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath}
var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath}
var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultDaemonConfigPath, cloud.AuthFilePath, cloud.InitdataPath}
var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath, forwarder.DefaultAPFConfigPath}

type Config struct {
fetchTimeout int
Expand Down
Loading