SecureComms: Activate PP SC using InitData

Add apj.json to InitData
apf.josn includes a secure-comms key used to activate secure comms (if not
already activated using an agent-protocol-forwarder.service flag)

Signed-off-by: David Hadas <[email protected]>

src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go

@@ -34,7 +34,9 @@ var logger = log.New(log.Writer(), "[forwarder] ", log.LstdFlags|log.Lmsgprefix)
 type Config struct {
 	tlsConfig           *tlsutil.TLSConfig
 	daemonConfig        daemon.Config
-	configPath          string
+	apfConfig           daemon.ApfConfig
+	daemonConfigPath    string
+	apfConfigPath       string
 	listenAddr          string
 	kataAgentSocketPath string
 	podNamespace        string
@@ -52,6 +54,8 @@ func load(path string, obj interface{}) error {
 		return fmt.Errorf("failed to decode a Agent Protocol Forwarder config file file: %s: %w", path, err)
 	}
 
+	logger.Printf("Succesfully loading config from %s\n", path)
+
 	return nil
 }
 
@@ -68,7 +72,8 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 
 	cmd.Parse(programName, os.Args, func(flags *flag.FlagSet) {
 		flags.BoolVar(&showVersion, "version", false, "Show version")
-		flags.StringVar(&cfg.configPath, "config", daemon.DefaultConfigPath, "Path to a daemon config file")
+		flags.StringVar(&cfg.daemonConfigPath, "config", daemon.DefaultDaemonConfigPath, "Path to a daemon config file")
+		flags.StringVar(&cfg.apfConfigPath, "apf-config", daemon.DefaultAPFConfigPath, "Path to APF config file")
 		flags.StringVar(&cfg.listenAddr, "listen", daemon.DefaultListenAddr, "Listen address")
 		flags.StringVar(&cfg.kataAgentSocketPath, "kata-agent-socket", daemon.DefaultKataAgentSocketPath, "Path to a kata agent socket")
 		flags.StringVar(&cfg.podNamespace, "pod-namespace", daemon.DefaultPodNamespace, "Path to the network namespace where the pod runs")
@@ -89,10 +94,18 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 		cmd.Exit(0)
 	}
 
-	if err := load(cfg.configPath, &cfg.daemonConfig); err != nil {
+	if err := load(cfg.daemonConfigPath, &cfg.daemonConfig); err != nil {
+		return nil, err
+	}
+
+	if err := load(cfg.apfConfigPath, &cfg.apfConfig); err != nil {
 		return nil, err
 	}
 
+	if cfg.apfConfig.SecureComms {
+		secureComms = true
+	}
+
 	if secureComms {
 		ppssh.Singleton()
 		host, port, err := net.SplitHostPort(cfg.listenAddr)

src/cloud-api-adaptor/docs/SecureComms.md

@@ -58,28 +58,64 @@ kubectl -n trustee-operator-system exec deployment/trustee-deployment --containe
 kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
 ```
 
-### Build a podvm that enforces Secure-Comms
+### Activate the Secure-Comms feature
+Activate Secure-Comms at the CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.  
 
-Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include:
 ```sh
-ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
+kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
 ```
 
-You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags.  [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+### Activate CAA Secure-Comms feature
+Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.  
 
-For example:
 ```sh
-ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:podns:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
-```
+kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
+=======
+Next, we need to signal the Peer Pod to also activate the Secure-Comms feature. This can be done using an `apf.json` file included in the InitData. InitData must be measured and attested together with the podvm measurement to be trusted. In use cases where the InitData is not trusted, [use a podvm image with Secure-Comms enabled by default as shown below.](#build-a-podvm-that-enforces-secure-comms-optional)
 
-Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).
 
+Set InitData to point AA and KBC services to IP address 127.0.0.1 and activate Secure-Comms at the Peer Pod:
+```sh
+cat <<EOF > /tmp/initdata.txt
+algorithm = "sha384"
+version = "0.1.0"
 
-### Activate CAA Secure-Comms feature
-Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.  
+[data]
+"aa.toml" = '''
+[token_configs]
+[token_configs.coco_as]
+url = 'http://127.0.0.1:8080'
 
+[token_configs.kbs]
+url = 'http://127.0.0.1:8080'
+'''
+"apf.json" = '''
+{
+    secure-comms: true
+}
+'''
+"cdh.toml"  = '''
+socket = 'unix:///run/confidential-containers/cdh.sock'
+credentials = []
+[kbc]
+name = 'cc_kbc'
+url = 'http://127.0.0.1:8080'
+'''
+EOF
+export INITDATA=`base64 -w 0 /tmp/initdata.txt`
+kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed 's/^  INITDATA: .*/  INITDATA: '$INITDATA'/'|kubectl apply -f -
+
+```
+On the Worker Node side, you may include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+
+Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
 ```sh
-kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
+apiVersion: v1
+data:
+    ...
+    SECURE_COMMS: "true"
+    SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
+    ...
 ```
 
 Set InitData to point KBC services to IP address 127.0.0.1 
@@ -116,7 +152,7 @@ apiVersion: v1
 data:
     ...
     SECURE_COMMS: "true"
-    SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
+    SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:podns:149.81.64.62:7777"
     ...
 ```
 
@@ -153,6 +189,24 @@ Each outbound tag is structured as `Phase:Name:Host:Port` or `Phase:Name:Port` w
 
 For example, an outbound tag such as `KUBERNETES_PHASE:ABC:myhost.com:1234` means that during the `Kubernetes phase`, an output of a tunnel named `ABC` is registered, such that information from a client connecting to ABC Inbound will be tunneled and forwarded to `myhost.com` port `1234`).
 
+### Build a podvm that enforces Secure-Comms (Optional)
+
+This stage is optional, it can be used to activate Secure Comms in the podvm for cases in which InitData is not trusted. InitData must be measured and attested together with the podvm measurement to be trusted. The below procedure can be used in cases where such measurement is not used. In such cases, you may force the a podvm to activate Secure-Comms using a flag as described below.  
+
+Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include:
+```sh
+ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
+```
+
+You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags.  [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+
+For example:
+```sh
+ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:podns:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
+```
+
+Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).
+
 ## Testing
 
 Testing securecomms as a standalone can be done by using:

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -287,7 +287,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 				Content: agentConfig,
 			},
 			{
-				Path:    forwarder.DefaultConfigPath,
+				Path:    forwarder.DefaultDaemonConfigPath,
 				Content: string(daemonJSON),
 			},
 		},

src/cloud-api-adaptor/pkg/forwarder/forwarder.go

@@ -27,13 +27,15 @@ const (
 	DefaultListenHost          = "0.0.0.0"
 	DefaultListenPort          = "15150"
 	DefaultListenAddr          = DefaultListenHost + ":" + DefaultListenPort
-	DefaultConfigPath          = "/run/peerpod/daemon.json"
+	DefaultDaemonConfigPath    = "/run/peerpod/daemon.json"
+	DefaultAPFConfigPath       = "/run/peerpod/apf.json"
 	DefaultPodNetworkSpecPath  = "/run/peerpod/podnetwork.json"
 	DefaultKataAgentSocketPath = "/run/kata-containers/agent.sock"
 	DefaultPodNamespace        = "/run/netns/podns"
 	AgentURLPath               = "/agent"
 )
 
+// Unmeasured Configuration via User Data
 type Config struct {
 	PodNetwork   *tunneler.Config `json:"pod-network"`
 	PodNamespace string           `json:"pod-namespace"`
@@ -44,6 +46,11 @@ type Config struct {
 	TLSClientCA   string `json:"tls-client-ca,omitempty"`
 }
 
+// Measured Configuration via InitData
+type ApfConfig struct {
+	SecureComms bool `json:"secure-comms,omitempty"`
+}
+
 type Daemon interface {
 	Start(ctx context.Context) error
 	Shutdown() error

src/cloud-api-adaptor/pkg/userdata/provision.go

@@ -32,8 +32,8 @@ const (
 )
 
 var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
-var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath}
-var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath}
+var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultDaemonConfigPath, cloud.AuthFilePath, cloud.InitdataPath}
+var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath, forwarder.DefaultAPFConfigPath}
 
 type Config struct {
 	fetchTimeout  int