Merge pull request docker#83 from cpuguy83/add_configs_to_compose

Add support for configs to compose format

cli/command/stack/deploy_composefile.go

@@ -79,6 +79,14 @@ func deployCompose(ctx context.Context, dockerCli command.Cli, opts deployOption
 		return err
 	}
 
+	configs, err := convert.Configs(namespace, config.Configs)
+	if err != nil {
+		return err
+	}
+	if err := createConfigs(ctx, dockerCli, namespace, configs); err != nil {
+		return err
+	}
+
 	services, err := convert.Services(namespace, config, dockerCli.Client())
 	if err != nil {
 		return err
@@ -208,6 +216,33 @@ func createSecrets(
 	return nil
 }
 
+func createConfigs(
+	ctx context.Context,
+	dockerCli command.Cli,
+	namespace convert.Namespace,
+	configs []swarm.ConfigSpec,
+) error {
+	client := dockerCli.Client()
+
+	for _, configSpec := range configs {
+		config, _, err := client.ConfigInspectWithRaw(ctx, configSpec.Name)
+		if err == nil {
+			// config already exists, then we update that
+			if err := client.ConfigUpdate(ctx, config.ID, config.Meta.Version, configSpec); err != nil {
+				return err
+			}
+		} else if apiclient.IsErrConfigNotFound(err) {
+			// config does not exist, then we create a new one.
+			if _, err := client.ConfigCreate(ctx, configSpec); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+	return nil
+}
+
 func createNetworks(
 	ctx context.Context,
 	dockerCli command.Cli,

cli/compose/convert/compose.go

@@ -116,3 +116,27 @@ func Secrets(namespace Namespace, secrets map[string]composetypes.SecretConfig)
 	}
 	return result, nil
 }
+
+// Configs converts config objects from the Compose type to the engine API type
+func Configs(namespace Namespace, configs map[string]composetypes.ConfigObjConfig) ([]swarm.ConfigSpec, error) {
+	result := []swarm.ConfigSpec{}
+	for name, config := range configs {
+		if config.External.External {
+			continue
+		}
+
+		data, err := ioutil.ReadFile(config.File)
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(result, swarm.ConfigSpec{
+			Annotations: swarm.Annotations{
+				Name:   namespace.Scope(name),
+				Labels: AddStackLabel(namespace, config.Labels),
+			},
+			Data: data,
+		})
+	}
+	return result, nil
+}

cli/compose/convert/compose_test.go

@@ -133,3 +133,34 @@ func TestSecrets(t *testing.T) {
 	}, secret.Labels)
 	assert.Equal(t, []byte(secretText), secret.Data)
 }
+
+func TestConfigs(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+
+	configText := "this is the first config"
+	configFile := tempfile.NewTempFile(t, "convert-configs", configText)
+	defer configFile.Remove()
+
+	source := map[string]composetypes.ConfigObjConfig{
+		"one": {
+			File:   configFile.Name(),
+			Labels: map[string]string{"monster": "mash"},
+		},
+		"ext": {
+			External: composetypes.External{
+				External: true,
+			},
+		},
+	}
+
+	specs, err := Configs(namespace, source)
+	assert.NoError(t, err)
+	require.Len(t, specs, 1)
+	config := specs[0]
+	assert.Equal(t, "foo_one", config.Name)
+	assert.Equal(t, map[string]string{
+		"monster":      "mash",
+		LabelNamespace: "foo",
+	}, config.Labels)
+	assert.Equal(t, []byte(configText), config.Data)
+}

cli/compose/convert/service.go

@@ -37,7 +37,12 @@ func Services(
 		if err != nil {
 			return nil, errors.Wrapf(err, "service %s", service.Name)
 		}
-		serviceSpec, err := convertService(client.ClientVersion(), namespace, service, networks, volumes, secrets)
+		configs, err := convertServiceConfigObjs(client, namespace, service.Configs, config.Configs)
+		if err != nil {
+			return nil, errors.Wrapf(err, "service %s", service.Name)
+		}
+
+		serviceSpec, err := convertService(client.ClientVersion(), namespace, service, networks, volumes, secrets, configs)
 		if err != nil {
 			return nil, errors.Wrapf(err, "service %s", service.Name)
 		}
@@ -54,6 +59,7 @@ func convertService(
 	networkConfigs map[string]composetypes.NetworkConfig,
 	volumes map[string]composetypes.VolumeConfig,
 	secrets []*swarm.SecretReference,
+	configs []*swarm.ConfigReference,
 ) (swarm.ServiceSpec, error) {
 	name := namespace.Scope(service.Name)
 
@@ -277,6 +283,57 @@ func convertServiceSecrets(
 	return servicecli.ParseSecrets(client, refs)
 }
 
+// TODO: fix configs API so that ConfigsAPIClient is not required here
+func convertServiceConfigObjs(
+	client client.ConfigAPIClient,
+	namespace Namespace,
+	configs []composetypes.ServiceConfigObjConfig,
+	configSpecs map[string]composetypes.ConfigObjConfig,
+) ([]*swarm.ConfigReference, error) {
+	refs := []*swarm.ConfigReference{}
+	for _, config := range configs {
+		target := config.Target
+		if target == "" {
+			target = config.Source
+		}
+
+		configSpec, exists := configSpecs[config.Source]
+		if !exists {
+			return nil, errors.Errorf("undefined config %q", config.Source)
+		}
+
+		source := namespace.Scope(config.Source)
+		if configSpec.External.External {
+			source = configSpec.External.Name
+		}
+
+		uid := config.UID
+		gid := config.GID
+		if uid == "" {
+			uid = "0"
+		}
+		if gid == "" {
+			gid = "0"
+		}
+		mode := config.Mode
+		if mode == nil {
+			mode = uint32Ptr(0444)
+		}
+
+		refs = append(refs, &swarm.ConfigReference{
+			File: &swarm.ConfigReferenceFileTarget{
+				Name: target,
+				UID:  uid,
+				GID:  gid,
+				Mode: os.FileMode(*mode),
+			},
+			ConfigName: source,
+		})
+	}
+
+	return servicecli.ParseConfigs(client, refs)
+}
+
 func uint32Ptr(value uint32) *uint32 {
 	return &value
 }

cli/compose/loader/loader.go

@@ -66,67 +66,56 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 	}
 
 	cfg := types.Config{}
-	lookupEnv := func(k string) (string, bool) {
-		v, ok := configDetails.Environment[k]
-		return v, ok
-	}
-	if services, ok := configDict["services"]; ok {
-		servicesConfig, err := interpolation.Interpolate(services.(map[string]interface{}), "service", lookupEnv)
-		if err != nil {
-			return nil, err
-		}
 
-		servicesList, err := LoadServices(servicesConfig, configDetails.WorkingDir, lookupEnv)
-		if err != nil {
-			return nil, err
-		}
+	config, err := interpolateConfig(configDict, configDetails.LookupEnv)
+	if err != nil {
+		return nil, err
+	}
 
-		cfg.Services = servicesList
+	cfg.Services, err = LoadServices(config["services"], configDetails.WorkingDir, configDetails.LookupEnv)
+	if err != nil {
+		return nil, err
 	}
 
-	if networks, ok := configDict["networks"]; ok {
-		networksConfig, err := interpolation.Interpolate(networks.(map[string]interface{}), "network", lookupEnv)
-		if err != nil {
-			return nil, err
-		}
+	cfg.Networks, err = LoadNetworks(config["networks"])
+	if err != nil {
+		return nil, err
+	}
 
-		networksMapping, err := LoadNetworks(networksConfig)
-		if err != nil {
-			return nil, err
-		}
+	cfg.Volumes, err = LoadVolumes(config["volumes"])
+	if err != nil {
+		return nil, err
+	}
 
-		cfg.Networks = networksMapping
+	cfg.Secrets, err = LoadSecrets(config["secrets"], configDetails.WorkingDir)
+	if err != nil {
+		return nil, err
 	}
 
-	if volumes, ok := configDict["volumes"]; ok {
-		volumesConfig, err := interpolation.Interpolate(volumes.(map[string]interface{}), "volume", lookupEnv)
-		if err != nil {
-			return nil, err
-		}
+	cfg.Configs, err = LoadConfigObjs(config["configs"], configDetails.WorkingDir)
+	if err != nil {
+		return nil, err
+	}
 
-		volumesMapping, err := LoadVolumes(volumesConfig)
-		if err != nil {
-			return nil, err
-		}
+	return &cfg, nil
+}
 
-		cfg.Volumes = volumesMapping
-	}
+func interpolateConfig(configDict map[string]interface{}, lookupEnv template.Mapping) (map[string]map[string]interface{}, error) {
+	config := make(map[string]map[string]interface{})
 
-	if secrets, ok := configDict["secrets"]; ok {
-		secretsConfig, err := interpolation.Interpolate(secrets.(map[string]interface{}), "secret", lookupEnv)
-		if err != nil {
-			return nil, err
+	for _, key := range []string{"services", "networks", "volumes", "secrets", "configs"} {
+		section, ok := configDict[key]
+		if !ok {
+			config[key] = make(map[string]interface{})
+			continue
 		}
-
-		secretsMapping, err := LoadSecrets(secretsConfig, configDetails.WorkingDir)
+		var err error
+		config[key], err = interpolation.Interpolate(section.(map[string]interface{}), key, lookupEnv)
 		if err != nil {
 			return nil, err
 		}
-
-		cfg.Secrets = secretsMapping
 	}
-
-	return &cfg, nil
+	return config, nil
 }
 
 // GetUnsupportedProperties returns the list of any unsupported properties that are
@@ -241,7 +230,9 @@ func transformHook(
 	case reflect.TypeOf([]types.ServicePortConfig{}):
 		return transformServicePort(data)
 	case reflect.TypeOf(types.ServiceSecretConfig{}):
-		return transformServiceSecret(data)
+		return transformStringSourceMap(data)
+	case reflect.TypeOf(types.ServiceConfigObjConfig{}):
+		return transformStringSourceMap(data)
 	case reflect.TypeOf(types.StringOrNumberList{}):
 		return transformStringOrNumberList(data)
 	case reflect.TypeOf(map[string]*types.ServiceNetworkConfig{}):
@@ -482,6 +473,25 @@ func LoadSecrets(source map[string]interface{}, workingDir string) (map[string]t
 	return secrets, nil
 }
 
+// LoadConfigObjs produces a ConfigObjConfig map from a compose file Dict
+// the source Dict is not validated if directly used. Use Load() to enable validation
+func LoadConfigObjs(source map[string]interface{}, workingDir string) (map[string]types.ConfigObjConfig, error) {
+	configs := make(map[string]types.ConfigObjConfig)
+	if err := transform(source, &configs); err != nil {
+		return configs, err
+	}
+	for name, config := range configs {
+		if config.External.External && config.External.Name == "" {
+			config.External.Name = name
+			configs[name] = config
+		}
+		if config.File != "" {
+			config.File = absPath(workingDir, config.File)
+		}
+	}
+	return configs, nil
+}
+
 func absPath(workingDir string, filepath string) string {
 	if path.IsAbs(filepath) {
 		return filepath
@@ -544,7 +554,7 @@ func transformServicePort(data interface{}) (interface{}, error) {
 	}
 }
 
-func transformServiceSecret(data interface{}) (interface{}, error) {
+func transformStringSourceMap(data interface{}) (interface{}, error) {
 	switch value := data.(type) {
 	case string:
 		return map[string]interface{}{"source": value}, nil

cli/compose/loader/loader_test.go

@@ -206,12 +206,17 @@ services:
     image: busybox
     credential_spec:
       File: "/foo"
+    configs: [super]
+configs:
+  super:
+    external: true
 `)
 	if !assert.NoError(t, err) {
 		return
 	}
 	assert.Equal(t, len(actual.Services), 1)
 	assert.Equal(t, actual.Services[0].CredentialSpec.File, "/foo")
+	assert.Equal(t, len(actual.Configs), 1)
 }
 
 func TestParseAndLoad(t *testing.T) {