support for comments, for compose file 3.0, and for yaml 2.1

.travis.yml

@@ -6,5 +6,5 @@ python:
   - "3.5-dev"
 before_install:
  - bundle install
-install: "pip install pyaml flake8 pylint"
+install: "pip install ruamel.pyaml flake8 pylint"
 script: "rake test"

README.md

@@ -5,10 +5,30 @@
 [![Docker pulls](https://img.shields.io/docker/pulls/funkwerk/compose_format.svg)](https://hub.docker.com/r/funkwerk/compose_format/)
 # compose_format
 
-Format docker-compose files.
+Format docker-compose files by distilling docker compose best practices into this tool.
+
+## Docker Compose Files are complex
+
+Docker Compose Files could be rather complex.
+If these files are complex, there are multiple ways to write the same thing.
+If there are multiple ways to format these files, these multiple ways will be used.
+Means that it will be not possible to diff your files, cause everybody writes them a bit different.
+
+## Alphabetical order vs. custom order
+
+Sorting would be easy, if everything could be sorted alphabetically.
+But in compose files the first thing mentioned for a service is the `image`.
+`compose_format` aims to distill these compose format best practices into a tool.
+
+## Comments
+
+Usually formatting tools destroy comments. But comments contain valueable TODO-markers or other hints.
+`compose_format` putted effort into supporting comments.
+
+## Support
 
 Note that this small utility is just valid until docker-compose has itself a format functionality.
-Currently docker-compose just support the "config" switch. Which joins multiple compose files and print them in a machine readable form.
+Currently docker-compose just support the "config" switch. Which joins multiple compose files and print them in a machine-readable form.
 
 ## Usage
 
@@ -29,7 +49,14 @@ Use it like:
 `cat docker-compose.yml | docker run -i funkwerk/compose_format`
 
 ## Features
- - Support for Version 2 and Version 1.
+ - Support for Version 3, 2.1, 2, and 1.
+ - Support for Comments
  - Orders Services, Volumes, Networks
  - Orders Definitions
  - Orders Port and Volume Lists
+
+## Contribution
+
+Feel free to add issues or provide Pull Requests.
+Especially when the order in some points violates the best practices.
+This tool should be changed based on the evolving best practices.

compose_format/__init__.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-from collections import OrderedDict
-import pyaml
-from yaml import load
+from ruamel.yaml import RoundTripDumper, RoundTripLoader, dump, load
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 
 class ComposeFormat:
@@ -12,25 +10,52 @@ class ComposeFormat:
         'build',
         'expose', 'ports',
         'net', 'network_mode', 'networks',
+        'deploy',
         'labels',
         'devices',
         'read_only',
+        'healthcheck',
         'env_file', 'environment',
         'cpu_shares', 'cpu_quota', 'cpuset', 'domainname', 'hostname', 'ipc',
         'mac_address', 'mem_limit', 'memswap_limit', 'privileged',
         'depends_on', 'extends', 'external_links',
         'stdin_open', 'user', 'working_dir',
         'extra_hosts', 'restart', 'ulimits', 'tty', 'dns', 'dns_search', 'pid',
         'security_opt', 'cap_add', 'cap_drop', 'cgroup_parent', 'logging', 'log_driver', 'log_opt',
-        'stopsignal',
+        'stopsignal', 'stop_signal', 'stop_grace_period',
+        'sysctls', 'userns_mode',
+        'autodestroy', 'autoredeploy',
+        'deployment_strategy', 'sequential_deployment', 'tags', 'target_num_containers',
+        'roles',
+    ]
+    DEPLOY_ORDER = [
+        'placement', 'replicas', 'mode',
+        'update_config',
+        'resources',
+        'restart_policy',
+        'labels',
+    ]
+    HEALTHCHECK_ORDER = [
+        'test',
+        'interval', 'timeout', 'retries',
+        'disable',
     ]
     BUILD_ORDER = ['context', 'dockerfile', 'args']
 
     ORDERS = {
         'version': TOPLEVEL_ORDER,
+        'services': TOPLEVEL_ORDER,
         'image': SERVICE_ORDER,
         'dockerfile': BUILD_ORDER,
+        'placement': DEPLOY_ORDER,
+        'replicas': DEPLOY_ORDER,
+        'test': HEALTHCHECK_ORDER,
     }
+    NON_SORTABLE_ARRAYS = [
+        'entrypoint',
+        'command',
+        'test',
+    ]
 
     def __init__(self):
         pass
@@ -49,48 +74,56 @@ def format(self, path, replace=False, strict=True):
         return original == formatted
 
     def format_string(self, data, replace=False, strict=True):
-        data = self.reorder(load(data), strict=strict)
+        data = self.reorder(load(data, RoundTripLoader), strict=strict)
+        formatted = dump(data, Dumper=RoundTripDumper, indent=2, width=120)
 
-        def is_legacy_version(data):
-            if 'version' not in data:
-                return True
-            return str(data['version']) != '2' and str(data['version']) != '\'2\''
-
-        vspacing = [1, 0] if is_legacy_version(data) else [0, 1, 0]
-
-        formatted = pyaml.dump(data, vspacing=vspacing, indent=2, width=110, string_val_style='plain')
         return formatted.strip() + '\n'
 
     @staticmethod
     def reorder(data, strict=True):
-        if type(data) is dict or type(data) is OrderedDict:
-            for key in ComposeFormat.ORDERS.keys():
-                if key not in data.keys():
+        if type(data) is CommentedMap:
+            order = ComposeFormat.order_map(list(data.keys()))
+            keys = list(data.keys())
+
+            while ComposeFormat.sorted_by_order(keys, order, strict) != keys:
+                for a, b in zip(ComposeFormat.sorted_by_order(keys, order, strict), keys):
+                    if a == b:
+                        continue
+                    data.move_to_end(b)
+                    break
+                keys = list(data.keys())
+            for key, item in data.items():
+                if key in ComposeFormat.NON_SORTABLE_ARRAYS:
                     continue
-                current_order = ComposeFormat.ORDERS[key]
-
-                def order(item):
-                    key, _ = item
-                    if strict:
-                        assert key in current_order, 'key: {0} not known'.format(key)
-
-                    if key in current_order:
-                        return current_order.index(key)
-                    return len(current_order) + ComposeFormat.name_to_order(key)
-
-                result = {key: ComposeFormat.reorder(value, strict=strict) for key, value in data.items()}
-                result = OrderedDict(sorted(result.items(), key=order))
-
-                return result
-            return {key: ComposeFormat.reorder(value, strict=strict) for key, value in data.items()}
-        if type(data) is list:
-            return sorted([ComposeFormat.reorder(item, strict=strict) for item in data])
-        if len(str(data)) >= 1 and str(data)[0].isdigit():
-            return '\'{0}\''.format(data)
-        if str(data).startswith('{'):
-            return '\'{0}\''.format(data)
+                ComposeFormat.reorder(item, strict)
+            return data
+        if type(data) is CommentedSeq:
+            data.sort()
+            return data
         return data
 
+    @staticmethod
+    def order_map(keys):
+        for key in ComposeFormat.ORDERS.keys():
+            if key in keys:
+                return ComposeFormat.ORDERS[key]
+        return None
+
+    @staticmethod
+    def sorted_by_order(keys, order, strict):
+        if order is None:
+            return sorted(keys)
+
+        def order_function(key):
+            if strict:
+                assert key in order, 'key: {0} not known'.format(key)
+
+            if key in order:
+                return order.index(key)
+            return len(order) + ComposeFormat.name_to_order(key)
+
+        return sorted(keys, key=order_function)
+
     @staticmethod
     def name_to_order(value):
         from functools import reduce

features/comments.feature

@@ -0,0 +1,33 @@
+Feature: Format
+  As a DevOps
+  I want to add comments
+  so that I can mark specialities
+
+  Scenario: Comment at End of Line
+    Given a file named "compose.yml" with:
+      """
+      foo:  # TODO: This service still uses hard-coded configuration
+        image: bar
+      """
+    When I run `bin/compose_format compose.yml`
+    Then it should pass with exactly:
+      """
+      foo:  # TODO: This service still uses hard-coded configuration
+        image: bar
+      """
+
+
+  Scenario: Single Line Comment
+    Given a file named "compose.yml" with:
+      """
+      foo:
+        # Test
+        image: bar
+      """
+    When I run `bin/compose_format compose.yml`
+    Then it should pass with exactly:
+      """
+      foo:
+        # Test
+        image: bar
+      """

features/format.feature

@@ -29,7 +29,6 @@ Feature: Format
       """
       version: '2'
       services:
-
         foo:
           image: bar
       """
@@ -53,16 +52,12 @@ Feature: Format
       """
       version: '2'
       services:
-
         a_service:
           image: image
-
         b_service:
           image: image
-
         c_service:
           image: image
-
         d_service:
           image: image
       """
@@ -103,12 +98,11 @@ Feature: Format
       """
       version: '2'
       services:
-
         service:
           image: image
           ports:
-            - '1'
-            - '2'
+          - '1'
+          - '2'
       """
 
   Scenario: Service Member Order
@@ -134,7 +128,6 @@ Feature: Format
       """
       version: '2'
       services:
-
         foo:
           image: not_relevant
           command: not_relevant
@@ -149,26 +142,6 @@ Feature: Format
           tty: not_relevant
       """
 
-  Scenario: Stringify Numbers
-    Given a file named "compose.yml" with:
-      """
-      version: 2
-      services:
-        foo:
-          ports:
-            - 10000
-      """
-    When I run `bin/compose_format compose.yml`
-    Then it should pass with exactly:
-      """
-      version: '2'
-      services:
-
-        foo:
-          ports:
-            - '10000'
-      """
-
   Scenario: Alphabetic Order for unknown keys (--non_strict)
     Given a file named "compose.yml" with:
       """

features/stack.feature

@@ -0,0 +1,71 @@
+Feature: Stack
+  As a DevOps
+  I want to format docker stack files
+  so that 
+
+  Scenario: Compose Version 3
+    Given a file named "compose.yml" with:
+      """
+      version: "3"
+      services:
+        foo:
+          image: bar
+      """
+    When I run `bin/compose_format compose.yml`
+    Then it should pass with exactly:
+      """
+      version: '3'
+      services:
+        foo:
+          image: bar
+      """
+
+  Scenario: Healthcheck
+    Given a file named "compose.yml" with:
+      """
+      version: "3"
+      services:
+        foo:
+          image: bar
+          healthcheck:
+            interval: 1m30s
+            timeout: 10s
+            retries: 3
+            test: ["CMD", "/bin/true"]
+      """
+    When I run `bin/compose_format compose.yml`
+    Then it should pass with exactly:
+      """
+      version: '3'
+      services:
+        foo:
+          image: bar
+          healthcheck:
+            test: [CMD, /bin/true]
+            interval: 1m30s
+            timeout: 10s
+            retries: 3
+      """
+
+  Scenario: Deploy
+    Given a file named "compose.yml" with:
+      """
+      version: "3"
+      services:
+        foo:
+          image: bar
+          deploy:
+            replicas: 7
+            mode: global
+      """
+    When I run `bin/compose_format compose.yml`
+    Then it should pass with exactly:
+      """
+      version: '3'
+      services:
+        foo:
+          image: bar
+          deploy:
+            replicas: 7
+            mode: global
+      """