feat(dependencies): git dependencies allow submodules (#987)

* feat(depdencies): git depds allow submodules

* tests: add tests for fetching git deps with submodules

* docs: update docs with submodule support example

* tests: provide correct test descriptions

* feat: enable submodules by default

* docs: update docs with new submodule behavior

* refactor: remove debugging and restructure if block

docs/pages/external_dependencies.md

@@ -4,9 +4,9 @@
 
 Supported dependencies types are:
 
-- [git](#git)
-- [http](#http)
-- [helm](#helm)
+- [git](#defining-dependencies)
+- [http](#defining-dependencies)
+- [helm](#defining-dependencies)
 
 
 ## Usage
@@ -65,9 +65,11 @@ Use the `force-fetch` option to force overwrite your local files in the `output_
 
 Kapitan also supports caching Use the `--cache` flag to cache the fetched items in the `.dependency_cache` directory in the root project directory.
 
-```shell
-kapitan compile --cache --fetch
-```
+    ```shell
+    kapitan compile --cache --fetch
+    ```
+
+### Defining dependencies
 
 === "git"
 
@@ -82,11 +84,13 @@ kapitan compile --cache --fetch
           source: git_url # mkdocs (1)!
           subdir: relative/path/from/repo/root (optional) # mkdocs (2)!
           ref: tag, commit, branch etc. (optional) # mkdocs (3)!
+          submodules: true/false (optional) # mkdocs (4)!
     ```
 
     1. Git types can fetch external `git` repositories through either HTTP/HTTPS or SSH URLs. 
     2. Optional supports for cloning just a sub-directory
     3. Optional support for accessing them in specific commits and branches (refs).
+    4. Optional support to disable fetching the submodules of a repo.
 
     !!! note
 
@@ -107,14 +111,15 @@ kapitan compile --cache --fetch
           source: [email protected]:kapicorp/kapitan.git
           subdir: kapitan
           ref: master
+          submodules: true
         compile:
         - input_paths:
           - source/kapitan/version.py
           input_type: jinja2 # just to copy the file over to target
           output_path: .
     ```
 
-=== "`http`"
+=== "http"
 
     ### Syntax
 
@@ -132,7 +137,7 @@ kapitan compile --cache --fetch
     2. http[s] types can fetch external dependencies available at `http://` or `https://` URL.
     3. archive mode: download and unpack
 
-    ### Examples
+    ### Example
 
     === "Single file"
 
@@ -157,7 +162,7 @@ kapitan compile --cache --fetch
           output_path: .
     ```
 
-=== "`helm`"
+=== "helm"
 
     ### Syntax
 
@@ -181,7 +186,7 @@ kapitan compile --cache --fetch
 
     `source` can be either the URL to a chart repository, or the URL to a chart on an OCI registry (supported since Helm 3.8.0).
 
-    ### Examples
+    ### Example
 
     If we want to download the prometheus helm chart we simply add the dependency to the monitoring target.
     We want a specific version `11.3.0` so we put that in.

kapitan/dependency_manager/base.py

@@ -114,6 +114,11 @@ def fetch_git_dependency(dep_mapping, save_dir, force, item_type="Dependency"):
         else:
             repo.git.checkout("master")  # default ref
 
+        # initialising submodules
+        if "submodules" not in dep or dep["submodules"]:
+            for submodule in repo.submodules:
+                submodule.update(init=True)
+
         if "subdir" in dep:
             sub_dir = dep["subdir"]
             full_subdir = os.path.join(cached_repo_path, sub_dir)

kapitan/targets.py

@@ -658,6 +658,7 @@ def valid_target_obj(target_obj, require_compile=True):
                         "unpack": {"type": "boolean"},
                         "version": {"type": "string"},
                         "force_fetch": {"type": "boolean"},
+                        "submodules": {"type": "boolean"},
                     },
                     "required": ["type", "output_path", "source"],
                     "additionalProperties": False,

tests/test_dependency_manager.py

@@ -51,7 +51,7 @@ def test_fetch_http_sources(self):
         rmtree(temp_dir)
 
     def test_fetch_git_sources(self):
-        "Tests clonning git repo"
+        "Tests cloning git repo"
         temp_dir = tempfile.mkdtemp()
         repo_dir = os.path.join(temp_dir, "7a8f3940kapitan.git")
         # TODO: also test git ssh urls
@@ -62,7 +62,7 @@ def test_fetch_git_sources(self):
 
     def test_clone_repo_subdir(self):
         """
-        Tests clonning git repo and copy its' subdir
+        Tests cloning git repo and copy its' subdir
         """
         temp_dir = tempfile.mkdtemp()
         output_dir = tempfile.mkdtemp()
@@ -79,6 +79,80 @@ def test_clone_repo_subdir(self):
         rmtree(temp_dir)
         rmtree(output_dir)
 
+    def test_clone_repo_submodules_false(self):
+        """
+        Tests cloning git repo and check that submodule folder is empty
+        """
+        temp_dir = tempfile.mkdtemp()
+        output_dir = tempfile.mkdtemp()
+        source = "https://github.com/kapicorp/kapitan.git"
+        dep = [
+            {
+                "output_path": output_dir,
+                "ref": "master",
+                "submodules": False,
+            }
+        ]
+        fetch_git_dependency((source, dep), temp_dir, force=False)
+        self.assertEqual(os.listdir(os.path.join(output_dir, "kapitan", "reclass")), [])
+        rmtree(temp_dir)
+        rmtree(output_dir)
+
+    def test_clone_repo_without_submodules(self):
+        """
+        Tests cloning a git repo without any submodules
+        """
+        temp_dir = tempfile.mkdtemp()
+        output_dir = tempfile.mkdtemp()
+        source = "https://github.com/kapicorp/reclass.git"
+        dep = [
+            {
+                "output_path": output_dir,
+                "ref": "master",
+            }
+        ]
+        fetch_git_dependency((source, dep), temp_dir, force=False)
+        self.assertTrue(os.path.isdir(os.path.join(output_dir, "reclass")))
+        rmtree(temp_dir)
+        rmtree(output_dir)
+
+    def test_clone_repo_with_submodules(self):
+        """
+        Tests cloning git repo and initialize its' submodule
+        """
+        temp_dir = tempfile.mkdtemp()
+        output_dir = tempfile.mkdtemp()
+        source = "https://github.com/kapicorp/kapitan.git"
+        dep = [
+            {
+                "output_path": output_dir,
+                "ref": "master",
+            }
+        ]
+        fetch_git_dependency((source, dep), temp_dir, force=False)
+        self.assertTrue(os.listdir(os.path.join(output_dir, "kapitan", "reclass")))
+        rmtree(temp_dir)
+        rmtree(output_dir)
+
+    def test_clone_repo_with_submodule_subdir(self):
+        """
+        Tests cloning subdir in a git repo and initialize its' submodule
+        """
+        temp_dir = tempfile.mkdtemp()
+        output_dir = tempfile.mkdtemp()
+        source = "https://github.com/kapicorp/kapitan.git"
+        dep = [
+            {
+                "output_path": output_dir,
+                "ref": "master",
+                "subdir": "kapitan",
+            }
+        ]
+        fetch_git_dependency((source, dep), temp_dir, force=False)
+        self.assertTrue(os.listdir(os.path.join(output_dir, "reclass")))
+        rmtree(temp_dir)
+        rmtree(output_dir)
+
     def test_fetch_helm_chart(self):
         """
         Tests fetching helm chart