Merge pull request #3 from pelson/phased_implementation

Added a DirectoryDestination for new build distributions to be placed into, and provided a ``list_metas`` function

conda-build-all.recipe/meta.yaml

@@ -20,6 +20,7 @@ requirements:
         - setuptools
         - conda
         - conda-build
+        - anaconda-client
 
 test:
   imports:

conda_build_all/artefact_destination.py

@@ -10,6 +10,7 @@
 
 import logging
 import os
+import shutil
 import subprocess
 from argparse import Namespace
 
@@ -48,6 +49,20 @@ def make_available(self, meta, built_dist_path, just_built):
         pass
 
 
+class DirectoryDestination(ArtefactDestination):
+    def __init__(self, directory):
+        self.directory = os.path.abspath(os.path.expanduser(directory))
+        if not os.path.exists(self.directory):
+            os.makedirs(self.directory)
+        if not os.path.isdir(self.directory):
+            raise IOError("The destination provided is not a directory.")
+
+    def make_available(self, meta, built_dist_path, just_built):
+        if just_built:
+            print(meta, built_dist_path, just_built)
+            shutil.copy(built_dist_path, self.directory)
+
+
 class AnacondaClientChannelDest(ArtefactDestination):
     def __init__(self, token, owner, channel):
         self.token = token

conda_build_all/builder.py

@@ -46,21 +46,33 @@ def distribution_exists(binstar_cli, owner, metadata):
     return exists
 
 
-def fetch_metas(directory):
+def list_metas(directory, max_depth=0):
     """
     Get the build metadata of all recipes in a directory.
 
-    The recipes will be sorted by the order of their directory name.
+    The order of metas from this function is not guaranteed.
+
+    Parameters
+    ----------
+    directory
+        Where to start looking for metas using os.walk.
+    max_depth : int
+        How deep to recurse when looking for recipes. 
+        A value ``<=0`` will recurse indefinitely. A value of 1
+        will look in the given directory for a meta.yaml.
+        (default: 0)
 
     """
     packages = []
-    for package_name in sorted(os.listdir(directory)):
-        package_dir = os.path.join(directory, package_name)
-        meta_yaml = os.path.join(package_dir, 'meta.yaml')
-
-        if os.path.isdir(package_dir) and os.path.exists(meta_yaml):
-            packages.append(MetaData(package_dir))
-
+    current_depth = max_depth
+    root = os.path.normpath(directory)
+    for new_root, dirs, files in os.walk(root):
+        depth = new_root[len(root):].count(os.path.sep) + 1
+        if max_depth > 0 and depth >= max_depth:
+            del dirs[:]
+
+        if 'meta.yaml' in files:
+            packages.append(MetaData(new_root))
     return packages
 
 
@@ -117,7 +129,7 @@ def fetch_all_metas(self):
 
         """
         conda_recipes_directory = os.path.abspath(os.path.expanduser(self.conda_recipes_directory))
-        recipe_metas = fetch_metas(conda_recipes_directory)
+        recipe_metas = list_metas(conda_recipes_directory)
         recipe_metas = sort_dependency_order(recipe_metas)
         return recipe_metas
 
@@ -149,7 +161,7 @@ def find_existing_built_dists(self, recipe_metas):
     def build(self, meta):
         print('Building ', meta.dist())
         with meta.vn_context():
-            build.build(meta.meta)
+            return bldpkg_path(build.build(meta.meta))
 
     def compute_build_distros(self, index, recipes):
         """

conda_build_all/cli.py

@@ -20,16 +20,11 @@ def main():
                         help='Skip a build if the equivalent disribution is already available in the specified directory.')
     parser.add_argument('--no-inspect-conda-bld-directory', default=True, action='store_false',
                         help='Skip a build if the equivalent disribution is already in the conda-bld directory.')
-    parser.add_argument('--build-artefact-destinations', nargs='*', default=[],
-                        help=('The channel(s) to upload built distributions to. It is '
-                              'rare to specify this without the --inspect-channel argument. '
-                              'If a file:// channel, the build will be copied to the directory. '
-                              'If a url:// channel, the build will be uploaded with the anaconda '
-                              'client functionality.'))
+    parser.add_argument('--artefact-directory',
+                        help='A directory for any newly built distributions to be placed.')
     parser.add_argument('--upload-channels', nargs='*', default=[],
                         help='The channel(s) to upload built distributions to.')
 
-
     parser.add_argument("--matrix-conditions", nargs='*', default=[],
                         help="Extra conditions for computing the build matrix.")
     parser.add_argument("--matrix-max-n-major-versions", default=2, type=int,
@@ -56,6 +51,8 @@ def main():
     artefact_destinations = []
     for channel in args.upload_channels:
         artefact_destinations.append(artefact_dest.AnacondaClientChannelDest.from_spec(channel))
+    if args.artefact_directory:
+        artefact_destinations.append(artefact_dest.DirectoryDestination(args.artefact_directory))
 
     artefact_dest.log.setLevel(logging.INFO)
     artefact_dest.log.addHandler(logging.StreamHandler())

conda_build_all/tests/integration/test_builder.py

@@ -7,6 +7,7 @@
 
 from conda_build.metadata import MetaData
 
+from conda_build_all.resolved_distribution import ResolvedDistribution
 from conda_build_all.builder import Builder
 from conda_build_all.tests.unit.dummy_index import DummyIndex
 
@@ -25,12 +26,28 @@ def tearDown(self):
 
     def write_meta(self, recipe_dir_name, spec):
         recipe_dir = os.path.join(self.recipes_root_dir, recipe_dir_name)
-        os.mkdir(recipe_dir)
+        if not os.path.exists(recipe_dir):
+            os.makedirs(recipe_dir)
         with open(os.path.join(recipe_dir, 'meta.yaml'), 'w') as fh:
             fh.write(textwrap.dedent(spec))
         return MetaData(recipe_dir)
 
 
+class Test_build(RecipeCreatingUnit):
+    def test(self):
+        pkg1 = self.write_meta('pkg1', """
+                    package:
+                        name: pkg1
+                        version: 1.0
+                    """)
+        pkg1_resolved = ResolvedDistribution(pkg1, (()))
+        builder = Builder(None, None, None, None, None)
+        r = builder.build(pkg1_resolved)
+        self.assertTrue(os.path.exists(r))
+        self.assertEqual(os.path.abspath(r), r)
+        self.assertEqual(os.path.basename(r), 'pkg1-1.0-0.tar.bz2')
+
+
 class Test__find_existing_built_dists(RecipeCreatingUnit):
     def make_channel(self, all_metas):
         for meta in all_metas:

conda_build_all/tests/unit/test_artefact_destination.py

@@ -3,13 +3,16 @@
 import logging
 import mock
 import os
+import shutil
 import sys
+import tempfile
 import unittest
 
 
 from conda_build_all.tests.unit.dummy_index import DummyIndex, DummyPackage
 from conda_build_all.artefact_destination import (ArtefactDestination,
-                                                  AnacondaClientChannelDest)
+                                                  AnacondaClientChannelDest,
+                                                  DirectoryDestination)
 import conda_build_all.artefact_destination
 
 
@@ -110,5 +113,25 @@ def test_from_spec_owner_and_channel(self):
         self.assertEqual(dest.channel, 'my_channel')
 
 
+class Test_DirectoryDestination(unittest.TestCase):
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp(prefix='recipes')
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def test_not_copying(self):
+        dd = DirectoryDestination(self.tmp_dir)
+        dd.make_available(mock.sentinel.dummy_meta, mock.sentinel.dummy_path,
+                          just_built=False)
+
+    def test_copying(self):
+        dd = DirectoryDestination(self.tmp_dir)
+        with mock.patch('shutil.copy') as copy:
+            dd.make_available(mock.sentinel.dummy_meta, mock.sentinel.dummy_path,
+                              just_built=True)
+        copy.assert_called_once_with(mock.sentinel.dummy_path, self.tmp_dir)
+
+
 if __name__ == '__main__':
     unittest.main()

conda_build_all/tests/unit/test_builder.py

@@ -0,0 +1,61 @@
+import os
+import shutil
+import tempfile
+import unittest
+import textwrap
+
+import conda_build.config
+from conda_build.metadata import MetaData
+
+from conda_build_all.builder import list_metas
+from conda_build_all.tests.integration.test_builder import RecipeCreatingUnit
+
+
+class Test_list_metas(RecipeCreatingUnit):
+    def setUp(self):
+        super(Test_list_metas, self).setUp()
+        m1 = self.write_meta('m1', """
+            package:
+                name: m1
+            """)
+        m2 = self.write_meta('.', """
+            package:
+                name: m2
+            """)
+        m3 = self.write_meta('d1/d2/d3/meta3', """
+            package:
+                name: m3
+            """)
+        m4 = self.write_meta('da1/da2/da3/meta4', """
+            package:
+                name: m4
+            """)
+
+    def test_depth_0(self):
+        metas = list_metas(self.recipes_root_dir, max_depth=0)
+        names = [meta.name() for meta in metas]
+        self.assertEqual(sorted(names), ['m1', 'm2', 'm3', 'm4'])
+
+    def test_depth_m1(self):
+        metas = list_metas(self.recipes_root_dir, max_depth=-1)
+        names = [meta.name() for meta in metas]
+        self.assertEqual(sorted(names), ['m1', 'm2', 'm3', 'm4'])
+
+    def test_depth_1(self):
+        metas = list_metas(self.recipes_root_dir, max_depth=1)
+        names = [meta.name() for meta in metas]
+        self.assertEqual(sorted(names), ['m2'])
+
+    def test_depth_2(self):
+        metas = list_metas(self.recipes_root_dir, max_depth=2)
+        names = [meta.name() for meta in metas]
+        self.assertEqual(sorted(names), ['m1', 'm2'])
+
+    def test_default_depth(self):
+        metas = list_metas(self.recipes_root_dir)
+        names = [meta.name() for meta in metas]
+        self.assertEqual(sorted(names), ['m1', 'm2', 'm3', 'm4'])
+
+
+if __name__ == '__main__':
+    unittest.main()