[WIP] add experimental wheel support

pythonforandroid/archs.py

@@ -8,6 +8,8 @@
 
 
 class Arch(object):
+    arch = None
+    '''The architecture name.'''
 
     toolchain_prefix = None
     '''The prefix for the toolchain dir in the NDK.'''
@@ -30,6 +32,10 @@ def include_dirs(self):
                 d.format(arch=self))
             for d in self.ctx.include_dirs]
 
+    @property
+    def android_python_abi(self):
+        return 'android{}_{}'.format(self.ctx.android_api, self.arch)
+
     def get_env(self, with_flags_in_cc=True):
         env = {}
 

pythonforandroid/build.py

@@ -46,6 +46,9 @@ class Context(object):
 
     recipe_build_order = None  # Will hold the list of all built recipes
 
+    wheel_sources = ['http://localhost:8080/simple/']
+    build_wheels = False
+
     @property
     def packages_path(self):
         '''Where packages are downloaded before being unpacked'''
@@ -581,7 +584,20 @@ def build_recipes(build_order, python_modules, ctx):
     return
 
 
-def run_pymodules_install(ctx, modules):
+def build_hostpython_path(hostpython, env):
+    hppath = []
+    hppath.append(join(dirname(hostpython), 'Lib'))
+    hppath.append(join(hppath[0], 'site-packages'))
+    builddir = join(dirname(hostpython), 'build')
+    hppath += [join(builddir, d) for d in listdir(builddir)
+               if isdir(join(builddir, d))]
+    if 'PYTHONPATH' in env:
+        env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']])
+    else:
+        env['PYTHONPATH'] = ':'.join(hppath)
+
+
+def run_pymodules_install(ctx, modules, only_binary=False):
     modules = filter(ctx.not_has_package, modules)
 
     if not modules:
@@ -614,10 +630,26 @@ def run_pymodules_install(ctx, modules):
 
         # This bash method is what old-p4a used
         # It works but should be replaced with something better
+        pip = ('import pip.pep425tags as t;'
+               't.supported_tags = [('
+               '    tag[0],'
+               '    "cp27m" if tag[1].startswith("cp") else tag[1],'
+               '    tag[2])'
+               'for tag in t.supported_tags];'
+               'print(t.supported_tags);'
+               'import pip, sys;'
+               'sys.exit(pip.main());')
+        abi = ctx.archs[0].android_python_abi
+        extra_index = ' '.join('--extra-index-url {}'.format(u)
+                               for u in ctx.wheel_sources)
         shprint(sh.bash, '-c', (
             "source venv/bin/activate && env CC=/bin/false CXX=/bin/false "
-            "PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt"
-        ).format(ctx.get_site_packages_dir()))
+            "PYTHONPATH={path} _PYTHON_HOST_PLATFORM={abi} python -c '{pip}' "
+            "-v --cache-dir {cache} install {index} --target '{path}' "
+            "{binary} --no-deps -r requirements.txt"
+        ).format(path=ctx.get_site_packages_dir(), abi=abi, index=extra_index,
+                 cache=realpath(join(ctx.build_dir, 'pipcache')), pip=pip,
+                 binary='--only-binary :all:' if only_binary else ''))
 
 
 def biglink(ctx, arch):

pythonforandroid/recipe.py

@@ -1,15 +1,15 @@
-from os.path import join, dirname, isdir, exists, isfile, split, realpath
+from os.path import join, dirname, isdir, exists, isfile, realpath
 import importlib
 import zipfile
 import glob
 from shutil import rmtree
 from six import PY2, with_metaclass
-
 import sh
 import shutil
 import fnmatch
 from os import listdir, unlink, environ, mkdir, curdir, walk
 from sys import stdout
+
 try:
     from urlparse import urlparse
 except ImportError:
@@ -716,7 +716,32 @@ class PythonRecipe(Recipe):
     This is almost always what you want to do.'''
 
     setup_extra_args = []
-    '''List of extra arugments to pass to setup.py'''
+    '''List of extra arguments to pass to setup.py'''
+
+    use_pip = False
+
+    wheel_name = None
+
+    def download(self):
+        if self.use_pip and not self.ctx.build_wheels:
+            info('Skipping source download, will use wheel')
+            return
+
+        super(PythonRecipe, self).download()
+
+    def unpack(self, arch):
+        if self.use_pip and not self.ctx.build_wheels:
+            info('Skipping source unpack, will use wheel')
+            return
+
+        super(PythonRecipe, self).unpack(arch)
+
+    def apply_patches(self, arch):
+        if self.use_pip and not self.ctx.build_wheels:
+            info('Skipping source patch, will use wheel')
+            return
+
+        super(PythonRecipe, self).apply_patches(arch)
 
     def clean_build(self, arch=None):
         super(PythonRecipe, self).clean_build(arch=arch)
@@ -752,18 +777,17 @@ def hostpython_location(self):
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env = super(PythonRecipe, self).get_recipe_env(arch, with_flags_in_cc)
         if not self.call_hostpython_via_targetpython:
-            hppath = []
-            hppath.append(join(dirname(self.hostpython_location), 'Lib'))
-            hppath.append(join(hppath[0], 'site-packages'))
-            builddir = join(dirname(self.hostpython_location), 'build')
-            hppath += [join(builddir, d) for d in listdir(builddir)
-                       if isdir(join(builddir, d))]
-            if 'PYTHONPATH' in env:
-                env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']])
-            else:
-                env['PYTHONPATH'] = ':'.join(hppath)
+            build_hostpython_path(self.hostpython_location, env)
         return env
 
+    def get_wheel_file(self, arch):
+        wheel_name = self.wheel_name or self.name
+        globber = join(self.get_build_dir(arch), wheel_name + '*.whl')
+        globbed = glob.glob(globber)
+        if globbed:
+            return realpath(globbed[0])
+        return None
+
     def should_build(self, arch):
         print('name is', self.site_packages_name, type(self))
         name = self.site_packages_name
@@ -779,45 +803,37 @@ def build_arch(self, arch):
         '''Install the Python module by calling setup.py install with
         the target Python dir.'''
         super(PythonRecipe, self).build_arch(arch)
+        self.build_wheel(arch)
         self.install_python_package(arch)
 
     def install_python_package(self, arch, name=None, env=None, is_dir=True):
         '''Automate the installation of a Python package (or a cython
         package where the cython components are pre-built).'''
-        # arch = self.filtered_archs[0]  # old kivy-ios way
         if name is None:
             name = self.name
         if env is None:
             env = self.get_recipe_env(arch)
 
-        info('Installing {} into site-packages'.format(self.name))
+        if self.use_pip:
+            info('Installing {} from wheel into site-packages'.format(name))
+            wheel = self.get_wheel_file(arch.arch)
+            if wheel:
+                info('Using wheel file {}'.format(wheel))
+            else:
+                wheel = name
+            run_pymodules_install(self.ctx, [wheel], True)
+            return
+
+        info('Installing {} into site-packages'.format(name))
 
         with current_directory(self.get_build_dir(arch.arch)):
             hostpython = sh.Command(self.hostpython_location)
-            # hostpython = sh.Command('python3.5')
-
 
             if self.ctx.python_recipe.from_crystax:
-                # hppath = join(dirname(self.hostpython_location), 'Lib',
-                #               'site-packages')
-                hpenv = env.copy()
-                # if 'PYTHONPATH' in hpenv:
-                #     hpenv['PYTHONPATH'] = ':'.join([hppath] +
-                #                                    hpenv['PYTHONPATH'].split(':'))
-                # else:
-                #     hpenv['PYTHONPATH'] = hppath
-                # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir()
-                # shprint(hostpython, 'setup.py', 'build',
-                #         _env=hpenv, *self.setup_extra_args)
                 shprint(hostpython, 'setup.py', 'install', '-O2',
                         '--root={}'.format(self.ctx.get_python_install_dir()),
                         '--install-lib=.',
-                        # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions)
-                        _env=hpenv, *self.setup_extra_args)
-                # site_packages_dir = self.ctx.get_site_packages_dir()
-                # built_files = glob.glob(join('build', 'lib*', '*'))
-                # for filen in built_files:
-                #     shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1]))
+                        _env=env, *self.setup_extra_args)
             elif self.call_hostpython_via_targetpython:
                 shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
                         *self.setup_extra_args)
@@ -853,6 +869,20 @@ def install_hostpython_package(self, arch):
                 '--install-lib=Lib/site-packages',
                 _env=env, *self.setup_extra_args)
 
+    def build_wheel(self, arch, env=None, hostpython=None, *args):
+        if self.use_pip and self.ctx.build_wheels:
+            if not env:
+                env = self.get_recipe_env(arch)
+            if not hostpython:
+                hostpython = sh.Command(self.hostpython_location)
+            with current_directory(self.get_build_dir(arch.arch)):
+                shprint(sh.rm, '-rf', 'dist')
+                shprint(hostpython, 'setup.py', 'bdist_wheel',
+                        '--plat-name={}'.format(arch.android_python_abi),
+                        _env=env, *args)
+                wheel = glob.glob(join('dist', self.wheel_name + '*.whl'))[0]
+                shprint(sh.mv, wheel, '.')
+
 
 class CompiledComponentsPythonRecipe(PythonRecipe):
     pre_build_ext = False
@@ -865,9 +895,14 @@ def build_arch(self, arch):
         '''
         Recipe.build_arch(self, arch)
         self.build_compiled_components(arch)
+        self.build_wheel(arch)
         self.install_python_package(arch)
 
     def build_compiled_components(self, arch):
+        if self.use_pip and not self.ctx.build_wheels:
+            info('Skipping compile, will use wheel')
+            return
+
         info('Building compiled components in {}'.format(self.name))
 
         env = self.get_recipe_env(arch)
@@ -913,9 +948,14 @@ def build_arch(self, arch):
         '''
         Recipe.build_arch(self, arch)
         self.build_cython_components(arch)
+        self.build_wheel(arch)
         self.install_python_package(arch)
 
     def build_cython_components(self, arch):
+        if self.use_pip and not self.ctx.build_wheels:
+            info('Skipping build, will use wheel')
+            return
+
         info('Cythonizing anything necessary in {}'.format(self.name))
 
         env = self.get_recipe_env(arch)
@@ -1050,3 +1090,6 @@ def prebuild_arch(self, arch):
     # def ctx(self, ctx):
     #     self._ctx = ctx
     #     ctx.python_recipe = self
+
+from pythonforandroid.build import run_pymodules_install, build_hostpython_path
+

pythonforandroid/recipes/kivy/__init__.py

@@ -1,22 +1,23 @@
 
-from pythonforandroid.toolchain import CythonRecipe, shprint, current_directory, ArchARM
-from os.path import exists, join
-import sh
-import glob
+from pythonforandroid.toolchain import CythonRecipe
+from os.path import join
 
 
 class KivyRecipe(CythonRecipe):
-    # version = 'stable'
     version = 'master'
     url = 'https://github.com/kivy/kivy/archive/{version}.zip'
     name = 'kivy'
 
-    depends = [('sdl2', 'pygame'), 'pyjnius']
+    depends = [('sdl2', 'pygame'), 'pyjnius', 'setuptools', 'wheel']
 
-    # patches = ['setargv.patch']
+    call_hostpython_via_targetpython = False
+
+    use_pip = True
+    wheel_name = 'Kivy'
 
     def get_recipe_env(self, arch):
         env = super(KivyRecipe, self).get_recipe_env(arch)
+        env['KIVY_USE_SETUPTOOLS'] = '1'
         if 'sdl2' in self.ctx.recipe_build_order:
             env['USE_SDL2'] = '1'
             env['KIVY_SDL2_PATH'] = ':'.join([
@@ -27,4 +28,5 @@ def get_recipe_env(self, arch):
                 ])
         return env
 
+
 recipe = KivyRecipe()

pythonforandroid/recipes/sqlalchemy/__init__.py

@@ -7,9 +7,13 @@ class SQLAlchemyRecipe(CompiledComponentsPythonRecipe):
     version = '1.0.9'
     url = 'https://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-{version}.tar.gz'
 
-    depends = [('python2', 'python3'), 'setuptools']
+    depends = [('python2', 'python3'), 'setuptools', 'wheel']
 
     patches = ['zipsafe.patch']
 
+    call_hostpython_via_targetpython = False
+    use_pip = True
+    wheel_name = 'SQLAlchemy'
+
 
 recipe = SQLAlchemyRecipe()

pythonforandroid/recipes/twisted/__init__.py

@@ -15,11 +15,14 @@ class TwistedRecipe(CythonRecipe):
     version = '15.4.0'
     url = 'https://pypi.python.org/packages/source/T/Twisted/Twisted-{version}.tar.bz2'
 
-    depends = ['setuptools', 'zope_interface']
+    depends = ['setuptools', 'zope_interface', 'wheel']
 
     call_hostpython_via_targetpython = False
     install_in_hostpython = True
 
+    use_pip = True
+    wheel_name = 'Twisted'
+
     def prebuild_arch(self, arch):
         super(TwistedRecipe, self).prebuild_arch(arch)
         # TODO Need to whitelist tty.pyo and termios.so here

pythonforandroid/recipes/wheel/__init__.py

@@ -0,0 +1,16 @@
+
+from pythonforandroid.toolchain import PythonRecipe
+
+
+class WheelRecipe(PythonRecipe):
+    version = '0.29.0'
+    url = 'https://pypi.python.org/packages/source/w/wheel/wheel-{version}.tar.gz'
+
+    depends = [('python2', 'python3crystax')]
+
+    call_hostpython_via_targetpython = False
+    install_in_targetpython = False
+    install_in_hostpython = True
+
+
+recipe = WheelRecipe()

pythonforandroid/toolchain.py

@@ -87,6 +87,7 @@ def wrapper_func(self, args):
                                       user_ndk_ver=self.ndk_version)
         dist = self._dist
         dist_args, args = parse_dist_args(args)
+        ctx.build_wheels = dist_args.build_wheels
         if dist.needs_build:
             info_notify('No dist exists that meets your requirements, '
                         'so one will be built.')
@@ -146,14 +147,20 @@ def build_dist_from_args(ctx, dist, args):
 
 def parse_dist_args(args_list):
     parser = argparse.ArgumentParser(
-            description='Create a newAndroid project')
+            description='Create a new Android project')
     parser.add_argument(
             '--bootstrap',
             help=('The name of the bootstrap type, \'pygame\' '
                   'or \'sdl2\', or leave empty to let a '
                   'bootstrap be chosen automatically from your '
                   'requirements.'),
             default=None)
+    parser.add_argument(
+            '--build-wheels',
+            help='Build wheels instead of downloading',
+            default=False,
+            dest='build_wheels',
+            action='store_true')
     args, unknown = parser.parse_known_args(args_list)
     return args, unknown