From 4f30fd05dc459784c64d93d9f4a7665f88159fd6 Mon Sep 17 00:00:00 2001 From: Ryan Pessa Date: Sat, 4 Jun 2016 18:08:00 -0500 Subject: [PATCH] add experimental wheel support --- pythonforandroid/archs.py | 6 + pythonforandroid/build.py | 38 +++++- pythonforandroid/recipe.py | 111 ++++++++++++------ pythonforandroid/recipes/kivy/__init__.py | 16 +-- .../recipes/sqlalchemy/__init__.py | 6 +- pythonforandroid/recipes/twisted/__init__.py | 5 +- pythonforandroid/recipes/wheel/__init__.py | 16 +++ pythonforandroid/toolchain.py | 9 +- 8 files changed, 160 insertions(+), 47 deletions(-) create mode 100644 pythonforandroid/recipes/wheel/__init__.py diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index c76442f88c..dd2f8e3c42 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -8,6 +8,8 @@ class Arch(object): + arch = None + '''The architecture name.''' toolchain_prefix = None '''The prefix for the toolchain dir in the NDK.''' @@ -30,6 +32,10 @@ def include_dirs(self): d.format(arch=self)) for d in self.ctx.include_dirs] + @property + def android_python_abi(self): + return 'android{}_{}'.format(self.ctx.android_api, self.arch) + def get_env(self, with_flags_in_cc=True): env = {} diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index c9f9d99d27..7b9bec063f 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -46,6 +46,9 @@ class Context(object): recipe_build_order = None # Will hold the list of all built recipes + wheel_sources = ['http://localhost:8080/simple/'] + build_wheels = False + @property def packages_path(self): '''Where packages are downloaded before being unpacked''' @@ -581,7 +584,20 @@ def build_recipes(build_order, python_modules, ctx): return -def run_pymodules_install(ctx, modules): +def build_hostpython_path(hostpython, env): + hppath = [] + hppath.append(join(dirname(hostpython), 'Lib')) + hppath.append(join(hppath[0], 'site-packages')) + builddir = join(dirname(hostpython), 'build') + hppath += [join(builddir, d) for d in listdir(builddir) + if isdir(join(builddir, d))] + if 'PYTHONPATH' in env: + env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']]) + else: + env['PYTHONPATH'] = ':'.join(hppath) + + +def run_pymodules_install(ctx, modules, only_binary=False): modules = filter(ctx.not_has_package, modules) if not modules: @@ -614,10 +630,26 @@ def run_pymodules_install(ctx, modules): # This bash method is what old-p4a used # It works but should be replaced with something better + pip = ('import pip.pep425tags as t;' + 't.supported_tags = [(' + ' tag[0],' + ' "cp27m" if tag[1].startswith("cp") else tag[1],' + ' tag[2])' + 'for tag in t.supported_tags];' + 'print(t.supported_tags);' + 'import pip, sys;' + 'sys.exit(pip.main());') + abi = ctx.archs[0].android_python_abi + extra_index = ' '.join('--extra-index-url {}'.format(u) + for u in ctx.wheel_sources) shprint(sh.bash, '-c', ( "source venv/bin/activate && env CC=/bin/false CXX=/bin/false " - "PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt" - ).format(ctx.get_site_packages_dir())) + "PYTHONPATH={path} _PYTHON_HOST_PLATFORM={abi} python -c '{pip}' " + "-v --cache-dir {cache} install {index} --target '{path}' " + "{binary} --no-deps -r requirements.txt" + ).format(path=ctx.get_site_packages_dir(), abi=abi, index=extra_index, + cache=realpath(join(ctx.build_dir, 'pipcache')), pip=pip, + binary='--only-binary :all:' if only_binary else '')) def biglink(ctx, arch): diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 97e7f076db..6a0888fd98 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,15 +1,15 @@ -from os.path import join, dirname, isdir, exists, isfile, split, realpath +from os.path import join, dirname, isdir, exists, isfile, realpath import importlib import zipfile import glob from shutil import rmtree from six import PY2, with_metaclass - import sh import shutil import fnmatch from os import listdir, unlink, environ, mkdir, curdir, walk from sys import stdout + try: from urlparse import urlparse except ImportError: @@ -716,7 +716,32 @@ class PythonRecipe(Recipe): This is almost always what you want to do.''' setup_extra_args = [] - '''List of extra arugments to pass to setup.py''' + '''List of extra arguments to pass to setup.py''' + + use_pip = False + + wheel_name = None + + def download(self): + if self.use_pip and not self.ctx.build_wheels: + info('Skipping source download, will use wheel') + return + + super(PythonRecipe, self).download() + + def unpack(self, arch): + if self.use_pip and not self.ctx.build_wheels: + info('Skipping source unpack, will use wheel') + return + + super(PythonRecipe, self).unpack(arch) + + def apply_patches(self, arch): + if self.use_pip and not self.ctx.build_wheels: + info('Skipping source patch, will use wheel') + return + + super(PythonRecipe, self).apply_patches(arch) def clean_build(self, arch=None): super(PythonRecipe, self).clean_build(arch=arch) @@ -752,18 +777,17 @@ def hostpython_location(self): def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super(PythonRecipe, self).get_recipe_env(arch, with_flags_in_cc) if not self.call_hostpython_via_targetpython: - hppath = [] - hppath.append(join(dirname(self.hostpython_location), 'Lib')) - hppath.append(join(hppath[0], 'site-packages')) - builddir = join(dirname(self.hostpython_location), 'build') - hppath += [join(builddir, d) for d in listdir(builddir) - if isdir(join(builddir, d))] - if 'PYTHONPATH' in env: - env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']]) - else: - env['PYTHONPATH'] = ':'.join(hppath) + build_hostpython_path(self.hostpython_location, env) return env + def get_wheel_file(self, arch): + wheel_name = self.wheel_name or self.name + globber = join(self.get_build_dir(arch), wheel_name + '*.whl') + globbed = glob.glob(globber) + if globbed: + return realpath(globbed[0]) + return None + def should_build(self, arch): print('name is', self.site_packages_name, type(self)) name = self.site_packages_name @@ -779,45 +803,37 @@ def build_arch(self, arch): '''Install the Python module by calling setup.py install with the target Python dir.''' super(PythonRecipe, self).build_arch(arch) + self.build_wheel(arch) self.install_python_package(arch) def install_python_package(self, arch, name=None, env=None, is_dir=True): '''Automate the installation of a Python package (or a cython package where the cython components are pre-built).''' - # arch = self.filtered_archs[0] # old kivy-ios way if name is None: name = self.name if env is None: env = self.get_recipe_env(arch) - info('Installing {} into site-packages'.format(self.name)) + if self.use_pip: + info('Installing {} from wheel into site-packages'.format(name)) + wheel = self.get_wheel_file(arch.arch) + if wheel: + info('Using wheel file {}'.format(wheel)) + else: + wheel = name + run_pymodules_install(self.ctx, [wheel], True) + return + + info('Installing {} into site-packages'.format(name)) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.hostpython_location) - # hostpython = sh.Command('python3.5') - if self.ctx.python_recipe.from_crystax: - # hppath = join(dirname(self.hostpython_location), 'Lib', - # 'site-packages') - hpenv = env.copy() - # if 'PYTHONPATH' in hpenv: - # hpenv['PYTHONPATH'] = ':'.join([hppath] + - # hpenv['PYTHONPATH'].split(':')) - # else: - # hpenv['PYTHONPATH'] = hppath - # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir() - # shprint(hostpython, 'setup.py', 'build', - # _env=hpenv, *self.setup_extra_args) shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', - # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions) - _env=hpenv, *self.setup_extra_args) - # site_packages_dir = self.ctx.get_site_packages_dir() - # built_files = glob.glob(join('build', 'lib*', '*')) - # for filen in built_files: - # shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1])) + _env=env, *self.setup_extra_args) elif self.call_hostpython_via_targetpython: shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, *self.setup_extra_args) @@ -853,6 +869,20 @@ def install_hostpython_package(self, arch): '--install-lib=Lib/site-packages', _env=env, *self.setup_extra_args) + def build_wheel(self, arch, env=None, hostpython=None, *args): + if self.use_pip and self.ctx.build_wheels: + if not env: + env = self.get_recipe_env(arch) + if not hostpython: + hostpython = sh.Command(self.hostpython_location) + with current_directory(self.get_build_dir(arch.arch)): + shprint(sh.rm, '-rf', 'dist') + shprint(hostpython, 'setup.py', 'bdist_wheel', + '--plat-name={}'.format(arch.android_python_abi), + _env=env, *args) + wheel = glob.glob(join('dist', self.wheel_name + '*.whl'))[0] + shprint(sh.mv, wheel, '.') + class CompiledComponentsPythonRecipe(PythonRecipe): pre_build_ext = False @@ -865,9 +895,14 @@ def build_arch(self, arch): ''' Recipe.build_arch(self, arch) self.build_compiled_components(arch) + self.build_wheel(arch) self.install_python_package(arch) def build_compiled_components(self, arch): + if self.use_pip and not self.ctx.build_wheels: + info('Skipping compile, will use wheel') + return + info('Building compiled components in {}'.format(self.name)) env = self.get_recipe_env(arch) @@ -913,9 +948,14 @@ def build_arch(self, arch): ''' Recipe.build_arch(self, arch) self.build_cython_components(arch) + self.build_wheel(arch) self.install_python_package(arch) def build_cython_components(self, arch): + if self.use_pip and not self.ctx.build_wheels: + info('Skipping build, will use wheel') + return + info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) @@ -1050,3 +1090,6 @@ def prebuild_arch(self, arch): # def ctx(self, ctx): # self._ctx = ctx # ctx.python_recipe = self + +from pythonforandroid.build import run_pymodules_install, build_hostpython_path + diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index d92fbe9367..dc26dcb8f8 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -1,22 +1,23 @@ -from pythonforandroid.toolchain import CythonRecipe, shprint, current_directory, ArchARM -from os.path import exists, join -import sh -import glob +from pythonforandroid.toolchain import CythonRecipe +from os.path import join class KivyRecipe(CythonRecipe): - # version = 'stable' version = 'master' url = 'https://github.com/kivy/kivy/archive/{version}.zip' name = 'kivy' - depends = [('sdl2', 'pygame'), 'pyjnius'] + depends = [('sdl2', 'pygame'), 'pyjnius', 'setuptools', 'wheel'] - # patches = ['setargv.patch'] + call_hostpython_via_targetpython = False + + use_pip = True + wheel_name = 'Kivy' def get_recipe_env(self, arch): env = super(KivyRecipe, self).get_recipe_env(arch) + env['KIVY_USE_SETUPTOOLS'] = '1' if 'sdl2' in self.ctx.recipe_build_order: env['USE_SDL2'] = '1' env['KIVY_SDL2_PATH'] = ':'.join([ @@ -27,4 +28,5 @@ def get_recipe_env(self, arch): ]) return env + recipe = KivyRecipe() diff --git a/pythonforandroid/recipes/sqlalchemy/__init__.py b/pythonforandroid/recipes/sqlalchemy/__init__.py index 79e0ebf8fc..b4e2518e96 100644 --- a/pythonforandroid/recipes/sqlalchemy/__init__.py +++ b/pythonforandroid/recipes/sqlalchemy/__init__.py @@ -7,9 +7,13 @@ class SQLAlchemyRecipe(CompiledComponentsPythonRecipe): version = '1.0.9' url = 'https://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-{version}.tar.gz' - depends = [('python2', 'python3'), 'setuptools'] + depends = [('python2', 'python3'), 'setuptools', 'wheel'] patches = ['zipsafe.patch'] + call_hostpython_via_targetpython = False + use_pip = True + wheel_name = 'SQLAlchemy' + recipe = SQLAlchemyRecipe() diff --git a/pythonforandroid/recipes/twisted/__init__.py b/pythonforandroid/recipes/twisted/__init__.py index cb28173a2d..d7719daaf4 100644 --- a/pythonforandroid/recipes/twisted/__init__.py +++ b/pythonforandroid/recipes/twisted/__init__.py @@ -15,11 +15,14 @@ class TwistedRecipe(CythonRecipe): version = '15.4.0' url = 'https://pypi.python.org/packages/source/T/Twisted/Twisted-{version}.tar.bz2' - depends = ['setuptools', 'zope_interface'] + depends = ['setuptools', 'zope_interface', 'wheel'] call_hostpython_via_targetpython = False install_in_hostpython = True + use_pip = True + wheel_name = 'Twisted' + def prebuild_arch(self, arch): super(TwistedRecipe, self).prebuild_arch(arch) # TODO Need to whitelist tty.pyo and termios.so here diff --git a/pythonforandroid/recipes/wheel/__init__.py b/pythonforandroid/recipes/wheel/__init__.py new file mode 100644 index 0000000000..baac7235ec --- /dev/null +++ b/pythonforandroid/recipes/wheel/__init__.py @@ -0,0 +1,16 @@ + +from pythonforandroid.toolchain import PythonRecipe + + +class WheelRecipe(PythonRecipe): + version = '0.29.0' + url = 'https://pypi.python.org/packages/source/w/wheel/wheel-{version}.tar.gz' + + depends = [('python2', 'python3crystax')] + + call_hostpython_via_targetpython = False + install_in_targetpython = False + install_in_hostpython = True + + +recipe = WheelRecipe() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index d457be849e..5db042783a 100755 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -87,6 +87,7 @@ def wrapper_func(self, args): user_ndk_ver=self.ndk_version) dist = self._dist dist_args, args = parse_dist_args(args) + ctx.build_wheels = dist_args.build_wheels if dist.needs_build: info_notify('No dist exists that meets your requirements, ' 'so one will be built.') @@ -146,7 +147,7 @@ def build_dist_from_args(ctx, dist, args): def parse_dist_args(args_list): parser = argparse.ArgumentParser( - description='Create a newAndroid project') + description='Create a new Android project') parser.add_argument( '--bootstrap', help=('The name of the bootstrap type, \'pygame\' ' @@ -154,6 +155,12 @@ def parse_dist_args(args_list): 'bootstrap be chosen automatically from your ' 'requirements.'), default=None) + parser.add_argument( + '--build-wheels', + help='Build wheels instead of downloading', + default=False, + dest='build_wheels', + action='store_true') args, unknown = parser.parse_known_args(args_list) return args, unknown