From bf21ddbdc5fbb3b8bb7f7a97aa3110141c086bd2 Mon Sep 17 00:00:00 2001 From: Christopher Hesse Date: Sat, 25 Apr 2020 21:11:07 -0700 Subject: [PATCH] python 3.8 support (#201) --- .travis.yml | 11 ++++++----- CHANGES.md | 5 +++++ README.md | 2 +- VERSION | 2 +- docker/build_images.sh | 8 ++++++++ docker/linux/Dockerfile | 4 ++++ docker/linux/build_scripts/build.sh | 3 +-- docker/linux/build_scripts/build_env.sh | 4 ++-- docker/linux/build_scripts/build_utils.sh | 4 ++-- docker/linux/build_scripts/prefetch.sh | 2 +- .../linux/build_scripts/py36-requirements.txt | 2 +- docker/scripts/build_qt5.sh | 2 +- docker/scripts/install_python.sh | 17 ++++++++++++++++- docker/windows/Dockerfile | 13 +++++++++---- setup.py | 7 ++++++- travis.py | 9 +++++---- 16 files changed, 69 insertions(+), 26 deletions(-) create mode 100755 docker/build_images.sh diff --git a/.travis.yml b/.travis.yml index 375166bae..c888d67e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,9 @@ cache: - $HOME/roms/ language: cpp # Needed for ccache env: - - PYVER=3.5 - PYVER=3.6 - PYVER=3.7 + - PYVER=3.8 os: - linux - osx @@ -21,15 +21,15 @@ matrix: - os: linux env: - CROSS=win64 - - PYVER=3.5 + - PYVER=3.6 - os: linux env: - CROSS=win64 - - PYVER=3.6 + - PYVER=3.7 - os: linux env: - CROSS=win64 - - PYVER=3.7 + - PYVER=3.8 before_script: - mkdir -p $HOME/roms @@ -40,6 +40,7 @@ script: else TAG=$TRAVIS_OS_NAME fi + TAG=${TAG}-v2 - > if [ "$TRAVIS_OS_NAME" == "linux" ]; then docker run @@ -60,7 +61,7 @@ script: if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update > /dev/null brew install qt5 capnp lua@5.1 ccache - curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh bash miniconda.sh -b export PATH=/usr/local/opt/ccache/libexec:$HOME/miniconda3/bin:$PATH hash -r diff --git a/CHANGES.md b/CHANGES.md index aaa2aa55f..7b672595b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ # Changelog +## 0.8.0 + +* add python 3.8 support +* drop python 3.5 due to build issues on windows + ## 0.7.1 * fix discrete and multi-discrete action space filtering diff --git a/README.md b/README.md index d96ab501a..8cd747169 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Supported platforms: Supported Pythons: -- 3.5 - 3.6 - 3.7 +- 3.8 Each game integration has files listing memory locations for in-game variables, reward functions based on those variables, episode end conditions, savestates at the beginning of levels and a file containing hashes of ROMs that work with these files. diff --git a/VERSION b/VERSION index 39e898a4f..a3df0a695 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1 +0.8.0 diff --git a/docker/build_images.sh b/docker/build_images.sh new file mode 100755 index 000000000..506bfa388 --- /dev/null +++ b/docker/build_images.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +cd "$(dirname $0)/../" +./docker/linux/build_scripts/prefetch.sh OPENSSL CURL +docker build . --tag openai/retro-build:linux-v2 --file docker/linux/Dockerfile +docker push openai/retro-build:linux-v2 +docker build . --tag openai/retro-build:win64-v2 --file docker/windows/Dockerfile +docker push openai/retro-build:win64-v2 \ No newline at end of file diff --git a/docker/linux/Dockerfile b/docker/linux/Dockerfile index d5802505e..95cc2a115 100644 --- a/docker/linux/Dockerfile +++ b/docker/linux/Dockerfile @@ -26,6 +26,10 @@ RUN /opt/python/cp35-cp35m/bin/python3.5 -m venv ~/venv3.5 && \ . ~/venv3.7/bin/activate && \ pip install wheel && \ pip install google-auth google-cloud-storage pytest requests && \ + /opt/python/cp38-cp38/bin/python3.8 -m venv ~/venv3.8 && \ + . ~/venv3.8/bin/activate && \ + pip install wheel && \ + pip install google-auth google-cloud-storage pytest requests && \ rm -rf ~/.cache && \ ln -s ~/venv$PYVER ~/venv && \ echo "source /root/venv\$PYVER/bin/activate" > ~/.bash_profile diff --git a/docker/linux/build_scripts/build.sh b/docker/linux/build_scripts/build.sh index 3a9fdfb8c..a099881c3 100755 --- a/docker/linux/build_scripts/build.sh +++ b/docker/linux/build_scripts/build.sh @@ -43,8 +43,7 @@ yum -y update # EPEL support yum -y install wget -# https://dl.fedoraproject.org/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm -cp $MY_DIR/epel-release-5-4.noarch.rpm . +wget -q http://archives.fedoraproject.org/pub/archive/epel/5/x86_64/epel-release-5-4.noarch.rpm check_sha256sum epel-release-5-4.noarch.rpm $EPEL_RPM_HASH # Dev toolset (for LLVM and other projects requiring C++11 support) diff --git a/docker/linux/build_scripts/build_env.sh b/docker/linux/build_scripts/build_env.sh index 7467a9550..007cdec8a 100644 --- a/docker/linux/build_scripts/build_env.sh +++ b/docker/linux/build_scripts/build_env.sh @@ -1,13 +1,13 @@ # source me PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python -CPYTHON_VERSIONS="3.5.6 3.6.7 3.7.1" +CPYTHON_VERSIONS="3.5.6 3.6.7 3.7.1 3.8.2" # openssl version to build, with expected sha256 hash of .tar.gz # archive. OPENSSL_ROOT=openssl-1.0.2p OPENSSL_HASH=50a98e07b1a89eb8f6a99477f262df71c6fa7bef77df4dc83025a2845c827d00 -OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source +OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/old/1.0.2 # Update to slightly newer, verified Git commit: # https://github.com/NixOS/patchelf/commit/2a9cefd7d637d160d12dc7946393778fa8abbc58 diff --git a/docker/linux/build_scripts/build_utils.sh b/docker/linux/build_scripts/build_utils.sh index 47003c069..d8d0b28c9 100755 --- a/docker/linux/build_scripts/build_utils.sh +++ b/docker/linux/build_scripts/build_utils.sh @@ -92,8 +92,8 @@ function build_cpythons { check_var $GET_PIP_URL curl -fsSLO $GET_PIP_URL # Import public keys used to verify downloaded Python source tarballs. - # https://www.python.org/static/files/pubkeys.txt - gpg --import ${MY_DIR}/cpython-pubkeys.txt + curl -fsSLO https://www.python.org/static/files/pubkeys.txt + gpg --import pubkeys.txt for py_ver in $@; do build_cpython $py_ver done diff --git a/docker/linux/build_scripts/prefetch.sh b/docker/linux/build_scripts/prefetch.sh index 19105f4f4..3cecfe7dc 100755 --- a/docker/linux/build_scripts/prefetch.sh +++ b/docker/linux/build_scripts/prefetch.sh @@ -2,7 +2,7 @@ # Prefetch tarballs so they don't need to be fetched in the container (which has # very old tools). # -# usage: prefetch.sh [name ...] +# usage: prefetch.sh [name ...] set -ex MY_DIR=$(dirname "${BASH_SOURCE[0]}") diff --git a/docker/linux/build_scripts/py36-requirements.txt b/docker/linux/build_scripts/py36-requirements.txt index 8fd1a8f76..374075a7e 100644 --- a/docker/linux/build_scripts/py36-requirements.txt +++ b/docker/linux/build_scripts/py36-requirements.txt @@ -8,7 +8,7 @@ auditwheel==1.9.0 \ --hash=sha256:fe8cbf74d16e7d1a89fd96b689f8e16f196edfb66cc98eb8a345bc03e28fed63 # this package required for auditwheel pyelftools==0.25 \ - --hash=sha256:89c6da6f56280c37a5ff33468591ba9a124e17d71fe42de971818cbff46c1b24 + --hash=sha256:1ec62792b31d9d4e5399a686a515a31091be84e307c4680fd3627121500a1be7 # this package required for auditwheel typing==3.6.6 \ --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \ diff --git a/docker/scripts/build_qt5.sh b/docker/scripts/build_qt5.sh index fb2961cd7..14dfbfba0 100755 --- a/docker/scripts/build_qt5.sh +++ b/docker/scripts/build_qt5.sh @@ -88,4 +88,4 @@ fi -no-sql-odbc \ -no-harfbuzz \ -no-openssl \ - -no-dbus && make -j2 install + -no-dbus && make -j$(nproc) install diff --git a/docker/scripts/install_python.sh b/docker/scripts/install_python.sh index 9cc72aa28..30132f897 100755 --- a/docker/scripts/install_python.sh +++ b/docker/scripts/install_python.sh @@ -16,15 +16,30 @@ function install_python { curl -LO https://www.python.org/ftp/python/$MAJOR.$MINOR.$PATCH/$PLATFORM/dev.msi 7z x dev.msi - mkdir -p $ROOT/lib/python$MAJOR.$MINOR include + mkdir -p $ROOT/lib/python$MAJOR.$MINOR include include/cpython include/internal mv libs_python.lib $ROOT/lib/python$MAJOR.$MINOR/python$MAJOR$MINOR.lib mv libs_python_stable.lib $ROOT/lib/python$MAJOR.$MINOR/python$MAJOR.lib mv libs_libpython.a $ROOT/lib/python$MAJOR.$MINOR/libpython.a + find . -maxdepth 1 -name include_cpython_\*.h -print0 | xargs -0 -n1 -I% bash -c 'mv % $(echo % | sed -e "s,_,/," | sed -e "s,_,/,")' + find . -maxdepth 1 -name include_internal_\*.h -print0 | xargs -0 -n1 -I% bash -c 'mv % $(echo % | sed -e "s,_,/," | sed -e "s,_,/,")' find . -maxdepth 1 -name include_\*.h -print0 | xargs -0 -n1 -I% bash -c 'mv % $(echo % | sed -e "s,_,/,")' mv include $ROOT/include/python$MAJOR.$MINOR rm dev.msi + + # python 3.8 is missing this file, create it from the dll + # https://github.com/openai/retro/pull/201#issuecomment-619326417 + if [[ ! -f $ROOT/lib/python$MAJOR.$MINOR/libpython.a ]] + then + curl -LO https://www.python.org/ftp/python/$MAJOR.$MINOR.$PATCH/$PLATFORM/core.msi + 7z x core.msi + apt-get install --yes mingw-w64-tools + gendef python.dll + /usr/bin/x86_64-w64-mingw32-dlltool --as-flags=--64 -m i386:x86-64 -k --output-lib $ROOT/lib/python$MAJOR.$MINOR/libpython.a --input-def python.def + rm *.dll *.def core.msi + fi } install_python 3 5 4 install_python 3 6 6 install_python 3 7 0 +install_python 3 8 2 diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile index 97361a240..e7aa5ed53 100644 --- a/docker/windows/Dockerfile +++ b/docker/windows/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:xenial SHELL ["/bin/bash", "-c"] ARG ARCH=x86_64 ARG BITS=64 -ENV PYVER=3.5 +ENV PYVER=3.6 RUN apt-get update RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential ccache cmake curl g++-mingw-w64-`echo $ARCH | tr _ -` \ @@ -16,7 +16,8 @@ RUN apt-add-repository -y ppa:deadsnakes/ppa && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libpython3.5-dev python3.5-venv \ libpython3.6-dev python3.6-venv \ - libpython3.7-dev python3.7-venv && \ + libpython3.7-dev python3.7-venv \ + libpython3.8-dev python3.8-venv && \ apt-get clean RUN python3.5 -m venv ~/venv3.5 && \ @@ -26,11 +27,15 @@ RUN python3.5 -m venv ~/venv3.5 && \ python3.6 -m venv ~/venv3.6 && \ . ~/venv3.6/bin/activate && \ pip install wheel && \ - pip install google-cloud google-resumable-media pytest requests && \ + pip install google-auth google-cloud-storage pytest requests && \ python3.7 -m venv ~/venv3.7 && \ . ~/venv3.7/bin/activate && \ pip install wheel && \ - pip install google-cloud google-resumable-media pytest requests && \ + pip install google-auth google-cloud-storage pytest requests && \ + python3.8 -m venv ~/venv3.8 && \ + . ~/venv3.8/bin/activate && \ + pip install wheel && \ + pip install google-auth google-cloud-storage pytest requests && \ rm -rf ~/.cache && \ ln -s ~/venv$PYVER ~/venv && \ echo "source /root/venv\$PYVER/bin/activate" > ~/.bash_profile diff --git a/setup.py b/setup.py index 620ab1c4c..fab566cd1 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,9 @@ VERSION_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'VERSION') +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +README = open(os.path.join(SCRIPT_DIR, "README.md")).read() + if not os.path.exists(os.path.join(os.path.dirname(__file__), '.git')): use_scm_version = False shutil.copy('VERSION', 'retro/VERSION.txt') @@ -65,11 +68,13 @@ def run(self): kwargs = {} if tuple(int(v) for v in setuptools_version.split('.')[:3]) >= (24, 2, 0): - kwargs['python_requires'] = '>=3.5.0' + kwargs['python_requires'] = '>=3.6.0' setup( name='gym-retro', + long_description=README, + long_description_content_type="text/markdown", author='OpenAI', author_email='csh@openai.com', url='https://github.com/openai/retro', diff --git a/travis.py b/travis.py index 139c72944..be46b23df 100644 --- a/travis.py +++ b/travis.py @@ -6,6 +6,9 @@ import sys import re import shutil +from google.oauth2 import service_account +from google.cloud import storage + class Fold: @@ -26,9 +29,6 @@ def call(cmd): def get_bucket(): - from google.oauth2 import service_account - from google.cloud import storage - credentials_info = json.loads(os.environ['GOOGLE_APPLICATION_CREDENTIALS_DATA']) credentials = service_account.Credentials.from_service_account_info(credentials_info) client = storage.Client(credentials=credentials, project=credentials_info['project_id']) @@ -67,8 +67,9 @@ def main(): if os_name == 'osx': cmake_options = ['-DCMAKE_PREFIX_PATH=/usr/local/opt/qt', '-DBUILD_UI=ON'] elif os_name == 'linux': + include_suffix = "m" if float(os.environ['PYVER']) < 3.8 else "" cmake_options = ['-DBUILD_MANYLINUX=ON', - '-DPYTHON_INCLUDE_DIR=%s/include/python%sm' % (sys.base_prefix, os.environ['PYVER'])] + '-DPYTHON_INCLUDE_DIR=%s/include/python%s%s' % (sys.base_prefix, os.environ['PYVER'], include_suffix)] if cross in ('win32', 'win64'): cmake_options = ['-DCMAKE_TOOLCHAIN_FILE=docker/cmake/%s.cmake' % cross, '-DBUILD_UI=ON'] if cross == 'win32':