From 99485c73b9ef61a3a3e0655a5b6000a2f9ae2126 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Tue, 18 Apr 2023 16:23:55 -0400 Subject: [PATCH 01/16] script to build on macos --- .github/workflows/pypi.yml | 6 ++-- .gitignore | 4 +++ python/decord/_ffi/libinfo.py | 2 +- python/setup.py | 2 +- scripts/create_macos_arm_wheels.sh | 34 +++++++++++++++++++ ...{build_macos_10_9.sh => build_macos_11.sh} | 12 +++---- tools/update_version.py | 2 +- 7 files changed, 50 insertions(+), 12 deletions(-) create mode 100755 scripts/create_macos_arm_wheels.sh rename tools/{build_macos_10_9.sh => build_macos_11.sh} (80%) mode change 100644 => 100755 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index b92ae90..21dcd10 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -105,7 +105,7 @@ jobs: run: | cd python python setup.py bdist_wheel - find ./dist/ -type f -iname "decord*.whl" -exec sh -c 'mv $0 ${0/\10_14/10_9}' {} \; + find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'mv $0 ${0/\10_14/10_9}' {} \; - name: Fix wheel by delocate run: | FFMPEG_DIR="$HOME"/ffmpeg_build @@ -115,12 +115,12 @@ jobs: mkdir -p ./python/dist/fixed_wheel cd ./python/dist/ cp "$FFMPEG_DIR"/lib/libvpx*.dylib . - find . -type f -iname "decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \; + find . -type f -iname "eva_decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \; ls -lh ./fixed_wheel - name: Sanity Test run: | ls ./python/dist/fixed_wheel - find ./python/dist/fixed_wheel -type f -iname "decord*.whl" -exec sh -c "python -m pip install '{}'" \; + find ./python/dist/fixed_wheel -type f -iname "eva_decord*.whl" -exec sh -c "python -m pip install '{}'" \; python -m nose -v ./tests/python/unittests/test_video_reader.py - name: Store the source distribution uses: actions/upload-artifact@v2 diff --git a/.gitignore b/.gitignore index 4b1f945..0223e54 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ __pycache__ python/build/ python/*egg-info/ python/dist/ + +delocate*/ +packaging*/ +setup.cfg \ No newline at end of file diff --git a/python/decord/_ffi/libinfo.py b/python/decord/_ffi/libinfo.py index e4dd0a8..e6fcda0 100644 --- a/python/decord/_ffi/libinfo.py +++ b/python/decord/_ffi/libinfo.py @@ -90,4 +90,4 @@ def find_lib_path(name=None, search_path=None, optional=False): # We use the version of the incoming release for code # that is under development. # The following line is set by decord/python/update_version.py -__version__ = "0.6.0" +__version__ = "0.7.0" diff --git a/python/setup.py b/python/setup.py index d6e4438..771e6fb 100644 --- a/python/setup.py +++ b/python/setup.py @@ -69,7 +69,7 @@ def get_lib_path(): setup( name='eva-decord', - version="0.6.0", + version="0.7.0", description='EVA\'s Decord Video Loader', zip_safe=False, maintainer='Decord committers', diff --git a/scripts/create_macos_arm_wheels.sh b/scripts/create_macos_arm_wheels.sh new file mode 100755 index 0000000..7941b76 --- /dev/null +++ b/scripts/create_macos_arm_wheels.sh @@ -0,0 +1,34 @@ +# Install Pip and Other Dependencies +python -m pip install --upgrade pip +pip install twine nose wheel + +# Setup.py Hack +echo "[install]" > python/setup.cfg +echo "install_lib=" >> python/setup.cfg + +# Build ffmpeg and decord +# chmod +x tools/build_macos_11.sh +# tools/build_macos_10_9.sh + +# Build Wheel +cd python +python setup.py bdist_wheel +find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'mv $0 ${0/\13/11}' {} \; +cd .. + +# Fix wheel by delocate +FFMPEG_DIR="$HOME"/ffmpeg_build +python -m pip install delocate +ls -lh ./python/dist/*.whl +find ./python/dist/ -type f -iname "decord*.whl" -exec sh -c "delocate-listdeps '{}'" \; +mkdir -p ./python/dist/fixed_wheel +cd ./python/dist/ +cp "$FFMPEG_DIR"/lib/libvpx*.dylib . +find . -type f -iname "eva_decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \; +ls -lh ./fixed_wheel +cd ../../ + +# Sanity Test +ls ./python/dist/fixed_wheel +find ./python/dist/fixed_wheel -type f -iname "eva_decord*.whl" -exec sh -c "python -m pip install '{}'" \; +# python -m nose -v ./tests/python/unittests/test_video_reader.py diff --git a/tools/build_macos_10_9.sh b/tools/build_macos_11.sh old mode 100644 new mode 100755 similarity index 80% rename from tools/build_macos_10_9.sh rename to tools/build_macos_11.sh index 59bb1dd..b8d50e8 --- a/tools/build_macos_10_9.sh +++ b/tools/build_macos_11.sh @@ -35,15 +35,15 @@ make install # ffmpeg cd ~/ffmpeg_sources -curl -O -L https://ffmpeg.org/releases/ffmpeg-4.1.6.tar.bz2 -tar xjf ffmpeg-4.1.6.tar.bz2 -cd ffmpeg-4.1.6 +curl -O -L https://ffmpeg.org/releases/ffmpeg-5.1.2.tar.bz2 +tar xjf ffmpeg-5.1.2.tar.bz2 +cd ffmpeg-5.1.2 ./configure \ --prefix="$HOME/ffmpeg_build" \ --enable-shared \ - --extra-cflags="-mmacosx-version-min=10.9 -I$HOME/ffmpeg_build/include" \ - --extra-cxxflags="-mmacosx-version-min=10.9 -I$HOME/ffmpeg_build/include" \ - --extra-ldflags="-mmacosx-version-min=10.9 -L$HOME/ffmpeg_build/lib" \ + --extra-cflags="-mmacosx-version-min=11 -I$HOME/ffmpeg_build/include" \ + --extra-cxxflags="-mmacosx-version-min=11 -I$HOME/ffmpeg_build/include" \ + --extra-ldflags="-mmacosx-version-min=11 -L$HOME/ffmpeg_build/lib" \ --bindir="$HOME/bin" \ --enable-gpl \ --enable-nonfree \ diff --git a/tools/update_version.py b/tools/update_version.py index 4fdc9b4..11adc3f 100644 --- a/tools/update_version.py +++ b/tools/update_version.py @@ -11,7 +11,7 @@ # current version # We use the version of the incoming release for code # that is under development -__version__ = "0.6.0" +__version__ = "0.7.0" # Implementations def update(file_name, pattern, repl): From 0c45e393546d666e190172d927859ce0e48b5569 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Tue, 18 Apr 2023 17:24:19 -0400 Subject: [PATCH 02/16] mac and arm specific code --- src/video/ffmpeg/ffmpeg_common.h | 2 +- src/video/nvcodec/cuda_threaded_decoder.cc | 4 ++-- src/video/nvcodec/cuda_threaded_decoder.h | 4 ++-- src/video/video_reader.cc | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/video/ffmpeg/ffmpeg_common.h b/src/video/ffmpeg/ffmpeg_common.h index d81897b..373977c 100644 --- a/src/video/ffmpeg/ffmpeg_common.h +++ b/src/video/ffmpeg/ffmpeg_common.h @@ -21,7 +21,7 @@ extern "C" { #endif #include -#ifdef __APPLE__ +#if defined(__arm__) && defined(__APPLE__) #include #endif #include diff --git a/src/video/nvcodec/cuda_threaded_decoder.cc b/src/video/nvcodec/cuda_threaded_decoder.cc index a44c338..d158699 100644 --- a/src/video/nvcodec/cuda_threaded_decoder.cc +++ b/src/video/nvcodec/cuda_threaded_decoder.cc @@ -17,7 +17,7 @@ namespace decord { namespace cuda { using namespace runtime; -#ifdef __APPLE__ +#if defined(__arm__) && defined(__APPLE__) CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat) #else CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat) @@ -74,7 +74,7 @@ CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, } } -#ifdef __APPLE__ +#if defined(__arm__) && defined(__APPLE__) void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat) { #else void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat) { diff --git a/src/video/nvcodec/cuda_threaded_decoder.h b/src/video/nvcodec/cuda_threaded_decoder.h index e1f4127..d766392 100644 --- a/src/video/nvcodec/cuda_threaded_decoder.h +++ b/src/video/nvcodec/cuda_threaded_decoder.h @@ -46,7 +46,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface { using FrameOrderQueuePtr = std::unique_ptr; public: - #ifdef __APPLE__ + #if defined(__arm__) && defined(__APPLE__) CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat); #else CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat); @@ -74,7 +74,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface { void LaunchThreadImpl(); void RecordInternalError(std::string message); void CheckErrorStatus(); - #ifdef __APPLE__ + #if defined(__arm__) && defined(__APPLE__) void InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat); #else void InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat); diff --git a/src/video/video_reader.cc b/src/video/video_reader.cc index df5e67e..8c7189f 100644 --- a/src/video/video_reader.cc +++ b/src/video/video_reader.cc @@ -145,7 +145,7 @@ VideoReader::~VideoReader(){ void VideoReader::SetVideoStream(int stream_nb) { if (!fmt_ctx_) return; - #ifdef __APPLE__ + #if defined(__arm__) && defined(__APPLE__) const AVCodec *dec; #else AVCodec *dec; From ad1ea00dc2b12eeceab66f79bafafc71c7929d14 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Tue, 18 Apr 2023 17:29:15 -0400 Subject: [PATCH 03/16] fix scripts --- scripts/create_macos_arm_wheels.sh | 4 +- tools/build_macos_10_9.sh | 65 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tools/build_macos_10_9.sh diff --git a/scripts/create_macos_arm_wheels.sh b/scripts/create_macos_arm_wheels.sh index 7941b76..1cd33f7 100755 --- a/scripts/create_macos_arm_wheels.sh +++ b/scripts/create_macos_arm_wheels.sh @@ -7,8 +7,8 @@ echo "[install]" > python/setup.cfg echo "install_lib=" >> python/setup.cfg # Build ffmpeg and decord -# chmod +x tools/build_macos_11.sh -# tools/build_macos_10_9.sh +chmod +x tools/build_macos_10_9.sh +tools/build_macos_10_9.sh # Build Wheel cd python diff --git a/tools/build_macos_10_9.sh b/tools/build_macos_10_9.sh new file mode 100644 index 0000000..4952477 --- /dev/null +++ b/tools/build_macos_10_9.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# this file is actually for building decord for macos >=10.9 on github action + +set -e + +# pwd +pwd + +# build tools +brew install cmake nasm yasm + +# cmake > 3.8 +cmake --version + +# workspace +pushd ~ +mkdir ~/ffmpeg_sources + +# libx264 +cd ~/ffmpeg_sources +git clone --depth 1 https://code.videolan.org/videolan/x264.git +cd x264 +./configure --prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" --enable-shared --extra-cflags=-mmacosx-version-min=10.9 --extra-ldflags=-mmacosx-version-min=10.9 +make -j$(nproc) +make install + +# libvpx +cd ~/ffmpeg_sources +git clone --depth 1 https://chromium.googlesource.com/webm/libvpx.git +cd libvpx +./configure --prefix="$HOME/ffmpeg_build" --disable-examples --disable-unit-tests --enable-vp9-highbitdepth --as=yasm --enable-shared --extra-cflags=-mmacosx-version-min=10.9 --extra-cxxflags=-mmacosx-version-min=10.9 +make -j$(nproc) +make install + +# ffmpeg +cd ~/ffmpeg_sources +curl -O -L https://ffmpeg.org/releases/ffmpeg-4.1.6.tar.bz2 +tar xjf ffmpeg-4.1.6.tar.bz2 +cd ffmpeg-4.1.6 +./configure \ + --prefix="$HOME/ffmpeg_build" \ + --enable-shared \ + --extra-cflags="-mmacosx-version-min=10.9 -I$HOME/ffmpeg_build/include" \ + --extra-cxxflags="-mmacosx-version-min=10.9 -I$HOME/ffmpeg_build/include" \ + --extra-ldflags="-mmacosx-version-min=10.9 -L$HOME/ffmpeg_build/lib" \ + --bindir="$HOME/bin" \ + --enable-gpl \ + --enable-nonfree \ + --enable-libvpx \ + --enable-libx264 \ + --disable-static +make +make install + +# built libs +ls ~/ffmpeg_build/lib + +# decord +popd +pwd +mkdir -p build && cd build +cmake .. -DUSE_CUDA=0 -DFFMPEG_DIR="$HOME/ffmpeg_build" +make -j$(nproc) +ls -lh \ No newline at end of file From 285d3f6c26de89b4379388221f2764968298106b Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 15:30:30 -0400 Subject: [PATCH 04/16] fix scripts --- scripts/create_macos_arm_wheels.sh | 51 ++++++++++++++++--- .../{build_macos_11.sh => build_macos_arm.sh} | 0 ...ild_macos_10_9.sh => build_macos_intel.sh} | 0 3 files changed, 43 insertions(+), 8 deletions(-) rename tools/{build_macos_11.sh => build_macos_arm.sh} (100%) rename tools/{build_macos_10_9.sh => build_macos_intel.sh} (100%) diff --git a/scripts/create_macos_arm_wheels.sh b/scripts/create_macos_arm_wheels.sh index 1cd33f7..ac3a376 100755 --- a/scripts/create_macos_arm_wheels.sh +++ b/scripts/create_macos_arm_wheels.sh @@ -1,26 +1,61 @@ +#!/bin/bash + +# Set default values for options +arch="intel" +target_version="10_9" +build_ffmpeg=false + +# Parse command-line options +while getopts ":a:f" opt; do + case ${opt} in + a) # Specify architecture + arch=$OPTARG + ;; + f) # Build ffmpeg + build_ffmpeg=true + ;; + \?) # Invalid option + echo "Usage: $0 [-a arm|intel] [-f]" 1>&2 + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +# run the build script only if the build_ffmpeg flag is set +if [ "$build_ffmpeg" = true ]; then + if [ "$arch" == "arm" ]; then + target_version="11_0" + chmod +x tools/build_macos_arm.sh + tools/build_macos_arm.sh + elif [ "$arch" == "intel" ]; then + chmod +x tools/build_macos_intel.sh + tools/build_macos_intel.sh + else + echo "Invalid architecture: $arch" + exit 1 + fi +fi + # Install Pip and Other Dependencies python -m pip install --upgrade pip pip install twine nose wheel # Setup.py Hack -echo "[install]" > python/setup.cfg -echo "install_lib=" >> python/setup.cfg - -# Build ffmpeg and decord -chmod +x tools/build_macos_10_9.sh -tools/build_macos_10_9.sh +echo "[install]" >python/setup.cfg +echo "install_lib=" >>python/setup.cfg # Build Wheel cd python python setup.py bdist_wheel -find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'mv $0 ${0/\13/11}' {} \; +find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'new_filename=$(echo "$0" | sed -E "s/^(.*-)[0-9]+_[0-9]+(.whl)$/\1'"${target_version}"'\2/"); mv "$0" "$new_filename"' {} \; cd .. # Fix wheel by delocate FFMPEG_DIR="$HOME"/ffmpeg_build python -m pip install delocate ls -lh ./python/dist/*.whl -find ./python/dist/ -type f -iname "decord*.whl" -exec sh -c "delocate-listdeps '{}'" \; +find ./python/dist/ -type f -iname "eva_decord*.whl" -exec sh -c "delocate-listdeps '{}'" \; mkdir -p ./python/dist/fixed_wheel cd ./python/dist/ cp "$FFMPEG_DIR"/lib/libvpx*.dylib . diff --git a/tools/build_macos_11.sh b/tools/build_macos_arm.sh similarity index 100% rename from tools/build_macos_11.sh rename to tools/build_macos_arm.sh diff --git a/tools/build_macos_10_9.sh b/tools/build_macos_intel.sh similarity index 100% rename from tools/build_macos_10_9.sh rename to tools/build_macos_intel.sh From 53a4bc6ce1eb7a52b267efa645b3d52a79a7babf Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 16:14:33 -0400 Subject: [PATCH 05/16] some fixes --- scripts/create_macos_arm_wheels.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/create_macos_arm_wheels.sh b/scripts/create_macos_arm_wheels.sh index ac3a376..a5b0af7 100755 --- a/scripts/create_macos_arm_wheels.sh +++ b/scripts/create_macos_arm_wheels.sh @@ -39,7 +39,8 @@ fi # Install Pip and Other Dependencies python -m pip install --upgrade pip -pip install twine nose wheel +pip install twine nose wheel delocate +source source ~/.bashrc # Setup.py Hack echo "[install]" >python/setup.cfg @@ -48,12 +49,11 @@ echo "install_lib=" >>python/setup.cfg # Build Wheel cd python python setup.py bdist_wheel -find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'new_filename=$(echo "$0" | sed -E "s/^(.*-)[0-9]+_[0-9]+(.whl)$/\1'"${target_version}"'\2/"); mv "$0" "$new_filename"' {} \; +find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'new_filename=$(echo "$0" | sed -E "s/_[0-9]*_[0-9]*/'"_${target_version}"'/"); mv "$0" "$new_filename"' {} \; cd .. # Fix wheel by delocate FFMPEG_DIR="$HOME"/ffmpeg_build -python -m pip install delocate ls -lh ./python/dist/*.whl find ./python/dist/ -type f -iname "eva_decord*.whl" -exec sh -c "delocate-listdeps '{}'" \; mkdir -p ./python/dist/fixed_wheel From 39850276f913bb7bd601fcbdde39997cc4bd7563 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 16:41:36 -0400 Subject: [PATCH 06/16] adds circleci config --- .circleci/config.yml | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..f101788 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,70 @@ +version: 2.1 + +jobs: + build_arm: + macos: + xcode: "13.2.1" + image: apple/silicon-macos-11.4 + steps: + - checkout + - run: + name: Build and upload wheels + command: | + for version in 3.8 3.9 3.10 3.11; do + if [[ "${version}" == "3.8" ]]; then + arch_flags="-a arm -f" + else + arch_flags="-a arm" + fi + + pyenv local "${version}" + python -m pip install -U pip + python -m pip install wheel + + scripts/create_macos_arm_wheels ${arch_flags} + done + + mkdir -p artifacts + cp python/dist/fixed_wheel/*.whl artifacts/ + + - store_artifacts: + path: artifacts + destination: wheels + + build_intel: + macos: + xcode: "13.2.1" + image: macos-10.15 + steps: + - checkout + - run: + name: Build and upload wheels + command: | + pyenv local 3.8 + python -m pip install -U pip + python -m pip install wheel + + arch_flags="-a intel -f" + scripts/create_macos_arm_wheels ${arch_flags} + + for version in 3.8 3.9 3.10 3.11; do + pyenv local "${version}" + python -m pip install -U pip + python -m pip install wheel + + arch_flags="-a intel" + scripts/create_macos_arm_wheels ${arch_flags} + done + + mkdir -p artifacts + cp python/dist/fixed_wheel/*.whl artifacts/ + + - store_artifacts: + path: artifacts + destination: wheels + +workflows: + build_and_upload_wheels: + jobs: + - build_arm + - build_intel From 7213e77471fa68d0f504bca166fd0d56ee84d2cf Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 16:51:15 -0400 Subject: [PATCH 07/16] fix config --- .circleci/config.yml | 34 +--------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f101788..d5b44a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,40 +1,9 @@ version: 2.1 jobs: - build_arm: - macos: - xcode: "13.2.1" - image: apple/silicon-macos-11.4 - steps: - - checkout - - run: - name: Build and upload wheels - command: | - for version in 3.8 3.9 3.10 3.11; do - if [[ "${version}" == "3.8" ]]; then - arch_flags="-a arm -f" - else - arch_flags="-a arm" - fi - - pyenv local "${version}" - python -m pip install -U pip - python -m pip install wheel - - scripts/create_macos_arm_wheels ${arch_flags} - done - - mkdir -p artifacts - cp python/dist/fixed_wheel/*.whl artifacts/ - - - store_artifacts: - path: artifacts - destination: wheels - build_intel: macos: - xcode: "13.2.1" - image: macos-10.15 + xcode: "14.2.0" steps: - checkout - run: @@ -66,5 +35,4 @@ jobs: workflows: build_and_upload_wheels: jobs: - - build_arm - build_intel From d9c2a13b44af7288a6f426aa2b06789b36a9f8bc Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 16:52:51 -0400 Subject: [PATCH 08/16] add brew install --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5b44a3..4da690d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,7 @@ jobs: - run: name: Build and upload wheels command: | + brew install python3.8 python3.9 python3.10 python3.11 pyenv local 3.8 python -m pip install -U pip python -m pip install wheel From 088565073a5f79c91d00e31ee48ddf8c5124f216 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 17:30:01 -0400 Subject: [PATCH 09/16] fix circleci --- .circleci/config.yml | 25 +++++++++++++++---------- tools/build_macos_arm.sh | 8 ++++---- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4da690d..429e53d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,21 +9,26 @@ jobs: - run: name: Build and upload wheels command: | - brew install python3.8 python3.9 python3.10 python3.11 - pyenv local 3.8 - python -m pip install -U pip - python -m pip install wheel - - arch_flags="-a intel -f" - scripts/create_macos_arm_wheels ${arch_flags} + brew install python@3.8 python@3.9 python@3.10 python@3.11 + + git submodule update --init --recursive for version in 3.8 3.9 3.10 3.11; do - pyenv local "${version}" + python"${version}" -m venv env"${version}" + source env"${version}"/bin/activate + python -m pip install -U pip python -m pip install wheel - arch_flags="-a intel" - scripts/create_macos_arm_wheels ${arch_flags} + if [[ "${version}" == "3.8" ]]; then + arch_flags="-a intel -f" + else + arch_flags="-a intel" + fi + + scripts/create_macos_arm_wheels.sh ${arch_flags} + + deactivate done mkdir -p artifacts diff --git a/tools/build_macos_arm.sh b/tools/build_macos_arm.sh index b8d50e8..c8acada 100755 --- a/tools/build_macos_arm.sh +++ b/tools/build_macos_arm.sh @@ -22,7 +22,7 @@ cd ~/ffmpeg_sources git clone --depth 1 https://code.videolan.org/videolan/x264.git cd x264 ./configure --prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" --enable-shared --extra-cflags=-mmacosx-version-min=10.9 --extra-ldflags=-mmacosx-version-min=10.9 -make -j$(nproc) +make make install # libvpx @@ -30,7 +30,7 @@ cd ~/ffmpeg_sources git clone --depth 1 https://chromium.googlesource.com/webm/libvpx.git cd libvpx ./configure --prefix="$HOME/ffmpeg_build" --disable-examples --disable-unit-tests --enable-vp9-highbitdepth --as=yasm --enable-shared --extra-cflags=-mmacosx-version-min=10.9 --extra-cxxflags=-mmacosx-version-min=10.9 -make -j$(nproc) +make make install # ffmpeg @@ -50,7 +50,7 @@ cd ffmpeg-5.1.2 --enable-libvpx \ --enable-libx264 \ --disable-static -make +mak make install # built libs @@ -61,5 +61,5 @@ popd pwd mkdir -p build && cd build cmake .. -DUSE_CUDA=0 -DFFMPEG_DIR="$HOME/ffmpeg_build" -make -j$(nproc) +make ls -lh From 2715fb0605f18994c363ecde4c3b152c09454c71 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 18:35:10 -0400 Subject: [PATCH 10/16] trying to generate all wheels --- .circleci/config.yml | 31 ++++++++++++++----- ...s_arm_wheels.sh => create_macos_wheels.sh} | 0 2 files changed, 24 insertions(+), 7 deletions(-) rename scripts/{create_macos_arm_wheels.sh => create_macos_wheels.sh} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 429e53d..13ec17c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,22 @@ version: 2.1 jobs: - build_intel: + build_macos_wheels: + parameters: + architecture: + type: string + default: "intel" + machine: + type: string + default: "medium" macos: xcode: "14.2.0" + resource_class: + <> steps: - checkout - run: - name: Build and upload wheels + name: Run Build Script command: | brew install python@3.8 python@3.9 python@3.10 python@3.11 @@ -21,24 +30,32 @@ jobs: python -m pip install wheel if [[ "${version}" == "3.8" ]]; then - arch_flags="-a intel -f" + arch_flags="-a ${ARCHITECTURE} -f" else - arch_flags="-a intel" + arch_flags="-a ${ARCHITECTURE}" fi - scripts/create_macos_arm_wheels.sh ${arch_flags} + scripts/create_macos_wheels.sh ${arch_flags} deactivate done mkdir -p artifacts cp python/dist/fixed_wheel/*.whl artifacts/ - - store_artifacts: path: artifacts destination: wheels + workflows: build_and_upload_wheels: jobs: - - build_intel + - build_macos_wheels: + name: build_macos_wheels_intel + architecture: "intel" + machine: "medium" + - build_macos_wheels: + name: build_macos_wheels_arm + architecture: "arm" + machine: "macos.m1.large.gen1" + diff --git a/scripts/create_macos_arm_wheels.sh b/scripts/create_macos_wheels.sh similarity index 100% rename from scripts/create_macos_arm_wheels.sh rename to scripts/create_macos_wheels.sh From d5bfe14d0d126abeb53fe9cee8206b6f0dcc2668 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 18:36:15 -0400 Subject: [PATCH 11/16] bug fix --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13ec17c..9cb9846 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,9 +30,9 @@ jobs: python -m pip install wheel if [[ "${version}" == "3.8" ]]; then - arch_flags="-a ${ARCHITECTURE} -f" + arch_flags="-a < -f" else - arch_flags="-a ${ARCHITECTURE}" + arch_flags="-a <" fi scripts/create_macos_wheels.sh ${arch_flags} From e3dcb1cc186433e3e70a0cca30b2be685cafe855 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Wed, 19 Apr 2023 18:36:57 -0400 Subject: [PATCH 12/16] config update --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9cb9846..9db4903 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,9 +30,9 @@ jobs: python -m pip install wheel if [[ "${version}" == "3.8" ]]; then - arch_flags="-a < -f" + arch_flags="-a <> -f" else - arch_flags="-a <" + arch_flags="-a <>" fi scripts/create_macos_wheels.sh ${arch_flags} From 14183da76f8d1562db4d45d84efe5b109db8e119 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Thu, 20 Apr 2023 00:43:44 -0400 Subject: [PATCH 13/16] testing linux build --- .circleci/config.yml | 70 ++++++++++++++++++++++---- scripts/create_linux_wheels.sh | 7 +++ scripts/create_macos_wheels.sh | 2 +- scripts/fix_wheels.py | 90 ++++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 scripts/create_linux_wheels.sh create mode 100644 scripts/fix_wheels.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 9db4903..770dda8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,18 +44,70 @@ jobs: cp python/dist/fixed_wheel/*.whl artifacts/ - store_artifacts: path: artifacts - destination: wheels + destination: mac_os_wheels + build_linux_wheels: + resource_class: medium + docker: + - image: circleci/python:3.9 + steps: + - checkout + - run: + name: Run Build Script + command: | + git submodule update --init --recursive + scripts/build_manylinux2010.sh + - run: + name: Run cibuildwheel + command: | + pip3 install --user cibuildwheel==2.12.3 + cd python + python3 -m cibuildwheel --output-dir dist + - store_artifacts: + path: python/dist/fixed_wheel + destination: linux_wheels + fix_linux_windows_wheels: + parameters: + eva_decord_version: + type: string + default: "0.7.0" + resource_class: medium + docker: + - image: circleci/python:3.8 + steps: + - checkout + - run: + name: Download And Fix Wheels + command: | + mkdir artifacts && cd artifacts + # Download the windows and linux wheels from existing releases + wget https://files.pythonhosted.org/packages/6c/be/e15b5b866da452e62635a7b27513f31cb581fa2ea9cc9b768b535d62a955/decord-0.6.0-py3-none-win_amd64.whl + wget https://files.pythonhosted.org/packages/11/79/936af42edf90a7bd4e41a6cac89c913d4b47fa48a26b042d5129a9242ee3/decord-0.6.0-py3-none-manylinux2010_x86_64.whl + + cd .. + python scripts/fix_wheels.py temp/ --force_version <> + - store_artifacts: + path: artifacts + destination: linux_windows_wheels + workflows: build_and_upload_wheels: jobs: - - build_macos_wheels: - name: build_macos_wheels_intel - architecture: "intel" - machine: "medium" - - build_macos_wheels: - name: build_macos_wheels_arm - architecture: "arm" - machine: "macos.m1.large.gen1" + - build_linux_wheels: + name: build_linux_wheels + # - build_macos_wheels: + # name: build_macos_wheels_intel + # architecture: "intel" + # machine: "medium" + # Our current pricing plan does not allow us to run on m1 machines + # - build_macos_wheels: + # name: build_macos_wheels_arm + # architecture: "arm" + # machine: "macos.m1.large.gen1" + # Old job that is not used anymore + # - fix_linux_windows_wheels: + # name: fix_linux_windows_wheels + # eva_decord_version: "0.7.0" + diff --git a/scripts/create_linux_wheels.sh b/scripts/create_linux_wheels.sh new file mode 100644 index 0000000..26c1b80 --- /dev/null +++ b/scripts/create_linux_wheels.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +python -m pip install --upgrade pip +pip install twine nose delocate + +echo "[install]" > python/setup.cfg +echo "install_lib=" >> python/setup.cfg \ No newline at end of file diff --git a/scripts/create_macos_wheels.sh b/scripts/create_macos_wheels.sh index a5b0af7..2f76b8f 100755 --- a/scripts/create_macos_wheels.sh +++ b/scripts/create_macos_wheels.sh @@ -40,7 +40,7 @@ fi # Install Pip and Other Dependencies python -m pip install --upgrade pip pip install twine nose wheel delocate -source source ~/.bashrc +source ~/.bashrc # Setup.py Hack echo "[install]" >python/setup.cfg diff --git a/scripts/fix_wheels.py b/scripts/fix_wheels.py new file mode 100644 index 0000000..dd519d9 --- /dev/null +++ b/scripts/fix_wheels.py @@ -0,0 +1,90 @@ +import os +import shutil +import zipfile +import argparse + + +METADATA_STR = '''Metadata-Version: 2.1 +Name: {} +Version: {} +Summary: EVA's Decord Video Loader +Home-page: https://github.com/georgia-tech-db/eva-decord +Maintainer: Decord committers +Maintainer-email: georgia.tech.db@gmail.com +License: APACHE +Classifier: Development Status :: 3 - Alpha +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: Apache Software License +Requires-Python: >=3.7.0 +Requires-Dist: numpy (>=1.14.0) + +''' + +WHEEL_STR = '''Wheel-Version: 1.0 +Generator: bdist_wheel (0.36.2) +Root-Is-Purelib: false +Tag: {} +''' + +def process_wheels(dir_path, force_package_name = 'eva-decord', force_version = '0.7.0'): + os.chdir(dir_path) + for filename in os.listdir('.'): + if filename.endswith('.whl'): + # Unzip the wheel + name_of_wheel = filename[:-4] # strips the .whl + print ("Unzipping: " + name_of_wheel) + package_name, package_version, python_version, abi_tag, platform_tag = name_of_wheel.split('-') + + # Even though we are building on macos13, we change to macos11 to make it more compatible + if platform_tag == 'macosx_13_0_arm64': + platform_tag = 'macosx_11_0_arm64' + + # Extract the wheel file + # Adds a folder decord and the dist-info folder + with zipfile.ZipFile(filename, 'r') as zip_ref: + zip_ref.extractall(name_of_wheel) + + dist_info_dir = f"{package_name.replace('-', '_')}-{package_version}.dist-info" + + package_version = force_version + + + # Edit the METADATA file + metadata_file = os.path.join(name_of_wheel, dist_info_dir, 'METADATA') + with open(metadata_file, 'w') as f: + f.write(METADATA_STR.format(force_package_name, force_version)) + + # Edit the WHEEL file + wheel_file = os.path.join(name_of_wheel, dist_info_dir, 'WHEEL') + with open(wheel_file, 'w') as f: + f.write(WHEEL_STR.format(f"{python_version}-{abi_tag}-{platform_tag}")) + + # Rename the dist_info_dir + new_dist_info_dir_name = f"{force_package_name.replace('-', '_')}-{package_version}.dist-info" + cur_dist_info_dir = os.path.join(name_of_wheel, dist_info_dir) + new_dist_info_dir = os.path.join(name_of_wheel, new_dist_info_dir_name) + os.rename(cur_dist_info_dir, new_dist_info_dir) + + # Zip the directory with the new name + new_name = f"{force_package_name.replace('-', '_')}-{package_version}-{python_version}-{abi_tag}-{platform_tag}" + print ("Zipping: " + new_name) + shutil.make_archive(new_name, 'zip', name_of_wheel) + + # Rename the zip file to .whl + os.rename(f"{new_name}.zip", f"{new_name}.whl") + + # Remove the unzipped directory + shutil.rmtree(name_of_wheel) + + if filename != f"{new_name}.whl": + # Remove the original wheel + os.remove(filename) + +if __name__ == "__main__": + # take directory, package name, and version as arguments + parser = argparse.ArgumentParser() + parser.add_argument("dir_path", help="Directory containing the wheels to be renamed") + parser.add_argument("--force_package_name", help="Package name to be used in the renamed wheel", default='eva-decord') + parser.add_argument("--force_version", help="Version to be used in the renamed wheel", default='0.7.0') + args = parser.parse_args() + process_wheels(args.dir_path, args.force_package_name, args.force_version) \ No newline at end of file From 397ef7be8f7b3ac44d126331a94765f47585ca3e Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Thu, 20 Apr 2023 00:44:40 -0400 Subject: [PATCH 14/16] bug fix --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 770dda8..6676dae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,7 +55,7 @@ jobs: name: Run Build Script command: | git submodule update --init --recursive - scripts/build_manylinux2010.sh + tools/build_manylinux2010.sh - run: name: Run cibuildwheel command: | From 81b8c7c835ef7306f101b2f4d0a8cf159acf6423 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Thu, 20 Apr 2023 00:45:26 -0400 Subject: [PATCH 15/16] more bug fix --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6676dae..56229e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,6 +55,7 @@ jobs: name: Run Build Script command: | git submodule update --init --recursive + chmod +x tools/build_manylinux2010.sh tools/build_manylinux2010.sh - run: name: Run cibuildwheel From 5a9dd40a9e03cf5d0cf0ca5fc7025c4b59c0cd71 Mon Sep 17 00:00:00 2001 From: Suryatej Reddy Vyalla Date: Thu, 20 Apr 2023 00:46:28 -0400 Subject: [PATCH 16/16] commenting line --- tools/build_manylinux2010.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_manylinux2010.sh b/tools/build_manylinux2010.sh index ce9e86f..100ada4 100644 --- a/tools/build_manylinux2010.sh +++ b/tools/build_manylinux2010.sh @@ -8,7 +8,7 @@ set -e pwd # build tools -yum install -y autoconf automake bzip2 bzip2-devel freetype-devel gcc gcc-c++ git libtool make mercurial pkgconfig zlib-devel +# yum install -y autoconf automake bzip2 bzip2-devel freetype-devel gcc gcc-c++ git libtool make mercurial pkgconfig zlib-devel # cmake pushd ~