diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..56229e6 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,114 @@ +version: 2.1 + +jobs: + build_macos_wheels: + parameters: + architecture: + type: string + default: "intel" + machine: + type: string + default: "medium" + macos: + xcode: "14.2.0" + resource_class: + <> + steps: + - checkout + - run: + name: Run Build Script + command: | + brew install python@3.8 python@3.9 python@3.10 python@3.11 + + git submodule update --init --recursive + + for version in 3.8 3.9 3.10 3.11; do + python"${version}" -m venv env"${version}" + source env"${version}"/bin/activate + + python -m pip install -U pip + python -m pip install wheel + + if [[ "${version}" == "3.8" ]]; then + arch_flags="-a <> -f" + else + arch_flags="-a <>" + fi + + scripts/create_macos_wheels.sh ${arch_flags} + + deactivate + done + + mkdir -p artifacts + cp python/dist/fixed_wheel/*.whl artifacts/ + - store_artifacts: + path: artifacts + destination: mac_os_wheels + build_linux_wheels: + resource_class: medium + docker: + - image: circleci/python:3.9 + steps: + - checkout + - run: + name: Run Build Script + command: | + git submodule update --init --recursive + chmod +x tools/build_manylinux2010.sh + tools/build_manylinux2010.sh + - run: + name: Run cibuildwheel + command: | + pip3 install --user cibuildwheel==2.12.3 + cd python + python3 -m cibuildwheel --output-dir dist + - store_artifacts: + path: python/dist/fixed_wheel + destination: linux_wheels + + + fix_linux_windows_wheels: + parameters: + eva_decord_version: + type: string + default: "0.7.0" + resource_class: medium + docker: + - image: circleci/python:3.8 + steps: + - checkout + - run: + name: Download And Fix Wheels + command: | + mkdir artifacts && cd artifacts + # Download the windows and linux wheels from existing releases + wget https://files.pythonhosted.org/packages/6c/be/e15b5b866da452e62635a7b27513f31cb581fa2ea9cc9b768b535d62a955/decord-0.6.0-py3-none-win_amd64.whl + wget https://files.pythonhosted.org/packages/11/79/936af42edf90a7bd4e41a6cac89c913d4b47fa48a26b042d5129a9242ee3/decord-0.6.0-py3-none-manylinux2010_x86_64.whl + + cd .. + python scripts/fix_wheels.py temp/ --force_version <> + - store_artifacts: + path: artifacts + destination: linux_windows_wheels + +workflows: + build_and_upload_wheels: + jobs: + - build_linux_wheels: + name: build_linux_wheels + # - build_macos_wheels: + # name: build_macos_wheels_intel + # architecture: "intel" + # machine: "medium" + # Our current pricing plan does not allow us to run on m1 machines + # - build_macos_wheels: + # name: build_macos_wheels_arm + # architecture: "arm" + # machine: "macos.m1.large.gen1" + # Old job that is not used anymore + # - fix_linux_windows_wheels: + # name: fix_linux_windows_wheels + # eva_decord_version: "0.7.0" + + diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index b92ae90..21dcd10 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -105,7 +105,7 @@ jobs: run: | cd python python setup.py bdist_wheel - find ./dist/ -type f -iname "decord*.whl" -exec sh -c 'mv $0 ${0/\10_14/10_9}' {} \; + find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'mv $0 ${0/\10_14/10_9}' {} \; - name: Fix wheel by delocate run: | FFMPEG_DIR="$HOME"/ffmpeg_build @@ -115,12 +115,12 @@ jobs: mkdir -p ./python/dist/fixed_wheel cd ./python/dist/ cp "$FFMPEG_DIR"/lib/libvpx*.dylib . - find . -type f -iname "decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \; + find . -type f -iname "eva_decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \; ls -lh ./fixed_wheel - name: Sanity Test run: | ls ./python/dist/fixed_wheel - find ./python/dist/fixed_wheel -type f -iname "decord*.whl" -exec sh -c "python -m pip install '{}'" \; + find ./python/dist/fixed_wheel -type f -iname "eva_decord*.whl" -exec sh -c "python -m pip install '{}'" \; python -m nose -v ./tests/python/unittests/test_video_reader.py - name: Store the source distribution uses: actions/upload-artifact@v2 diff --git a/.gitignore b/.gitignore index 4b1f945..0223e54 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ __pycache__ python/build/ python/*egg-info/ python/dist/ + +delocate*/ +packaging*/ +setup.cfg \ No newline at end of file diff --git a/python/decord/_ffi/libinfo.py b/python/decord/_ffi/libinfo.py index e4dd0a8..e6fcda0 100644 --- a/python/decord/_ffi/libinfo.py +++ b/python/decord/_ffi/libinfo.py @@ -90,4 +90,4 @@ def find_lib_path(name=None, search_path=None, optional=False): # We use the version of the incoming release for code # that is under development. # The following line is set by decord/python/update_version.py -__version__ = "0.6.0" +__version__ = "0.7.0" diff --git a/python/setup.py b/python/setup.py index d6e4438..771e6fb 100644 --- a/python/setup.py +++ b/python/setup.py @@ -69,7 +69,7 @@ def get_lib_path(): setup( name='eva-decord', - version="0.6.0", + version="0.7.0", description='EVA\'s Decord Video Loader', zip_safe=False, maintainer='Decord committers', diff --git a/scripts/create_linux_wheels.sh b/scripts/create_linux_wheels.sh new file mode 100644 index 0000000..26c1b80 --- /dev/null +++ b/scripts/create_linux_wheels.sh @@ -0,0 +1,7 @@ +#! /bin/bash + +python -m pip install --upgrade pip +pip install twine nose delocate + +echo "[install]" > python/setup.cfg +echo "install_lib=" >> python/setup.cfg \ No newline at end of file diff --git a/scripts/create_macos_wheels.sh b/scripts/create_macos_wheels.sh new file mode 100755 index 0000000..2f76b8f --- /dev/null +++ b/scripts/create_macos_wheels.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Set default values for options +arch="intel" +target_version="10_9" +build_ffmpeg=false + +# Parse command-line options +while getopts ":a:f" opt; do + case ${opt} in + a) # Specify architecture + arch=$OPTARG + ;; + f) # Build ffmpeg + build_ffmpeg=true + ;; + \?) # Invalid option + echo "Usage: $0 [-a arm|intel] [-f]" 1>&2 + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +# run the build script only if the build_ffmpeg flag is set +if [ "$build_ffmpeg" = true ]; then + if [ "$arch" == "arm" ]; then + target_version="11_0" + chmod +x tools/build_macos_arm.sh + tools/build_macos_arm.sh + elif [ "$arch" == "intel" ]; then + chmod +x tools/build_macos_intel.sh + tools/build_macos_intel.sh + else + echo "Invalid architecture: $arch" + exit 1 + fi +fi + +# Install Pip and Other Dependencies +python -m pip install --upgrade pip +pip install twine nose wheel delocate +source ~/.bashrc + +# Setup.py Hack +echo "[install]" >python/setup.cfg +echo "install_lib=" >>python/setup.cfg + +# Build Wheel +cd python +python setup.py bdist_wheel +find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'new_filename=$(echo "$0" | sed -E "s/_[0-9]*_[0-9]*/'"_${target_version}"'/"); mv "$0" "$new_filename"' {} \; +cd .. + +# Fix wheel by delocate +FFMPEG_DIR="$HOME"/ffmpeg_build +ls -lh ./python/dist/*.whl +find ./python/dist/ -type f -iname "eva_decord*.whl" -exec sh -c "delocate-listdeps '{}'" \; +mkdir -p ./python/dist/fixed_wheel +cd ./python/dist/ +cp "$FFMPEG_DIR"/lib/libvpx*.dylib . +find . -type f -iname "eva_decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \; +ls -lh ./fixed_wheel +cd ../../ + +# Sanity Test +ls ./python/dist/fixed_wheel +find ./python/dist/fixed_wheel -type f -iname "eva_decord*.whl" -exec sh -c "python -m pip install '{}'" \; +# python -m nose -v ./tests/python/unittests/test_video_reader.py diff --git a/scripts/fix_wheels.py b/scripts/fix_wheels.py new file mode 100644 index 0000000..dd519d9 --- /dev/null +++ b/scripts/fix_wheels.py @@ -0,0 +1,90 @@ +import os +import shutil +import zipfile +import argparse + + +METADATA_STR = '''Metadata-Version: 2.1 +Name: {} +Version: {} +Summary: EVA's Decord Video Loader +Home-page: https://github.com/georgia-tech-db/eva-decord +Maintainer: Decord committers +Maintainer-email: georgia.tech.db@gmail.com +License: APACHE +Classifier: Development Status :: 3 - Alpha +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: Apache Software License +Requires-Python: >=3.7.0 +Requires-Dist: numpy (>=1.14.0) + +''' + +WHEEL_STR = '''Wheel-Version: 1.0 +Generator: bdist_wheel (0.36.2) +Root-Is-Purelib: false +Tag: {} +''' + +def process_wheels(dir_path, force_package_name = 'eva-decord', force_version = '0.7.0'): + os.chdir(dir_path) + for filename in os.listdir('.'): + if filename.endswith('.whl'): + # Unzip the wheel + name_of_wheel = filename[:-4] # strips the .whl + print ("Unzipping: " + name_of_wheel) + package_name, package_version, python_version, abi_tag, platform_tag = name_of_wheel.split('-') + + # Even though we are building on macos13, we change to macos11 to make it more compatible + if platform_tag == 'macosx_13_0_arm64': + platform_tag = 'macosx_11_0_arm64' + + # Extract the wheel file + # Adds a folder decord and the dist-info folder + with zipfile.ZipFile(filename, 'r') as zip_ref: + zip_ref.extractall(name_of_wheel) + + dist_info_dir = f"{package_name.replace('-', '_')}-{package_version}.dist-info" + + package_version = force_version + + + # Edit the METADATA file + metadata_file = os.path.join(name_of_wheel, dist_info_dir, 'METADATA') + with open(metadata_file, 'w') as f: + f.write(METADATA_STR.format(force_package_name, force_version)) + + # Edit the WHEEL file + wheel_file = os.path.join(name_of_wheel, dist_info_dir, 'WHEEL') + with open(wheel_file, 'w') as f: + f.write(WHEEL_STR.format(f"{python_version}-{abi_tag}-{platform_tag}")) + + # Rename the dist_info_dir + new_dist_info_dir_name = f"{force_package_name.replace('-', '_')}-{package_version}.dist-info" + cur_dist_info_dir = os.path.join(name_of_wheel, dist_info_dir) + new_dist_info_dir = os.path.join(name_of_wheel, new_dist_info_dir_name) + os.rename(cur_dist_info_dir, new_dist_info_dir) + + # Zip the directory with the new name + new_name = f"{force_package_name.replace('-', '_')}-{package_version}-{python_version}-{abi_tag}-{platform_tag}" + print ("Zipping: " + new_name) + shutil.make_archive(new_name, 'zip', name_of_wheel) + + # Rename the zip file to .whl + os.rename(f"{new_name}.zip", f"{new_name}.whl") + + # Remove the unzipped directory + shutil.rmtree(name_of_wheel) + + if filename != f"{new_name}.whl": + # Remove the original wheel + os.remove(filename) + +if __name__ == "__main__": + # take directory, package name, and version as arguments + parser = argparse.ArgumentParser() + parser.add_argument("dir_path", help="Directory containing the wheels to be renamed") + parser.add_argument("--force_package_name", help="Package name to be used in the renamed wheel", default='eva-decord') + parser.add_argument("--force_version", help="Version to be used in the renamed wheel", default='0.7.0') + args = parser.parse_args() + process_wheels(args.dir_path, args.force_package_name, args.force_version) \ No newline at end of file diff --git a/src/video/ffmpeg/ffmpeg_common.h b/src/video/ffmpeg/ffmpeg_common.h index d81897b..373977c 100644 --- a/src/video/ffmpeg/ffmpeg_common.h +++ b/src/video/ffmpeg/ffmpeg_common.h @@ -21,7 +21,7 @@ extern "C" { #endif #include -#ifdef __APPLE__ +#if defined(__arm__) && defined(__APPLE__) #include #endif #include diff --git a/src/video/nvcodec/cuda_threaded_decoder.cc b/src/video/nvcodec/cuda_threaded_decoder.cc index a44c338..d158699 100644 --- a/src/video/nvcodec/cuda_threaded_decoder.cc +++ b/src/video/nvcodec/cuda_threaded_decoder.cc @@ -17,7 +17,7 @@ namespace decord { namespace cuda { using namespace runtime; -#ifdef __APPLE__ +#if defined(__arm__) && defined(__APPLE__) CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat) #else CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat) @@ -74,7 +74,7 @@ CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, } } -#ifdef __APPLE__ +#if defined(__arm__) && defined(__APPLE__) void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat) { #else void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat) { diff --git a/src/video/nvcodec/cuda_threaded_decoder.h b/src/video/nvcodec/cuda_threaded_decoder.h index e1f4127..d766392 100644 --- a/src/video/nvcodec/cuda_threaded_decoder.h +++ b/src/video/nvcodec/cuda_threaded_decoder.h @@ -46,7 +46,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface { using FrameOrderQueuePtr = std::unique_ptr; public: - #ifdef __APPLE__ + #if defined(__arm__) && defined(__APPLE__) CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat); #else CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat); @@ -74,7 +74,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface { void LaunchThreadImpl(); void RecordInternalError(std::string message); void CheckErrorStatus(); - #ifdef __APPLE__ + #if defined(__arm__) && defined(__APPLE__) void InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat); #else void InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat); diff --git a/src/video/video_reader.cc b/src/video/video_reader.cc index df5e67e..8c7189f 100644 --- a/src/video/video_reader.cc +++ b/src/video/video_reader.cc @@ -145,7 +145,7 @@ VideoReader::~VideoReader(){ void VideoReader::SetVideoStream(int stream_nb) { if (!fmt_ctx_) return; - #ifdef __APPLE__ + #if defined(__arm__) && defined(__APPLE__) const AVCodec *dec; #else AVCodec *dec; diff --git a/tools/build_macos_arm.sh b/tools/build_macos_arm.sh new file mode 100755 index 0000000..c8acada --- /dev/null +++ b/tools/build_macos_arm.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# this file is actually for building decord for macos >=10.9 on github action + +set -e + +# pwd +pwd + +# build tools +brew install cmake nasm yasm + +# cmake > 3.8 +cmake --version + +# workspace +pushd ~ +mkdir ~/ffmpeg_sources + +# libx264 +cd ~/ffmpeg_sources +git clone --depth 1 https://code.videolan.org/videolan/x264.git +cd x264 +./configure --prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" --enable-shared --extra-cflags=-mmacosx-version-min=10.9 --extra-ldflags=-mmacosx-version-min=10.9 +make +make install + +# libvpx +cd ~/ffmpeg_sources +git clone --depth 1 https://chromium.googlesource.com/webm/libvpx.git +cd libvpx +./configure --prefix="$HOME/ffmpeg_build" --disable-examples --disable-unit-tests --enable-vp9-highbitdepth --as=yasm --enable-shared --extra-cflags=-mmacosx-version-min=10.9 --extra-cxxflags=-mmacosx-version-min=10.9 +make +make install + +# ffmpeg +cd ~/ffmpeg_sources +curl -O -L https://ffmpeg.org/releases/ffmpeg-5.1.2.tar.bz2 +tar xjf ffmpeg-5.1.2.tar.bz2 +cd ffmpeg-5.1.2 +./configure \ + --prefix="$HOME/ffmpeg_build" \ + --enable-shared \ + --extra-cflags="-mmacosx-version-min=11 -I$HOME/ffmpeg_build/include" \ + --extra-cxxflags="-mmacosx-version-min=11 -I$HOME/ffmpeg_build/include" \ + --extra-ldflags="-mmacosx-version-min=11 -L$HOME/ffmpeg_build/lib" \ + --bindir="$HOME/bin" \ + --enable-gpl \ + --enable-nonfree \ + --enable-libvpx \ + --enable-libx264 \ + --disable-static +mak +make install + +# built libs +ls ~/ffmpeg_build/lib + +# decord +popd +pwd +mkdir -p build && cd build +cmake .. -DUSE_CUDA=0 -DFFMPEG_DIR="$HOME/ffmpeg_build" +make +ls -lh diff --git a/tools/build_macos_10_9.sh b/tools/build_macos_intel.sh similarity index 99% rename from tools/build_macos_10_9.sh rename to tools/build_macos_intel.sh index 59bb1dd..4952477 100644 --- a/tools/build_macos_10_9.sh +++ b/tools/build_macos_intel.sh @@ -62,4 +62,4 @@ pwd mkdir -p build && cd build cmake .. -DUSE_CUDA=0 -DFFMPEG_DIR="$HOME/ffmpeg_build" make -j$(nproc) -ls -lh +ls -lh \ No newline at end of file diff --git a/tools/build_manylinux2010.sh b/tools/build_manylinux2010.sh index ce9e86f..100ada4 100644 --- a/tools/build_manylinux2010.sh +++ b/tools/build_manylinux2010.sh @@ -8,7 +8,7 @@ set -e pwd # build tools -yum install -y autoconf automake bzip2 bzip2-devel freetype-devel gcc gcc-c++ git libtool make mercurial pkgconfig zlib-devel +# yum install -y autoconf automake bzip2 bzip2-devel freetype-devel gcc gcc-c++ git libtool make mercurial pkgconfig zlib-devel # cmake pushd ~ diff --git a/tools/update_version.py b/tools/update_version.py index 4fdc9b4..11adc3f 100644 --- a/tools/update_version.py +++ b/tools/update_version.py @@ -11,7 +11,7 @@ # current version # We use the version of the incoming release for code # that is under development -__version__ = "0.6.0" +__version__ = "0.7.0" # Implementations def update(file_name, pattern, repl):