fix: eva-decord runtime failure for intel Macs

.circleci/config.yml

@@ -0,0 +1,114 @@
+version: 2.1
+
+jobs:
+  build_macos_wheels:
+    parameters:
+      architecture:
+        type: string
+        default: "intel"
+      machine:
+        type: string
+        default: "medium"
+    macos:
+      xcode: "14.2.0"
+    resource_class:
+      <<parameters.machine>>
+    steps:
+      - checkout
+      - run:
+          name: Run Build Script
+          command: |
+            brew install [email protected] [email protected] [email protected] [email protected]
+
+            git submodule update --init --recursive
+
+            for version in 3.8 3.9 3.10 3.11; do
+              python"${version}" -m venv env"${version}"
+              source env"${version}"/bin/activate
+
+              python -m pip install -U pip
+              python -m pip install wheel
+
+              if [[ "${version}" == "3.8" ]]; then
+                arch_flags="-a <<parameters.architecture>> -f"
+              else
+                arch_flags="-a <<parameters.architecture>>"
+              fi
+
+              scripts/create_macos_wheels.sh ${arch_flags}
+
+              deactivate
+            done
+
+            mkdir -p artifacts
+            cp python/dist/fixed_wheel/*.whl artifacts/
+      - store_artifacts:
+          path: artifacts
+          destination: mac_os_wheels
+  build_linux_wheels:
+    resource_class: medium
+    docker:
+      - image: circleci/python:3.9
+    steps:
+      - checkout
+      - run:
+          name: Run Build Script
+          command: |
+            git submodule update --init --recursive
+            chmod +x tools/build_manylinux2010.sh
+            tools/build_manylinux2010.sh
+      - run:
+          name: Run cibuildwheel
+          command: |
+            pip3 install --user cibuildwheel==2.12.3
+            cd python
+            python3 -m cibuildwheel --output-dir dist
+      - store_artifacts:
+          path: python/dist/fixed_wheel
+          destination: linux_wheels
+
+
+  fix_linux_windows_wheels:
+    parameters:
+      eva_decord_version:
+        type: string
+        default: "0.7.0"
+    resource_class: medium
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout
+      - run:
+          name: Download And Fix Wheels
+          command: |
+            mkdir artifacts && cd artifacts
+            # Download the windows and linux wheels from existing releases
+            wget https://files.pythonhosted.org/packages/6c/be/e15b5b866da452e62635a7b27513f31cb581fa2ea9cc9b768b535d62a955/decord-0.6.0-py3-none-win_amd64.whl
+            wget https://files.pythonhosted.org/packages/11/79/936af42edf90a7bd4e41a6cac89c913d4b47fa48a26b042d5129a9242ee3/decord-0.6.0-py3-none-manylinux2010_x86_64.whl 
+
+            cd ..
+            python scripts/fix_wheels.py temp/ --force_version <<parameters.eva_decord_version>>
+      - store_artifacts:
+          path: artifacts
+          destination: linux_windows_wheels
+
+workflows:
+  build_and_upload_wheels:
+    jobs:
+      - build_linux_wheels:
+          name: build_linux_wheels
+      # - build_macos_wheels:
+      #     name: build_macos_wheels_intel
+      #     architecture: "intel"
+      #     machine: "medium"
+      # Our current pricing plan does not allow us to run on m1 machines
+      # - build_macos_wheels:
+      #     name: build_macos_wheels_arm
+      #     architecture: "arm"
+      #     machine: "macos.m1.large.gen1"
+      # Old job that is not used anymore
+      # - fix_linux_windows_wheels:
+      #     name: fix_linux_windows_wheels
+      #     eva_decord_version: "0.7.0"
+
+

.github/workflows/pypi.yml

@@ -105,7 +105,7 @@ jobs:
       run: |
         cd python
         python setup.py bdist_wheel
-        find ./dist/ -type f -iname "decord*.whl" -exec sh -c 'mv $0 ${0/\10_14/10_9}' {} \;
+        find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'mv $0 ${0/\10_14/10_9}' {} \;
     - name: Fix wheel by delocate
       run: |
         FFMPEG_DIR="$HOME"/ffmpeg_build
@@ -115,12 +115,12 @@ jobs:
         mkdir -p ./python/dist/fixed_wheel
         cd ./python/dist/
         cp "$FFMPEG_DIR"/lib/libvpx*.dylib .
-        find . -type f -iname "decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \;
+        find . -type f -iname "eva_decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \;
         ls -lh ./fixed_wheel
     - name: Sanity Test
       run: |
         ls ./python/dist/fixed_wheel
-        find ./python/dist/fixed_wheel -type f -iname "decord*.whl" -exec sh -c "python -m pip install '{}'" \;
+        find ./python/dist/fixed_wheel -type f -iname "eva_decord*.whl" -exec sh -c "python -m pip install '{}'" \;
         python -m nose -v ./tests/python/unittests/test_video_reader.py
     - name: Store the source distribution
       uses: actions/upload-artifact@v2

.gitignore

@@ -43,3 +43,7 @@ __pycache__
 python/build/
 python/*egg-info/
 python/dist/
+
+delocate*/
+packaging*/
+setup.cfg

python/decord/_ffi/libinfo.py

@@ -90,4 +90,4 @@ def find_lib_path(name=None, search_path=None, optional=False):
 # We use the version of the incoming release for code
 # that is under development.
 # The following line is set by decord/python/update_version.py
-__version__ = "0.6.0"
+__version__ = "0.7.0"

python/setup.py

@@ -69,7 +69,7 @@ def get_lib_path():
 
 setup(
     name='eva-decord',
-    version="0.6.0",
+    version="0.7.0",
     description='EVA\'s Decord Video Loader',
     zip_safe=False,
     maintainer='Decord committers',

scripts/create_linux_wheels.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+python -m pip install --upgrade pip
+pip install twine nose delocate
+
+echo "[install]" > python/setup.cfg
+echo "install_lib=" >> python/setup.cfg

scripts/create_macos_wheels.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Set default values for options
+arch="intel"
+target_version="10_9"
+build_ffmpeg=false
+
+# Parse command-line options
+while getopts ":a:f" opt; do
+    case ${opt} in
+    a) # Specify architecture
+        arch=$OPTARG
+        ;;
+    f) # Build ffmpeg
+        build_ffmpeg=true
+        ;;
+    \?) # Invalid option
+        echo "Usage: $0 [-a arm|intel] [-f]" 1>&2
+        exit 1
+        ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+# run the build script only if the build_ffmpeg flag is set
+if [ "$build_ffmpeg" = true ]; then
+    if [ "$arch" == "arm" ]; then
+        target_version="11_0"
+        chmod +x tools/build_macos_arm.sh
+        tools/build_macos_arm.sh
+    elif [ "$arch" == "intel" ]; then
+        chmod +x tools/build_macos_intel.sh
+        tools/build_macos_intel.sh
+    else
+        echo "Invalid architecture: $arch"
+        exit 1
+    fi
+fi
+
+# Install Pip and Other Dependencies
+python -m pip install --upgrade pip
+pip install twine nose wheel delocate
+source ~/.bashrc
+
+# Setup.py Hack
+echo "[install]" >python/setup.cfg
+echo "install_lib=" >>python/setup.cfg
+
+# Build Wheel
+cd python
+python setup.py bdist_wheel
+find ./dist/ -type f -iname "eva_decord*.whl" -exec sh -c 'new_filename=$(echo "$0" | sed -E "s/_[0-9]*_[0-9]*/'"_${target_version}"'/"); mv "$0" "$new_filename"' {} \;
+cd ..
+
+# Fix wheel by delocate
+FFMPEG_DIR="$HOME"/ffmpeg_build
+ls -lh ./python/dist/*.whl
+find ./python/dist/ -type f -iname "eva_decord*.whl" -exec sh -c "delocate-listdeps '{}'" \;
+mkdir -p ./python/dist/fixed_wheel
+cd ./python/dist/
+cp "$FFMPEG_DIR"/lib/libvpx*.dylib .
+find . -type f -iname "eva_decord*.whl" -exec sh -c "delocate-wheel -w fixed_wheel -v '{}'" \;
+ls -lh ./fixed_wheel
+cd ../../
+
+# Sanity Test
+ls ./python/dist/fixed_wheel
+find ./python/dist/fixed_wheel -type f -iname "eva_decord*.whl" -exec sh -c "python -m pip install '{}'" \;
+# python -m nose -v ./tests/python/unittests/test_video_reader.py

scripts/fix_wheels.py

@@ -0,0 +1,90 @@
+import os
+import shutil
+import zipfile
+import argparse
+
+
+METADATA_STR = '''Metadata-Version: 2.1
+Name: {}
+Version: {}
+Summary: EVA's Decord Video Loader
+Home-page: https://github.com/georgia-tech-db/eva-decord
+Maintainer: Decord committers
+Maintainer-email: [email protected]
+License: APACHE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.7.0
+Requires-Dist: numpy (>=1.14.0)
+
+'''
+
+WHEEL_STR = '''Wheel-Version: 1.0
+Generator: bdist_wheel (0.36.2)
+Root-Is-Purelib: false
+Tag: {}
+'''
+
+def process_wheels(dir_path, force_package_name = 'eva-decord', force_version = '0.7.0'):
+    os.chdir(dir_path)
+    for filename in os.listdir('.'):
+        if filename.endswith('.whl'):
+            # Unzip the wheel
+            name_of_wheel = filename[:-4] # strips the .whl
+            print ("Unzipping: " + name_of_wheel)
+            package_name, package_version, python_version, abi_tag, platform_tag = name_of_wheel.split('-')
+
+            # Even though we are building on macos13, we change to macos11 to make it more compatible
+            if platform_tag == 'macosx_13_0_arm64':
+                platform_tag = 'macosx_11_0_arm64'
+
+            # Extract the wheel file
+            # Adds a folder decord and the dist-info folder
+            with zipfile.ZipFile(filename, 'r') as zip_ref: 
+                zip_ref.extractall(name_of_wheel)
+
+            dist_info_dir = f"{package_name.replace('-', '_')}-{package_version}.dist-info"
+
+            package_version = force_version
+
+
+            # Edit the METADATA file
+            metadata_file = os.path.join(name_of_wheel, dist_info_dir, 'METADATA')
+            with open(metadata_file, 'w') as f:
+                f.write(METADATA_STR.format(force_package_name, force_version))
+
+            # Edit the WHEEL file
+            wheel_file = os.path.join(name_of_wheel, dist_info_dir, 'WHEEL')
+            with open(wheel_file, 'w') as f:
+                f.write(WHEEL_STR.format(f"{python_version}-{abi_tag}-{platform_tag}"))
+
+            # Rename the dist_info_dir
+            new_dist_info_dir_name = f"{force_package_name.replace('-', '_')}-{package_version}.dist-info"
+            cur_dist_info_dir  = os.path.join(name_of_wheel, dist_info_dir)
+            new_dist_info_dir = os.path.join(name_of_wheel, new_dist_info_dir_name)
+            os.rename(cur_dist_info_dir, new_dist_info_dir)
+
+            # Zip the directory with the new name
+            new_name = f"{force_package_name.replace('-', '_')}-{package_version}-{python_version}-{abi_tag}-{platform_tag}"
+            print ("Zipping: " + new_name)
+            shutil.make_archive(new_name, 'zip', name_of_wheel)
+
+            # Rename the zip file to .whl
+            os.rename(f"{new_name}.zip", f"{new_name}.whl")
+
+            # Remove the unzipped directory
+            shutil.rmtree(name_of_wheel)
+
+            if filename != f"{new_name}.whl":
+                # Remove the original wheel
+                os.remove(filename)
+
+if __name__ == "__main__":
+    # take directory, package name, and version as arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("dir_path", help="Directory containing the wheels to be renamed")
+    parser.add_argument("--force_package_name", help="Package name to be used in the renamed wheel", default='eva-decord')
+    parser.add_argument("--force_version", help="Version to be used in the renamed wheel", default='0.7.0')
+    args = parser.parse_args()
+    process_wheels(args.dir_path, args.force_package_name, args.force_version)

src/video/ffmpeg/ffmpeg_common.h

@@ -21,7 +21,7 @@
 extern "C" {
 #endif
 #include <libavcodec/avcodec.h>
-#ifdef __APPLE__
+#if defined(__arm__) && defined(__APPLE__)
     #include <libavcodec/bsf.h>
 #endif
 #include <libavformat/avformat.h>

src/video/nvcodec/cuda_threaded_decoder.cc

@@ -17,7 +17,7 @@ namespace decord {
 namespace cuda {
 using namespace runtime;
 
-#ifdef __APPLE__
+#if defined(__arm__) && defined(__APPLE__)
 CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat)
 #else 
 CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat)
@@ -74,7 +74,7 @@ CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar,
     }
 }
 
-#ifdef __APPLE__
+#if defined(__arm__) && defined(__APPLE__)
 void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat) {
 #else 
 void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat) {

src/video/nvcodec/cuda_threaded_decoder.h

@@ -46,7 +46,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface {
     using FrameOrderQueuePtr = std::unique_ptr<FrameOrderQueue>;
 
     public:
-        #ifdef __APPLE__
+        #if defined(__arm__) && defined(__APPLE__)
             CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat);
         #else 
             CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat);
@@ -74,7 +74,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface {
         void LaunchThreadImpl();
         void RecordInternalError(std::string message);
         void CheckErrorStatus();
-        #ifdef __APPLE__
+        #if defined(__arm__) && defined(__APPLE__)
             void InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat);
         #else
             void InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat);

src/video/video_reader.cc

@@ -145,7 +145,7 @@ VideoReader::~VideoReader(){
 
 void VideoReader::SetVideoStream(int stream_nb) {
     if (!fmt_ctx_) return;
-    #ifdef __APPLE__
+    #if defined(__arm__) && defined(__APPLE__)
         const AVCodec *dec;
     #else
         AVCodec *dec;