Basis plugins: support basis_universal 1.50

doc/building-plugins.dox

@@ -226,10 +226,11 @@ thus the Magnum Plugins package doesn't have it enabled by default. In order to
 enable them, do the following:
 
 -   For @ref Trade::BasisImporter "BasisImporter" or
-    @ref Trade::BasisImageConverter "BasisImageConverter", [download the
-    `v1_15_update2` tag of the Basis Universal repo](https://github.com/BinomialLLC/basis_universal/archive/v1_15_update2.tar.gz),
-    extract it into `src/external/basis-universal` (note the dash instead of an
-    underscore) and set `MAGNUM_WITH_BASISIMPORTER` /
+    @ref Trade::BasisImageConverter "BasisImageConverter", download the
+    [`v1_50_0_2`](https://github.com/BinomialLLC/basis_universal/archive/v1_50_0_2.tar.gz)
+    `1.16.4` or `v1_15_update2` tag of the Basis Universal repo, extract it
+    into `src/external/basis-universal` (note the dash instead of an
+    underscore) set `MAGNUM_WITH_BASISIMPORTER` /
     `MAGNUM_WITH_BASISIMAGECONVERTER` to `ON` in `package/debian/rules`
 -   For @ref Trade::MeshOptimizerSceneConverter "MeshOptimizerSceneConverter",
     [clone the MeshOptimizer repo](https://github.com/zeux/meshoptimizer) to

modules/FindBasisUniversal.cmake

@@ -87,6 +87,8 @@ macro(_basis_setup_source_file source)
 
     # Basis shouldn't override the MSVC iterator debug level as it would make
     # it inconsistent with the rest of the code
+    # Only needed until 1.15, in later versions this doesn't do
+    # anything anymore.
     if(CORRADE_TARGET_WINDOWS)
         set_property(SOURCE ${source} APPEND PROPERTY COMPILE_DEFINITIONS
             BASISU_NO_ITERATOR_DEBUG_LEVEL)
@@ -161,30 +163,81 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
                 # Alternatively, look into creating stubs for the library
                 # functions used by basis_universal.
                 set(BasisUniversalEncoder_SOURCES
-                    ${BasisUniversalEncoder_DIR}/apg_bmp.c
-                    ${BasisUniversalEncoder_DIR}/basisu_astc_decomp.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_backend.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_basis_file.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_bc7enc.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_comp.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_enc.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_etc.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_frontend.cpp
-                    ${BasisUniversalEncoder_DIR}/basisu_global_selector_palette_helpers.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_gpu_texture.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_kernels_sse.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_pvrtc1_4.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_resampler.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_resample_filters.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_ssim.cpp
-                    ${BasisUniversalEncoder_DIR}/basisu_uastc_enc.cpp
-                    ${BasisUniversalEncoder_DIR}/jpgd.cpp
-                    ${BasisUniversalEncoder_DIR}/lodepng.cpp)
+                    ${BasisUniversalEncoder_DIR}/basisu_uastc_enc.cpp)
+
+                # Files not present in all supported basis versions, treat them
+                # as optional and do nothing if not found.
+                foreach(_file
+                    # Removed in 1.16
+                    basisu_astc_decomp.cpp
+                    basisu_global_selector_palette_helpers.cpp
+                    # Added in 1.16
+                    basisu_opencl.cpp
+                    # Added in 1.50
+                    basisu_astc_hdr_enc.cpp
+                    3rdparty/android_astc_decomp.cpp)
+                    # Disable the find root path here, it overrides the
+                    # CMAKE_FIND_ROOT_PATH_MODE_INCLUDE setting potentially set in
+                    # toolchains.
+                    find_file(BasisUniversalEncoder_${_file}_SOURCE NAMES ${_file}
+                        HINTS ${BasisUniversalEncoder_DIR} NO_CMAKE_FIND_ROOT_PATH)
+
+                    if(BasisUniversalEncoder_${_file}_SOURCE)
+                        list(APPEND BasisUniversalEncoder_SOURCES
+                            ${BasisUniversalEncoder_${_file}_SOURCE})
+                    endif()
+                endforeach()
+
+                # The encoder unconditionally calls assert() for all OpenCL
+                # errors, including a non-fatal error when falling back to a
+                # CPU device: https://github.com/BinomialLLC/basis_universal/issues/378
+                # To allow testing on CPU OpenCL implementations, disable
+                # asserts in this specific file.
+                if(BasisUniversalEncoder_basisu_opencl.cpp_SOURCE)
+                    set_property(SOURCE ${BasisUniversalEncoder_basisu_opencl.cpp_SOURCE}
+                        APPEND PROPERTY COMPILE_DEFINITIONS NDEBUG)
+                endif()
+
+                # Source files for image file loading are not added here.
+                # BasisImageConverter provides stubs for all used functions.
+                # See src/external/basis-uncrapifier for more info.
 
                 foreach(_file ${BasisUniversalEncoder_SOURCES})
                     _basis_setup_source_file(${_file})
                 endforeach()
 
+                set(BasisUniversalEncoder_DEFINITIONS "BASISU_NO_ITERATOR_DEBUG_LEVEL")
+
+                # Basis already checks for GCC < 5 to work around missing
+                # std::is_trivially_copyable but then uses the wrong built-in.
+                # So close, yet so far.
+                # Needs to be transitive because it's used in a header file.
+                if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0")
+                    list(APPEND BasisUniversalEncoder_DEFINITIONS
+                        "__is_trivially_copyable=__has_trivial_copy")
+                endif()
+
+                # Try to find an external OpenCL library and enable support for
+                # it in basis if found.
+                find_package(OpenCL)
+                if(OpenCL_FOUND)
+                    list(APPEND BasisUniversalEncoder_DEFINITIONS
+                        "BASISU_SUPPORT_OPENCL=1")
+                endif()
+
                 # Disable the find root path here, it overrides the
                 # CMAKE_FIND_ROOT_PATH_MODE_INCLUDE setting potentially set in
                 # toolchains.
@@ -196,8 +249,16 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
                 add_library(BasisUniversal::Encoder INTERFACE IMPORTED)
                 set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
                     INTERFACE_INCLUDE_DIRECTORIES ${BasisUniversalEncoder_INCLUDE_DIR})
+                set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
+                    INTERFACE_COMPILE_DEFINITIONS ${BasisUniversalEncoder_DEFINITIONS})
                 set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
                     INTERFACE_SOURCES "${BasisUniversalEncoder_SOURCES}")
+                if(OpenCL_FOUND)
+                    set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
+                        INTERFACE_INCLUDE_DIRECTORIES ${OpenCL_INCLUDE_DIRS})
+                    set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
+                        INTERFACE_LINK_LIBRARIES ${OpenCL_LIBRARIES})
+                endif()
                 # Explicitly *not* linking this to Threads::Threads because
                 # when done like that, std::thread creation will die on a null
                 # function pointer call (inside __gthread_create, which weakly
@@ -214,8 +275,6 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
                 # itself.
                 set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
                     INTERFACE_LINK_LIBRARIES BasisUniversal::Transcoder)
-                set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
-                    INTERFACE_COMPILE_DEFINITIONS "BASISU_NO_ITERATOR_DEBUG_LEVEL")
             endif()
         else()
             set(BasisUniversal_Encoder_FOUND TRUE)
@@ -273,6 +332,11 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
 
                 set(BasisUniversalTranscoder_DEFINITIONS "BASISU_NO_ITERATOR_DEBUG_LEVEL")
 
+                if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0")
+                    list(APPEND BasisUniversalTranscoder_DEFINITIONS
+                        "__is_trivially_copyable=__has_trivial_copy")
+                endif()
+
                 # Try to find an external Zstandard library because that's the
                 # sanest and most flexible option.
                 #

package/archlinux/magnum-plugins-git/PKGBUILD

@@ -1,7 +1,7 @@
 # Author: mosra <[email protected]>
 pkgname=magnum-plugins-git
 pkgver=2020.06.r1061.g24295477
-_basis_pkgver=1_15_update2
+_basis_pkgver=1_50_0_2
 pkgrel=1
 pkgdesc="Plugins for the Magnum C++11/C++14 graphics engine (Git version)"
 arch=('i686' 'x86_64')
@@ -14,7 +14,7 @@ conflicts=('magnum-plugins')
 source=("git+https://github.com/mosra/magnum-plugins.git"
         "https://github.com/BinomialLLC/basis_universal/archive/v${_basis_pkgver}.tar.gz")
 sha1sums=('SKIP'
-          'b9615d48ebfc62a53f333ebf8a582558a058b0e9')
+          '4946d685424556efb07376b97e95a4cbb3245e40')
 
 pkgver() {
     cd "$srcdir/${pkgname%-git}"

package/ci/appveyor-desktop.bat

@@ -20,6 +20,9 @@ IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" set EXCEPT_MSVC2015=O
 IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" set EXCEPT_MSVC2017=OFF
 IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" set EXCEPT_MSVC2017=OFF
 
+set ONLY_MSVC2019=OFF
+IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2019" set ONLY_MSVC2019=ON
+
 rem Build Corrade
 git clone --depth 1 https://github.com/mosra/corrade.git || exit /b
 cd corrade || exit /b
@@ -65,13 +68,16 @@ cd .. && cd ..
 
 rem Build. MAGNUM_BUILD_GL_TESTS is enabled just to be sure, it should not be
 rem needed by any plugin.
+rem OpenCL is disabled on MSVC 2019 to get some test coverage for
+rem BasisImageConverter without OpenCL support compiled in.
 mkdir build && cd build || exit /b
 cmake .. ^
     -DCMAKE_BUILD_TYPE=Debug ^
     -DCMAKE_INSTALL_PREFIX=%APPVEYOR_BUILD_FOLDER%/deps ^
     -DCMAKE_PREFIX_PATH=%APPVEYOR_BUILD_FOLDER%/openal;%APPVEYOR_BUILD_FOLDER%/libwebp;%APPVEYOR_BUILD_FOLDER%/devil;C:/Tools/vcpkg/installed/x64-windows ^
     -DMAGNUM_WITH_ASSIMPIMPORTER=%EXCEPT_MSVC2015% ^
     -DMAGNUM_WITH_ASTCIMPORTER=ON ^
+    -DCMAKE_DISABLE_FIND_PACKAGE_OpenCL=%ONLY_MSVC2019% ^
     -DMAGNUM_WITH_BASISIMAGECONVERTER=%EXCEPT_MSVC2015% ^
     -DMAGNUM_WITH_BASISIMPORTER=%EXCEPT_MSVC2015% -DBASIS_UNIVERSAL_DIR=%APPVEYOR_BUILD_FOLDER%/basis_universal ^
     -DMAGNUM_WITH_BCDECIMAGECONVERTER=ON ^

package/ci/appveyor.yml

@@ -121,10 +121,20 @@ install:
 - IF "%TARGET%" == "desktop" IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2019" IF "%COMPILER:~0,4%" == "msvc" appveyor DownloadFile https://ci.magnum.graphics/freetype-2.10.4-windows-2019.zip && 7z x freetype-2.10.4-windows-2019.zip -o%APPVEYOR_BUILD_FOLDER%\deps
 - IF "%TARGET%" == "desktop" IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" IF "%COMPILER:~0,4%" == "msvc" appveyor DownloadFile https://ci.magnum.graphics/freetype-2.10.4-windows-2016.zip && 7z x freetype-2.10.4-windows-2016.zip -o%APPVEYOR_BUILD_FOLDER%\deps
 
-# Basis Universal
-- set BASIS_VERSION=1_15_update2
-- IF NOT EXIST %APPVEYOR_BUILD_FOLDER%\v%BASIS_VERSION%.zip appveyor DownloadFile https://github.com/BinomialLLC/basis_universal/archive/v%BASIS_VERSION%.zip
-- 7z x v%BASIS_VERSION%.zip && ren basis_universal-%BASIS_VERSION% basis_universal
+# Basis Universal, all the recent supported versions. Default to latest and
+# use a few older versions on 2017 and 2019.
+- set BASIS_VERSION=v1_50_0_2
+- IF "%TARGET%" == "desktop" IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2019" IF "%COMPILER%" == "msvc" set BASIS_VERSION=1.16.4
+- IF "%TARGET%" == "desktop" IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" IF "%COMPILER%" == "msvc" set BASIS_VERSION=v1_15_update2
+# Basis plugins aren't built on MSVC 2015
+- IF "%TARGET%" == "rt" set BASIS_VERSION=1.16.4
+- IF NOT EXIST %APPVEYOR_BUILD_FOLDER%\%BASIS_VERSION%.zip appveyor DownloadFile https://github.com/BinomialLLC/basis_universal/archive/%BASIS_VERSION%.zip
+# Github removes the v prefix in the folder name. 7zip has no way to skip the
+# topmost folder so strip it manually.
+- set BASIS_FOLDER=%BASIS_VERSION%
+- IF "%BASIS_FOLDER:~0,1%" == "v" set BASIS_FOLDER=%BASIS_FOLDER:~1%
+- set BASIS_FOLDER=basis_universal-%BASIS_FOLDER%
+- 7z x %BASIS_VERSION%.zip && ren %BASIS_FOLDER% basis_universal
 # We want to use the external Zstd instead of this
 - rmdir /s /q basis_universal\zstd
 

package/ci/circleci.yml

@@ -224,14 +224,17 @@ commands:
           ninja install
 
   install-basis:
+    parameters:
+      tag:
+        type: string
+        default: "v1_50_0_2"
     steps:
     - run:
-        name: Install Basis Universal
+        name: Install Basis Universal << parameters.tag >>
         command: |
-          export BASIS_VERSION=v1_15_update2
           mkdir -p $HOME/basis_universal && cd $HOME/basis_universal
-          wget -nc https://github.com/BinomialLLC/basis_universal/archive/$BASIS_VERSION.tar.gz
-          tar --strip-components 1 -xzf $BASIS_VERSION.tar.gz
+          wget -nc https://github.com/BinomialLLC/basis_universal/archive/<< parameters.tag >>.tar.gz
+          tar --strip-components 1 -xzf << parameters.tag >>.tar.gz
           # We want to use the external Zstd instead of this
           rm -r zstd
 
@@ -338,17 +341,23 @@ jobs:
       CMAKE_CXX_FLAGS: --coverage
       LCOV_EXTRA_OPTS: --gcov-tool /usr/bin/gcov-4.8
       CONFIGURATION: Debug
+      # Testing building BasisImageConverter with OpenCL present on the system,
+      # but explicitly turned off via CMAKE_DISABLE_FIND_PACKAGE_OpenCL
+      DISABLE_OPENCL: "ON"
     steps:
     - install-base-linux:
-        extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libwebp-dev libzstd-dev wget
+        # We want the OpenCL headers and ICD loader libraries to test disabling
+        # find_package(), but don't need the actual ICD
+        extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libwebp-dev libzstd-dev ocl-icd-opencl-dev wget
     # Crashes or gets stuck with the default setting, only on GCC. 24 is not
     # enough, probably due to compatibility includes.
     - cap-ninja-jobs:
         count: 20
     - install-gcc-4_8
     - install-cmake:
         version: "3.5.2"
-    - install-basis
+    - install-basis:
+        tag: "1.16.4"
     - install-meshoptimizer
     - install-libspng
     # In repos only since 20.04, but even then would probably hit the GCC 4.8 /
@@ -374,8 +383,9 @@ jobs:
     # speed up. These are tested more than enough on other jobs, plus here we
     # have a fresher Ubuntu so we can take them directly from the repos.
     - install-base-linux-arm64:
-        extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libzstd-dev glslang-dev libopenexr-dev libwebp-dev spirv-tools
-    - install-basis
+        extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libzstd-dev ocl-icd-opencl-dev pocl-opencl-icd glslang-dev libopenexr-dev libwebp-dev spirv-tools
+    - install-basis:
+        tag: "1.16.4"
     - install-meshoptimizer
     - install-libspng
     - build:
@@ -392,6 +402,7 @@ jobs:
       CONFIGURATION: Debug
     steps:
     - install-base-linux:
+        # Basis 1.15 doesn't support OpenCL, so those packages aren't needed
         extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libwebp-dev libzstd-dev wget
     # Crashes or gets stuck with the default setting, only on GCC. 24 is not
     # enough.
@@ -400,7 +411,8 @@ jobs:
     - install-gcc-4_8
     - install-cmake:
         version: "3.5.2"
-    - install-basis
+    - install-basis:
+        tag: "v1_15_update2"
     - install-meshoptimizer
     - install-libspng
     # These two are in repos only since 20.04, but even then would probably hit
@@ -425,7 +437,7 @@ jobs:
       CONFIGURATION: Release
     steps:
     - install-base-linux:
-        extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libwebp-dev libzstd-dev wget
+        extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libwebp-dev libzstd-dev ocl-icd-opencl-dev pocl-opencl-icd wget
     # Crashes or gets stuck with the default setting, only on GCC
     - cap-ninja-jobs
     - install-gcc-4_8
@@ -455,6 +467,10 @@ jobs:
       BUILD_APPLICATIONS: "OFF"
       CMAKE_CXX_FLAGS: -fsanitize=address
       CONFIGURATION: Debug
+      # Basis, or the OpenCL ICD loader, or PoCL cause sanitizer errors when
+      # encoding with OpenCL. We're not installing any OpenCL packages, but
+      # better to be sure in case the image comes with it.
+      DISABLE_OPENCL: "ON"
     steps:
     - install-base-linux:
         extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libwebp-dev libzstd-dev wget
@@ -488,6 +504,8 @@ jobs:
       BUILD_APPLICATIONS: "OFF"
       CMAKE_CXX_FLAGS: -fsanitize=thread
       CONFIGURATION: Debug
+      # See linux-sanitizers for why this is necessary
+      DISABLE_OPENCL: "ON"
     steps:
     - install-base-linux:
         extra: libfaad-dev libfreetype6-dev libjpeg-dev libopenal-dev libpng-dev libdevil-dev libharfbuzz-dev libassimp-dev libopenexr-dev libwebp-dev libzstd-dev wget
@@ -510,6 +528,7 @@ jobs:
     environment:
       CMAKE_CXX_FLAGS: --coverage
       CONFIGURATION: Debug
+      DISABLE_OPENCL: "ON"
     steps:
     - install-base-macos:
         extra: faad2 harfbuzz assimp devil spirv-tools glslang openexr webp wget zstd libspng
@@ -529,7 +548,8 @@ jobs:
     steps:
     - install-base-macos:
         extra: faad2 harfbuzz assimp devil spirv-tools glslang openexr webp wget zstd libspng
-    - install-basis
+    - install-basis:
+        tag: "1.16.4"
     - install-meshoptimizer
     - build:
         script: unix-desktop.sh

package/ci/unix-desktop.sh

@@ -54,6 +54,7 @@ cmake .. \
     -DCMAKE_INSTALL_PREFIX=$HOME/deps \
     -DCMAKE_INSTALL_RPATH=$HOME/deps/lib \
     -DCMAKE_BUILD_TYPE=$CONFIGURATION \
+    -DCMAKE_DISABLE_FIND_PACKAGE_OpenCL=${DISABLE_OPENCL:-OFF} \
     -DMAGNUM_WITH_ASSIMPIMPORTER=ON \
     -DMAGNUM_WITH_ASTCIMPORTER=ON \
     -DMAGNUM_WITH_BASISIMAGECONVERTER=ON \

package/homebrew/magnum-plugins.rb

@@ -30,13 +30,13 @@ class MagnumPlugins < Formula
   depends_on "webp" => :recommended
 
   def install
-    # Bundle Basis Universal, v1_15_update2 for HEAD builds, a commit that's
+    # Bundle Basis Universal, v1_50_0_2 for HEAD builds, a commit that's
     # before the UASTC support (which was not implemented yet) on 2020.06.
     # The repo has massive useless files in its history, so we're downloading
     # just a snapshot instead of a git clone. Also, WHY THE FUCK curl needs -L
     # and -o?! why can't it just work?!
     if build.head?
-      system "curl", "-L", "https://github.com/BinomialLLC/basis_universal/archive/v1_15_update2.tar.gz", "-o", "src/external/basis-universal.tar.gz"
+      system "curl", "-L", "https://github.com/BinomialLLC/basis_universal/archive/v1_50_0_2.tar.gz", "-o", "src/external/basis-universal.tar.gz"
     else
       system "curl", "-L", "https://github.com/BinomialLLC/basis_universal/archive/2f43afcc97d0a5dafdb73b4e24e123cf9687a418.tar.gz", "-o", "src/external/basis-universal.tar.gz"
     end

src/MagnumPlugins/BasisImageConverter/BasisImageConverter.conf

@@ -77,6 +77,10 @@ rdo_uastc_favor_simpler_modes_in_rdo_mode=true
 ktx2_uastc_supercompression=true
 ktx2_zstd_supercompression_level=6
 
+# OpenCL acceleration. Falls back to CPU encoding if OpenCL isn't supported or
+# fails during encoding.
+use_opencl=false
+
 # Set various fields in the Basis file header
 userdata0=0
 userdata1=0