diff --git a/Makefile b/Makefile index c9f550f30577e..4e1451b0d5156 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ BUILD_TARGETS = \ llama-speculative \ llama-tokenize \ llama-vdot \ + llama-simple-vision \ llama-cvector-generator \ llama-gen-docs \ tests/test-c.o @@ -1351,6 +1352,11 @@ llama-gbnf-validator: examples/gbnf-validator/gbnf-validator.cpp \ $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +llama-simple-vision: examples/simple-vision/simple-vision.cpp \ + $(OBJ_ALL) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + ifdef GGML_RPC rpc-server: examples/rpc/rpc-server.cpp \ $(OBJ_GGML) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 7ef1ff74063d2..25c6cd75a585e 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -73,6 +73,7 @@ add_library(${TARGET} STATIC vision.h vision.cpp stb_image.h + stb_image_resize2.h ) if (BUILD_SHARED_LIBS) diff --git a/common/arg.cpp b/common/arg.cpp index deb11378657f4..bc4b4dd7cd445 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1357,7 +1357,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.image.emplace_back(value); } - ).set_examples({LLAMA_EXAMPLE_LLAVA})); + ).set_examples({LLAMA_EXAMPLE_LLAVA, LLAMA_EXAMPLE_VISION})); + add_opt(common_arg( + {"--n-tiles"}, "N", + "number of tiles to split the image into. use with multimodal models.", + [](common_params & params, int value) { + params.n_tiles = value; + } + ).set_examples({LLAMA_EXAMPLE_VISION})); + add_opt(common_arg( + {"--cross-attention"}, + "enables cross attention for multimodal models.", + [](common_params & params) { + params.cross_attention = true; + } + ).set_examples({LLAMA_EXAMPLE_VISION})); if (llama_supports_rpc()) { add_opt(common_arg( {"--rpc"}, "SERVERS", diff --git a/common/common.cpp b/common/common.cpp index 286b9bd9c9f6a..df1c96d06885d 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1587,7 +1587,7 @@ std::vector llama_tokenize_with_img( std::vector output; for (const auto & part : parts) { bool add_bos = &parts.front() == ∂ - auto tokens = llama_tokenize(ctx, part, add_special && add_bos, parse_special); + auto tokens = common_tokenize(ctx, part, add_special && add_bos, parse_special); output.insert(output.end(), tokens.begin(), tokens.end()); if (&parts.back() != &part) { // add image token to middle of 2 parts diff --git a/common/common.h b/common/common.h index 3e602c0c36dae..a1e8c6d2e8852 100644 --- a/common/common.h +++ b/common/common.h @@ -83,6 +83,7 @@ enum llama_example { LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_COUNT, + LLAMA_EXAMPLE_VISION, }; enum common_sampler_type { @@ -306,6 +307,8 @@ struct common_params { // multimodal models (see examples/llava) std::string mmproj = ""; // path to multimodal projector // NOLINT std::vector image; // path to image file(s) + bool cross_attention = false; + int n_tiles = 1; // embedding bool embedding = false; // get only sentence embedding @@ -385,6 +388,7 @@ struct common_params { // batched-bench params bool batched_bench_output_jsonl = false; + }; // call once at the start of a program if it uses libcommon diff --git a/common/stb_image_resize2.h b/common/stb_image_resize2.h new file mode 100644 index 0000000000000..2f26274636e6b --- /dev/null +++ b/common/stb_image_resize2.h @@ -0,0 +1,10601 @@ +/* stb_image_resize2 - v2.12 - public domain image resizing + + by Jeff Roberts (v2) and Jorge L Rodriguez + http://github.com/nothings/stb + + Can be threaded with the extended API. SSE2, AVX, Neon and WASM SIMD support. Only + scaling and translation is supported, no rotations or shears. + + COMPILING & LINKING + In one C/C++ file that #includes this file, do this: + #define STB_IMAGE_RESIZE_IMPLEMENTATION + before the #include. That will create the implementation in that file. + + EASY API CALLS: + Easy API downsamples w/Mitchell filter, upsamples w/cubic interpolation, clamps to edge. + + stbir_resize_uint8_srgb( input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + pixel_layout_enum ) + + stbir_resize_uint8_linear( input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + pixel_layout_enum ) + + stbir_resize_float_linear( input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + pixel_layout_enum ) + + If you pass NULL or zero for the output_pixels, we will allocate the output buffer + for you and return it from the function (free with free() or STBIR_FREE). + As a special case, XX_stride_in_bytes of 0 means packed continuously in memory. + + API LEVELS + There are three levels of API - easy-to-use, medium-complexity and extended-complexity. + + See the "header file" section of the source for API documentation. + + ADDITIONAL DOCUMENTATION + + MEMORY ALLOCATION + By default, we use malloc and free for memory allocation. To override the + memory allocation, before the implementation #include, add a: + + #define STBIR_MALLOC(size,user_data) ... + #define STBIR_FREE(ptr,user_data) ... + + Each resize makes exactly one call to malloc/free (unless you use the + extended API where you can do one allocation for many resizes). Under + address sanitizer, we do separate allocations to find overread/writes. + + PERFORMANCE + This library was written with an emphasis on performance. When testing + stb_image_resize with RGBA, the fastest mode is STBIR_4CHANNEL with + STBIR_TYPE_UINT8 pixels and CLAMPed edges (which is what many other resize + libs do by default). Also, make sure SIMD is turned on of course (default + for 64-bit targets). Avoid WRAP edge mode if you want the fastest speed. + + This library also comes with profiling built-in. If you define STBIR_PROFILE, + you can use the advanced API and get low-level profiling information by + calling stbir_resize_extended_profile_info() or stbir_resize_split_profile_info() + after a resize. + + SIMD + Most of the routines have optimized SSE2, AVX, NEON and WASM versions. + + On Microsoft compilers, we automatically turn on SIMD for 64-bit x64 and + ARM; for 32-bit x86 and ARM, you select SIMD mode by defining STBIR_SSE2 or + STBIR_NEON. For AVX and AVX2, we auto-select it by detecting the /arch:AVX + or /arch:AVX2 switches. You can also always manually turn SSE2, AVX or AVX2 + support on by defining STBIR_SSE2, STBIR_AVX or STBIR_AVX2. + + On Linux, SSE2 and Neon is on by default for 64-bit x64 or ARM64. For 32-bit, + we select x86 SIMD mode by whether you have -msse2, -mavx or -mavx2 enabled + on the command line. For 32-bit ARM, you must pass -mfpu=neon-vfpv4 for both + clang and GCC, but GCC also requires an additional -mfp16-format=ieee to + automatically enable NEON. + + On x86 platforms, you can also define STBIR_FP16C to turn on FP16C instructions + for converting back and forth to half-floats. This is autoselected when we + are using AVX2. Clang and GCC also require the -mf16c switch. ARM always uses + the built-in half float hardware NEON instructions. + + You can also tell us to use multiply-add instructions with STBIR_USE_FMA. + Because x86 doesn't always have fma, we turn it off by default to maintain + determinism across all platforms. If you don't care about non-FMA determinism + and are willing to restrict yourself to more recent x86 CPUs (around the AVX + timeframe), then fma will give you around a 15% speedup. + + You can force off SIMD in all cases by defining STBIR_NO_SIMD. You can turn + off AVX or AVX2 specifically with STBIR_NO_AVX or STBIR_NO_AVX2. AVX is 10% + to 40% faster, and AVX2 is generally another 12%. + + ALPHA CHANNEL + Most of the resizing functions provide the ability to control how the alpha + channel of an image is processed. + + When alpha represents transparency, it is important that when combining + colors with filtering, the pixels should not be treated equally; they + should use a weighted average based on their alpha values. For example, + if a pixel is 1% opaque bright green and another pixel is 99% opaque + black and you average them, the average will be 50% opaque, but the + unweighted average and will be a middling green color, while the weighted + average will be nearly black. This means the unweighted version introduced + green energy that didn't exist in the source image. + + (If you want to know why this makes sense, you can work out the math for + the following: consider what happens if you alpha composite a source image + over a fixed color and then average the output, vs. if you average the + source image pixels and then composite that over the same fixed color. + Only the weighted average produces the same result as the ground truth + composite-then-average result.) + + Therefore, it is in general best to "alpha weight" the pixels when applying + filters to them. This essentially means multiplying the colors by the alpha + values before combining them, and then dividing by the alpha value at the + end. + + The computer graphics industry introduced a technique called "premultiplied + alpha" or "associated alpha" in which image colors are stored in image files + already multiplied by their alpha. This saves some math when compositing, + and also avoids the need to divide by the alpha at the end (which is quite + inefficient). However, while premultiplied alpha is common in the movie CGI + industry, it is not commonplace in other industries like videogames, and most + consumer file formats are generally expected to contain not-premultiplied + colors. For example, Photoshop saves PNG files "unpremultiplied", and web + browsers like Chrome and Firefox expect PNG images to be unpremultiplied. + + Note that there are three possibilities that might describe your image + and resize expectation: + + 1. images are not premultiplied, alpha weighting is desired + 2. images are not premultiplied, alpha weighting is not desired + 3. images are premultiplied + + Both case #2 and case #3 require the exact same math: no alpha weighting + should be applied or removed. Only case 1 requires extra math operations; + the other two cases can be handled identically. + + stb_image_resize expects case #1 by default, applying alpha weighting to + images, expecting the input images to be unpremultiplied. This is what the + COLOR+ALPHA buffer types tell the resizer to do. + + When you use the pixel layouts STBIR_RGBA, STBIR_BGRA, STBIR_ARGB, + STBIR_ABGR, STBIR_RX, or STBIR_XR you are telling us that the pixels are + non-premultiplied. In these cases, the resizer will alpha weight the colors + (effectively creating the premultiplied image), do the filtering, and then + convert back to non-premult on exit. + + When you use the pixel layouts STBIR_RGBA_PM, STBIR_RGBA_PM, STBIR_RGBA_PM, + STBIR_RGBA_PM, STBIR_RX_PM or STBIR_XR_PM, you are telling that the pixels + ARE premultiplied. In this case, the resizer doesn't have to do the + premultipling - it can filter directly on the input. This about twice as + fast as the non-premultiplied case, so it's the right option if your data is + already setup correctly. + + When you use the pixel layout STBIR_4CHANNEL or STBIR_2CHANNEL, you are + telling us that there is no channel that represents transparency; it may be + RGB and some unrelated fourth channel that has been stored in the alpha + channel, but it is actually not alpha. No special processing will be + performed. + + The difference between the generic 4 or 2 channel layouts, and the + specialized _PM versions is with the _PM versions you are telling us that + the data *is* alpha, just don't premultiply it. That's important when + using SRGB pixel formats, we need to know where the alpha is, because + it is converted linearly (rather than with the SRGB converters). + + Because alpha weighting produces the same effect as premultiplying, you + even have the option with non-premultiplied inputs to let the resizer + produce a premultiplied output. Because the intially computed alpha-weighted + output image is effectively premultiplied, this is actually more performant + than the normal path which un-premultiplies the output image as a final step. + + Finally, when converting both in and out of non-premulitplied space (for + example, when using STBIR_RGBA), we go to somewhat heroic measures to + ensure that areas with zero alpha value pixels get something reasonable + in the RGB values. If you don't care about the RGB values of zero alpha + pixels, you can call the stbir_set_non_pm_alpha_speed_over_quality() + function - this runs a premultiplied resize about 25% faster. That said, + when you really care about speed, using premultiplied pixels for both in + and out (STBIR_RGBA_PM, etc) much faster than both of these premultiplied + options. + + PIXEL LAYOUT CONVERSION + The resizer can convert from some pixel layouts to others. When using the + stbir_set_pixel_layouts(), you can, for example, specify STBIR_RGBA + on input, and STBIR_ARGB on output, and it will re-organize the channels + during the resize. Currently, you can only convert between two pixel + layouts with the same number of channels. + + DETERMINISM + We commit to being deterministic (from x64 to ARM to scalar to SIMD, etc). + This requires compiling with fast-math off (using at least /fp:precise). + Also, you must turn off fp-contracting (which turns mult+adds into fmas)! + We attempt to do this with pragmas, but with Clang, you usually want to add + -ffp-contract=off to the command line as well. + + For 32-bit x86, you must use SSE and SSE2 codegen for determinism. That is, + if the scalar x87 unit gets used at all, we immediately lose determinism. + On Microsoft Visual Studio 2008 and earlier, from what we can tell there is + no way to be deterministic in 32-bit x86 (some x87 always leaks in, even + with fp:strict). On 32-bit x86 GCC, determinism requires both -msse2 and + -fpmath=sse. + + Note that we will not be deterministic with float data containing NaNs - + the NaNs will propagate differently on different SIMD and platforms. + + If you turn on STBIR_USE_FMA, then we will be deterministic with other + fma targets, but we will differ from non-fma targets (this is unavoidable, + because a fma isn't simply an add with a mult - it also introduces a + rounding difference compared to non-fma instruction sequences. + + FLOAT PIXEL FORMAT RANGE + Any range of values can be used for the non-alpha float data that you pass + in (0 to 1, -1 to 1, whatever). However, if you are inputting float values + but *outputting* bytes or shorts, you must use a range of 0 to 1 so that we + scale back properly. The alpha channel must also be 0 to 1 for any format + that does premultiplication prior to resizing. + + Note also that with float output, using filters with negative lobes, the + output filtered values might go slightly out of range. You can define + STBIR_FLOAT_LOW_CLAMP and/or STBIR_FLOAT_HIGH_CLAMP to specify the range + to clamp to on output, if that's important. + + MAX/MIN SCALE FACTORS + The input pixel resolutions are in integers, and we do the internal pointer + resolution in size_t sized integers. However, the scale ratio from input + resolution to output resolution is calculated in float form. This means + the effective possible scale ratio is limited to 24 bits (or 16 million + to 1). As you get close to the size of the float resolution (again, 16 + million pixels wide or high), you might start seeing float inaccuracy + issues in general in the pipeline. If you have to do extreme resizes, + you can usually do this is multiple stages (using float intermediate + buffers). + + FLIPPED IMAGES + Stride is just the delta from one scanline to the next. This means you can + use a negative stride to handle inverted images (point to the final + scanline and use a negative stride). You can invert the input or output, + using negative strides. + + DEFAULT FILTERS + For functions which don't provide explicit control over what filters to + use, you can change the compile-time defaults with: + + #define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_something + #define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_something + + See stbir_filter in the header-file section for the list of filters. + + NEW FILTERS + A number of 1D filter kernels are supplied. For a list of supported + filters, see the stbir_filter enum. You can install your own filters by + using the stbir_set_filter_callbacks function. + + PROGRESS + For interactive use with slow resize operations, you can use the the + scanline callbacks in the extended API. It would have to be a *very* large + image resample to need progress though - we're very fast. + + CEIL and FLOOR + In scalar mode, the only functions we use from math.h are ceilf and floorf, + but if you have your own versions, you can define the STBIR_CEILF(v) and + STBIR_FLOORF(v) macros and we'll use them instead. In SIMD, we just use + our own versions. + + ASSERT + Define STBIR_ASSERT(boolval) to override assert() and not use assert.h + + PORTING FROM VERSION 1 + The API has changed. You can continue to use the old version of stb_image_resize.h, + which is available in the "deprecated/" directory. + + If you're using the old simple-to-use API, porting is straightforward. + (For more advanced APIs, read the documentation.) + + stbir_resize_uint8(): + - call `stbir_resize_uint8_linear`, cast channel count to `stbir_pixel_layout` + + stbir_resize_float(): + - call `stbir_resize_float_linear`, cast channel count to `stbir_pixel_layout` + + stbir_resize_uint8_srgb(): + - function name is unchanged + - cast channel count to `stbir_pixel_layout` + - above is sufficient unless your image has alpha and it's not RGBA/BGRA + - in that case, follow the below instructions for stbir_resize_uint8_srgb_edgemode + + stbir_resize_uint8_srgb_edgemode() + - switch to the "medium complexity" API + - stbir_resize(), very similar API but a few more parameters: + - pixel_layout: cast channel count to `stbir_pixel_layout` + - data_type: STBIR_TYPE_UINT8_SRGB + - edge: unchanged (STBIR_EDGE_WRAP, etc.) + - filter: STBIR_FILTER_DEFAULT + - which channel is alpha is specified in stbir_pixel_layout, see enum for details + + FUTURE TODOS + * For polyphase integral filters, we just memcpy the coeffs to dupe + them, but we should indirect and use the same coeff memory. + * Add pixel layout conversions for sensible different channel counts + (maybe, 1->3/4, 3->4, 4->1, 3->1). + * For SIMD encode and decode scanline routines, do any pre-aligning + for bad input/output buffer alignments and pitch? + * For very wide scanlines, we should we do vertical strips to stay within + L2 cache. Maybe do chunks of 1K pixels at a time. There would be + some pixel reconversion, but probably dwarfed by things falling out + of cache. Probably also something possible with alternating between + scattering and gathering at high resize scales? + * Rewrite the coefficient generator to do many at once. + * AVX-512 vertical kernels - worried about downclocking here. + * Convert the reincludes to macros when we know they aren't changing. + * Experiment with pivoting the horizontal and always using the + vertical filters (which are faster, but perhaps not enough to overcome + the pivot cost and the extra memory touches). Need to buffer the whole + image so have to balance memory use. + * Most of our code is internally function pointers, should we compile + all the SIMD stuff always and dynamically dispatch? + + CONTRIBUTORS + Jeff Roberts: 2.0 implementation, optimizations, SIMD + Martins Mozeiko: NEON simd, WASM simd, clang and GCC whisperer + Fabian Giesen: half float and srgb converters + Sean Barrett: API design, optimizations + Jorge L Rodriguez: Original 1.0 implementation + Aras Pranckevicius: bugfixes + Nathan Reed: warning fixes for 1.0 + + REVISIONS + 2.12 (2024-10-18) fix incorrect use of user_data with STBIR_FREE + 2.11 (2024-09-08) fix harmless asan warnings in 2-channel and 3-channel mode + with AVX-2, fix some weird scaling edge conditions with + point sample mode. + 2.10 (2024-07-27) fix the defines GCC and mingw for loop unroll control, + fix MSVC 32-bit arm half float routines. + 2.09 (2024-06-19) fix the defines for 32-bit ARM GCC builds (was selecting + hardware half floats). + 2.08 (2024-06-10) fix for RGB->BGR three channel flips and add SIMD (thanks + to Ryan Salsbury), fix for sub-rect resizes, use the + pragmas to control unrolling when they are available. + 2.07 (2024-05-24) fix for slow final split during threaded conversions of very + wide scanlines when downsampling (caused by extra input + converting), fix for wide scanline resamples with many + splits (int overflow), fix GCC warning. + 2.06 (2024-02-10) fix for identical width/height 3x or more down-scaling + undersampling a single row on rare resize ratios (about 1%). + 2.05 (2024-02-07) fix for 2 pixel to 1 pixel resizes with wrap (thanks Aras), + fix for output callback (thanks Julien Koenen). + 2.04 (2023-11-17) fix for rare AVX bug, shadowed symbol (thanks Nikola Smiljanic). + 2.03 (2023-11-01) ASAN and TSAN warnings fixed, minor tweaks. + 2.00 (2023-10-10) mostly new source: new api, optimizations, simd, vertical-first, etc + 2x-5x faster without simd, 4x-12x faster with simd, + in some cases, 20x to 40x faster esp resizing large to very small. + 0.96 (2019-03-04) fixed warnings + 0.95 (2017-07-23) fixed warnings + 0.94 (2017-03-18) fixed warnings + 0.93 (2017-03-03) fixed bug with certain combinations of heights + 0.92 (2017-01-02) fix integer overflow on large (>2GB) images + 0.91 (2016-04-02) fix warnings; fix handling of subpixel regions + 0.90 (2014-09-17) first released version + + LICENSE + See end of file for license information. +*/ + +#if !defined(STB_IMAGE_RESIZE_DO_HORIZONTALS) && !defined(STB_IMAGE_RESIZE_DO_VERTICALS) && !defined(STB_IMAGE_RESIZE_DO_CODERS) // for internal re-includes + +#ifndef STBIR_INCLUDE_STB_IMAGE_RESIZE2_H +#define STBIR_INCLUDE_STB_IMAGE_RESIZE2_H + +#include +#ifdef _MSC_VER +typedef unsigned char stbir_uint8; +typedef unsigned short stbir_uint16; +typedef unsigned int stbir_uint32; +typedef unsigned __int64 stbir_uint64; +#else +#include +typedef uint8_t stbir_uint8; +typedef uint16_t stbir_uint16; +typedef uint32_t stbir_uint32; +typedef uint64_t stbir_uint64; +#endif + +#ifdef _M_IX86_FP +#if ( _M_IX86_FP >= 1 ) +#ifndef STBIR_SSE +#define STBIR_SSE +#endif +#endif +#endif + +#if defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(_M_AMD64) || defined(__SSE2__) || defined(STBIR_SSE) || defined(STBIR_SSE2) + #ifndef STBIR_SSE2 + #define STBIR_SSE2 + #endif + #if defined(__AVX__) || defined(STBIR_AVX2) + #ifndef STBIR_AVX + #ifndef STBIR_NO_AVX + #define STBIR_AVX + #endif + #endif + #endif + #if defined(__AVX2__) || defined(STBIR_AVX2) + #ifndef STBIR_NO_AVX2 + #ifndef STBIR_AVX2 + #define STBIR_AVX2 + #endif + #if defined( _MSC_VER ) && !defined(__clang__) + #ifndef STBIR_FP16C // FP16C instructions are on all AVX2 cpus, so we can autoselect it here on microsoft - clang needs -m16c + #define STBIR_FP16C + #endif + #endif + #endif + #endif + #ifdef __F16C__ + #ifndef STBIR_FP16C // turn on FP16C instructions if the define is set (for clang and gcc) + #define STBIR_FP16C + #endif + #endif +#endif + +#if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || ((__ARM_NEON_FP & 4) != 0) || defined(__ARM_NEON__) +#ifndef STBIR_NEON +#define STBIR_NEON +#endif +#endif + +#if defined(_M_ARM) || defined(__arm__) +#ifdef STBIR_USE_FMA +#undef STBIR_USE_FMA // no FMA for 32-bit arm on MSVC +#endif +#endif + +#if defined(__wasm__) && defined(__wasm_simd128__) +#ifndef STBIR_WASM +#define STBIR_WASM +#endif +#endif + +#ifndef STBIRDEF +#ifdef STB_IMAGE_RESIZE_STATIC +#define STBIRDEF static +#else +#ifdef __cplusplus +#define STBIRDEF extern "C" +#else +#define STBIRDEF extern +#endif +#endif +#endif + +////////////////////////////////////////////////////////////////////////////// +//// start "header file" /////////////////////////////////////////////////// +// +// Easy-to-use API: +// +// * stride is the offset between successive rows of image data +// in memory, in bytes. specify 0 for packed continuously in memory +// * colorspace is linear or sRGB as specified by function name +// * Uses the default filters +// * Uses edge mode clamped +// * returned result is 1 for success or 0 in case of an error. + + +// stbir_pixel_layout specifies: +// number of channels +// order of channels +// whether color is premultiplied by alpha +// for back compatibility, you can cast the old channel count to an stbir_pixel_layout +typedef enum +{ + STBIR_1CHANNEL = 1, + STBIR_2CHANNEL = 2, + STBIR_RGB = 3, // 3-chan, with order specified (for channel flipping) + STBIR_BGR = 0, // 3-chan, with order specified (for channel flipping) + STBIR_4CHANNEL = 5, + + STBIR_RGBA = 4, // alpha formats, where alpha is NOT premultiplied into color channels + STBIR_BGRA = 6, + STBIR_ARGB = 7, + STBIR_ABGR = 8, + STBIR_RA = 9, + STBIR_AR = 10, + + STBIR_RGBA_PM = 11, // alpha formats, where alpha is premultiplied into color channels + STBIR_BGRA_PM = 12, + STBIR_ARGB_PM = 13, + STBIR_ABGR_PM = 14, + STBIR_RA_PM = 15, + STBIR_AR_PM = 16, + + STBIR_RGBA_NO_AW = 11, // alpha formats, where NO alpha weighting is applied at all! + STBIR_BGRA_NO_AW = 12, // these are just synonyms for the _PM flags (which also do + STBIR_ARGB_NO_AW = 13, // no alpha weighting). These names just make it more clear + STBIR_ABGR_NO_AW = 14, // for some folks). + STBIR_RA_NO_AW = 15, + STBIR_AR_NO_AW = 16, + +} stbir_pixel_layout; + +//=============================================================== +// Simple-complexity API +// +// If output_pixels is NULL (0), then we will allocate the buffer and return it to you. +//-------------------------------- + +STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_type ); + +STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_type ); + +STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_type ); +//=============================================================== + +//=============================================================== +// Medium-complexity API +// +// This extends the easy-to-use API as follows: +// +// * Can specify the datatype - U8, U8_SRGB, U16, FLOAT, HALF_FLOAT +// * Edge wrap can selected explicitly +// * Filter can be selected explicitly +//-------------------------------- + +typedef enum +{ + STBIR_EDGE_CLAMP = 0, + STBIR_EDGE_REFLECT = 1, + STBIR_EDGE_WRAP = 2, // this edge mode is slower and uses more memory + STBIR_EDGE_ZERO = 3, +} stbir_edge; + +typedef enum +{ + STBIR_FILTER_DEFAULT = 0, // use same filter type that easy-to-use API chooses + STBIR_FILTER_BOX = 1, // A trapezoid w/1-pixel wide ramps, same result as box for integer scale ratios + STBIR_FILTER_TRIANGLE = 2, // On upsampling, produces same results as bilinear texture filtering + STBIR_FILTER_CUBICBSPLINE = 3, // The cubic b-spline (aka Mitchell-Netrevalli with B=1,C=0), gaussian-esque + STBIR_FILTER_CATMULLROM = 4, // An interpolating cubic spline + STBIR_FILTER_MITCHELL = 5, // Mitchell-Netrevalli filter with B=1/3, C=1/3 + STBIR_FILTER_POINT_SAMPLE = 6, // Simple point sampling + STBIR_FILTER_OTHER = 7, // User callback specified +} stbir_filter; + +typedef enum +{ + STBIR_TYPE_UINT8 = 0, + STBIR_TYPE_UINT8_SRGB = 1, + STBIR_TYPE_UINT8_SRGB_ALPHA = 2, // alpha channel, when present, should also be SRGB (this is very unusual) + STBIR_TYPE_UINT16 = 3, + STBIR_TYPE_FLOAT = 4, + STBIR_TYPE_HALF_FLOAT = 5 +} stbir_datatype; + +// medium api +STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_layout, stbir_datatype data_type, + stbir_edge edge, stbir_filter filter ); +//=============================================================== + + + +//=============================================================== +// Extended-complexity API +// +// This API exposes all resize functionality. +// +// * Separate filter types for each axis +// * Separate edge modes for each axis +// * Separate input and output data types +// * Can specify regions with subpixel correctness +// * Can specify alpha flags +// * Can specify a memory callback +// * Can specify a callback data type for pixel input and output +// * Can be threaded for a single resize +// * Can be used to resize many frames without recalculating the sampler info +// +// Use this API as follows: +// 1) Call the stbir_resize_init function on a local STBIR_RESIZE structure +// 2) Call any of the stbir_set functions +// 3) Optionally call stbir_build_samplers() if you are going to resample multiple times +// with the same input and output dimensions (like resizing video frames) +// 4) Resample by calling stbir_resize_extended(). +// 5) Call stbir_free_samplers() if you called stbir_build_samplers() +//-------------------------------- + + +// Types: + +// INPUT CALLBACK: this callback is used for input scanlines +typedef void const * stbir_input_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context ); + +// OUTPUT CALLBACK: this callback is used for output scanlines +typedef void stbir_output_callback( void const * output_ptr, int num_pixels, int y, void * context ); + +// callbacks for user installed filters +typedef float stbir__kernel_callback( float x, float scale, void * user_data ); // centered at zero +typedef float stbir__support_callback( float scale, void * user_data ); + +// internal structure with precomputed scaling +typedef struct stbir__info stbir__info; + +typedef struct STBIR_RESIZE // use the stbir_resize_init and stbir_override functions to set these values for future compatibility +{ + void * user_data; + void const * input_pixels; + int input_w, input_h; + double input_s0, input_t0, input_s1, input_t1; + stbir_input_callback * input_cb; + void * output_pixels; + int output_w, output_h; + int output_subx, output_suby, output_subw, output_subh; + stbir_output_callback * output_cb; + int input_stride_in_bytes; + int output_stride_in_bytes; + int splits; + int fast_alpha; + int needs_rebuild; + int called_alloc; + stbir_pixel_layout input_pixel_layout_public; + stbir_pixel_layout output_pixel_layout_public; + stbir_datatype input_data_type; + stbir_datatype output_data_type; + stbir_filter horizontal_filter, vertical_filter; + stbir_edge horizontal_edge, vertical_edge; + stbir__kernel_callback * horizontal_filter_kernel; stbir__support_callback * horizontal_filter_support; + stbir__kernel_callback * vertical_filter_kernel; stbir__support_callback * vertical_filter_support; + stbir__info * samplers; +} STBIR_RESIZE; + +// extended complexity api + + +// First off, you must ALWAYS call stbir_resize_init on your resize structure before any of the other calls! +STBIRDEF void stbir_resize_init( STBIR_RESIZE * resize, + const void *input_pixels, int input_w, int input_h, int input_stride_in_bytes, // stride can be zero + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, // stride can be zero + stbir_pixel_layout pixel_layout, stbir_datatype data_type ); + +//=============================================================== +// You can update these parameters any time after resize_init and there is no cost +//-------------------------------- + +STBIRDEF void stbir_set_datatypes( STBIR_RESIZE * resize, stbir_datatype input_type, stbir_datatype output_type ); +STBIRDEF void stbir_set_pixel_callbacks( STBIR_RESIZE * resize, stbir_input_callback * input_cb, stbir_output_callback * output_cb ); // no callbacks by default +STBIRDEF void stbir_set_user_data( STBIR_RESIZE * resize, void * user_data ); // pass back STBIR_RESIZE* by default +STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, int input_stride_in_bytes, void * output_pixels, int output_stride_in_bytes ); + +//=============================================================== + + +//=============================================================== +// If you call any of these functions, you will trigger a sampler rebuild! +//-------------------------------- + +STBIRDEF int stbir_set_pixel_layouts( STBIR_RESIZE * resize, stbir_pixel_layout input_pixel_layout, stbir_pixel_layout output_pixel_layout ); // sets new buffer layouts +STBIRDEF int stbir_set_edgemodes( STBIR_RESIZE * resize, stbir_edge horizontal_edge, stbir_edge vertical_edge ); // CLAMP by default + +STBIRDEF int stbir_set_filters( STBIR_RESIZE * resize, stbir_filter horizontal_filter, stbir_filter vertical_filter ); // STBIR_DEFAULT_FILTER_UPSAMPLE/DOWNSAMPLE by default +STBIRDEF int stbir_set_filter_callbacks( STBIR_RESIZE * resize, stbir__kernel_callback * horizontal_filter, stbir__support_callback * horizontal_support, stbir__kernel_callback * vertical_filter, stbir__support_callback * vertical_support ); + +STBIRDEF int stbir_set_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ); // sets both sub-regions (full regions by default) +STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, double s0, double t0, double s1, double t1 ); // sets input sub-region (full region by default) +STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ); // sets output sub-region (full region by default) + +// when inputting AND outputting non-premultiplied alpha pixels, we use a slower but higher quality technique +// that fills the zero alpha pixel's RGB values with something plausible. If you don't care about areas of +// zero alpha, you can call this function to get about a 25% speed improvement for STBIR_RGBA to STBIR_RGBA +// types of resizes. +STBIRDEF int stbir_set_non_pm_alpha_speed_over_quality( STBIR_RESIZE * resize, int non_pma_alpha_speed_over_quality ); +//=============================================================== + + +//=============================================================== +// You can call build_samplers to prebuild all the internal data we need to resample. +// Then, if you call resize_extended many times with the same resize, you only pay the +// cost once. +// If you do call build_samplers, you MUST call free_samplers eventually. +//-------------------------------- + +// This builds the samplers and does one allocation +STBIRDEF int stbir_build_samplers( STBIR_RESIZE * resize ); + +// You MUST call this, if you call stbir_build_samplers or stbir_build_samplers_with_splits +STBIRDEF void stbir_free_samplers( STBIR_RESIZE * resize ); +//=============================================================== + + +// And this is the main function to perform the resize synchronously on one thread. +STBIRDEF int stbir_resize_extended( STBIR_RESIZE * resize ); + + +//=============================================================== +// Use these functions for multithreading. +// 1) You call stbir_build_samplers_with_splits first on the main thread +// 2) Then stbir_resize_with_split on each thread +// 3) stbir_free_samplers when done on the main thread +//-------------------------------- + +// This will build samplers for threading. +// You can pass in the number of threads you'd like to use (try_splits). +// It returns the number of splits (threads) that you can call it with. +/// It might be less if the image resize can't be split up that many ways. + +STBIRDEF int stbir_build_samplers_with_splits( STBIR_RESIZE * resize, int try_splits ); + +// This function does a split of the resizing (you call this fuction for each +// split, on multiple threads). A split is a piece of the output resize pixel space. + +// Note that you MUST call stbir_build_samplers_with_splits before stbir_resize_extended_split! + +// Usually, you will always call stbir_resize_split with split_start as the thread_index +// and "1" for the split_count. +// But, if you have a weird situation where you MIGHT want 8 threads, but sometimes +// only 4 threads, you can use 0,2,4,6 for the split_start's and use "2" for the +// split_count each time to turn in into a 4 thread resize. (This is unusual). + +STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start, int split_count ); +//=============================================================== + + +//=============================================================== +// Pixel Callbacks info: +//-------------------------------- + +// The input callback is super flexible - it calls you with the input address +// (based on the stride and base pointer), it gives you an optional_output +// pointer that you can fill, or you can just return your own pointer into +// your own data. +// +// You can also do conversion from non-supported data types if necessary - in +// this case, you ignore the input_ptr and just use the x and y parameters to +// calculate your own input_ptr based on the size of each non-supported pixel. +// (Something like the third example below.) +// +// You can also install just an input or just an output callback by setting the +// callback that you don't want to zero. +// +// First example, progress: (getting a callback that you can monitor the progress): +// void const * my_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context ) +// { +// percentage_done = y / input_height; +// return input_ptr; // use buffer from call +// } +// +// Next example, copying: (copy from some other buffer or stream): +// void const * my_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context ) +// { +// CopyOrStreamData( optional_output, other_data_src, num_pixels * pixel_width_in_bytes ); +// return optional_output; // return the optional buffer that we filled +// } +// +// Third example, input another buffer without copying: (zero-copy from other buffer): +// void const * my_callback( void * optional_output, void const * input_ptr, int num_pixels, int x, int y, void * context ) +// { +// void * pixels = ( (char*) other_image_base ) + ( y * other_image_stride ) + ( x * other_pixel_width_in_bytes ); +// return pixels; // return pointer to your data without copying +// } +// +// +// The output callback is considerably simpler - it just calls you so that you can dump +// out each scanline. You could even directly copy out to disk if you have a simple format +// like TGA or BMP. You can also convert to other output types here if you want. +// +// Simple example: +// void const * my_output( void * output_ptr, int num_pixels, int y, void * context ) +// { +// percentage_done = y / output_height; +// fwrite( output_ptr, pixel_width_in_bytes, num_pixels, output_file ); +// } +//=============================================================== + + + + +//=============================================================== +// optional built-in profiling API +//-------------------------------- + +#ifdef STBIR_PROFILE + +typedef struct STBIR_PROFILE_INFO +{ + stbir_uint64 total_clocks; + + // how many clocks spent (of total_clocks) in the various resize routines, along with a string description + // there are "resize_count" number of zones + stbir_uint64 clocks[ 8 ]; + char const ** descriptions; + + // count of clocks and descriptions + stbir_uint32 count; +} STBIR_PROFILE_INFO; + +// use after calling stbir_resize_extended (or stbir_build_samplers or stbir_build_samplers_with_splits) +STBIRDEF void stbir_resize_build_profile_info( STBIR_PROFILE_INFO * out_info, STBIR_RESIZE const * resize ); + +// use after calling stbir_resize_extended +STBIRDEF void stbir_resize_extended_profile_info( STBIR_PROFILE_INFO * out_info, STBIR_RESIZE const * resize ); + +// use after calling stbir_resize_extended_split +STBIRDEF void stbir_resize_split_profile_info( STBIR_PROFILE_INFO * out_info, STBIR_RESIZE const * resize, int split_start, int split_num ); + +//=============================================================== + +#endif + + +//// end header file ///////////////////////////////////////////////////// +#endif // STBIR_INCLUDE_STB_IMAGE_RESIZE2_H + +#if defined(STB_IMAGE_RESIZE_IMPLEMENTATION) || defined(STB_IMAGE_RESIZE2_IMPLEMENTATION) + +#ifndef STBIR_ASSERT +#include +#define STBIR_ASSERT(x) assert(x) +#endif + +#ifndef STBIR_MALLOC +#include +#define STBIR_MALLOC(size,user_data) ((void)(user_data), malloc(size)) +#define STBIR_FREE(ptr,user_data) ((void)(user_data), free(ptr)) +// (we used the comma operator to evaluate user_data, to avoid "unused parameter" warnings) +#endif + +#ifdef _MSC_VER + +#define stbir__inline __forceinline + +#else + +#define stbir__inline __inline__ + +// Clang address sanitizer +#if defined(__has_feature) + #if __has_feature(address_sanitizer) || __has_feature(memory_sanitizer) + #ifndef STBIR__SEPARATE_ALLOCATIONS + #define STBIR__SEPARATE_ALLOCATIONS + #endif + #endif +#endif + +#endif + +// GCC and MSVC +#if defined(__SANITIZE_ADDRESS__) + #ifndef STBIR__SEPARATE_ALLOCATIONS + #define STBIR__SEPARATE_ALLOCATIONS + #endif +#endif + +// Always turn off automatic FMA use - use STBIR_USE_FMA if you want. +// Otherwise, this is a determinism disaster. +#ifndef STBIR_DONT_CHANGE_FP_CONTRACT // override in case you don't want this behavior +#if defined(_MSC_VER) && !defined(__clang__) +#if _MSC_VER > 1200 +#pragma fp_contract(off) +#endif +#elif defined(__GNUC__) && !defined(__clang__) +#pragma GCC optimize("fp-contract=off") +#else +#pragma STDC FP_CONTRACT OFF +#endif +#endif + +#ifdef _MSC_VER +#define STBIR__UNUSED(v) (void)(v) +#else +#define STBIR__UNUSED(v) (void)sizeof(v) +#endif + +#define STBIR__ARRAY_SIZE(a) (sizeof((a))/sizeof((a)[0])) + + +#ifndef STBIR_DEFAULT_FILTER_UPSAMPLE +#define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_CATMULLROM +#endif + +#ifndef STBIR_DEFAULT_FILTER_DOWNSAMPLE +#define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_MITCHELL +#endif + + +#ifndef STBIR__HEADER_FILENAME +#define STBIR__HEADER_FILENAME "stb_image_resize2.h" +#endif + +// the internal pixel layout enums are in a different order, so we can easily do range comparisons of types +// the public pixel layout is ordered in a way that if you cast num_channels (1-4) to the enum, you get something sensible +typedef enum +{ + STBIRI_1CHANNEL = 0, + STBIRI_2CHANNEL = 1, + STBIRI_RGB = 2, + STBIRI_BGR = 3, + STBIRI_4CHANNEL = 4, + + STBIRI_RGBA = 5, + STBIRI_BGRA = 6, + STBIRI_ARGB = 7, + STBIRI_ABGR = 8, + STBIRI_RA = 9, + STBIRI_AR = 10, + + STBIRI_RGBA_PM = 11, + STBIRI_BGRA_PM = 12, + STBIRI_ARGB_PM = 13, + STBIRI_ABGR_PM = 14, + STBIRI_RA_PM = 15, + STBIRI_AR_PM = 16, +} stbir_internal_pixel_layout; + +// define the public pixel layouts to not compile inside the implementation (to avoid accidental use) +#define STBIR_BGR bad_dont_use_in_implementation +#define STBIR_1CHANNEL STBIR_BGR +#define STBIR_2CHANNEL STBIR_BGR +#define STBIR_RGB STBIR_BGR +#define STBIR_RGBA STBIR_BGR +#define STBIR_4CHANNEL STBIR_BGR +#define STBIR_BGRA STBIR_BGR +#define STBIR_ARGB STBIR_BGR +#define STBIR_ABGR STBIR_BGR +#define STBIR_RA STBIR_BGR +#define STBIR_AR STBIR_BGR +#define STBIR_RGBA_PM STBIR_BGR +#define STBIR_BGRA_PM STBIR_BGR +#define STBIR_ARGB_PM STBIR_BGR +#define STBIR_ABGR_PM STBIR_BGR +#define STBIR_RA_PM STBIR_BGR +#define STBIR_AR_PM STBIR_BGR + +// must match stbir_datatype +static unsigned char stbir__type_size[] = { + 1,1,1,2,4,2 // STBIR_TYPE_UINT8,STBIR_TYPE_UINT8_SRGB,STBIR_TYPE_UINT8_SRGB_ALPHA,STBIR_TYPE_UINT16,STBIR_TYPE_FLOAT,STBIR_TYPE_HALF_FLOAT +}; + +// When gathering, the contributors are which source pixels contribute. +// When scattering, the contributors are which destination pixels are contributed to. +typedef struct +{ + int n0; // First contributing pixel + int n1; // Last contributing pixel +} stbir__contributors; + +typedef struct +{ + int lowest; // First sample index for whole filter + int highest; // Last sample index for whole filter + int widest; // widest single set of samples for an output +} stbir__filter_extent_info; + +typedef struct +{ + int n0; // First pixel of decode buffer to write to + int n1; // Last pixel of decode that will be written to + int pixel_offset_for_input; // Pixel offset into input_scanline +} stbir__span; + +typedef struct stbir__scale_info +{ + int input_full_size; + int output_sub_size; + float scale; + float inv_scale; + float pixel_shift; // starting shift in output pixel space (in pixels) + int scale_is_rational; + stbir_uint32 scale_numerator, scale_denominator; +} stbir__scale_info; + +typedef struct +{ + stbir__contributors * contributors; + float* coefficients; + stbir__contributors * gather_prescatter_contributors; + float * gather_prescatter_coefficients; + stbir__scale_info scale_info; + float support; + stbir_filter filter_enum; + stbir__kernel_callback * filter_kernel; + stbir__support_callback * filter_support; + stbir_edge edge; + int coefficient_width; + int filter_pixel_width; + int filter_pixel_margin; + int num_contributors; + int contributors_size; + int coefficients_size; + stbir__filter_extent_info extent_info; + int is_gather; // 0 = scatter, 1 = gather with scale >= 1, 2 = gather with scale < 1 + int gather_prescatter_num_contributors; + int gather_prescatter_coefficient_width; + int gather_prescatter_contributors_size; + int gather_prescatter_coefficients_size; +} stbir__sampler; + +typedef struct +{ + stbir__contributors conservative; + int edge_sizes[2]; // this can be less than filter_pixel_margin, if the filter and scaling falls off + stbir__span spans[2]; // can be two spans, if doing input subrect with clamp mode WRAP +} stbir__extents; + +typedef struct +{ +#ifdef STBIR_PROFILE + union + { + struct { stbir_uint64 total, looping, vertical, horizontal, decode, encode, alpha, unalpha; } named; + stbir_uint64 array[8]; + } profile; + stbir_uint64 * current_zone_excluded_ptr; +#endif + float* decode_buffer; + + int ring_buffer_first_scanline; + int ring_buffer_last_scanline; + int ring_buffer_begin_index; // first_scanline is at this index in the ring buffer + int start_output_y, end_output_y; + int start_input_y, end_input_y; // used in scatter only + + #ifdef STBIR__SEPARATE_ALLOCATIONS + float** ring_buffers; // one pointer for each ring buffer + #else + float* ring_buffer; // one big buffer that we index into + #endif + + float* vertical_buffer; + + char no_cache_straddle[64]; +} stbir__per_split_info; + +typedef void stbir__decode_pixels_func( float * decode, int width_times_channels, void const * input ); +typedef void stbir__alpha_weight_func( float * decode_buffer, int width_times_channels ); +typedef void stbir__horizontal_gather_channels_func( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, + stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ); +typedef void stbir__alpha_unweight_func(float * encode_buffer, int width_times_channels ); +typedef void stbir__encode_pixels_func( void * output, int width_times_channels, float const * encode ); + +struct stbir__info +{ +#ifdef STBIR_PROFILE + union + { + struct { stbir_uint64 total, build, alloc, horizontal, vertical, cleanup, pivot; } named; + stbir_uint64 array[7]; + } profile; + stbir_uint64 * current_zone_excluded_ptr; +#endif + stbir__sampler horizontal; + stbir__sampler vertical; + + void const * input_data; + void * output_data; + + int input_stride_bytes; + int output_stride_bytes; + int ring_buffer_length_bytes; // The length of an individual entry in the ring buffer. The total number of ring buffers is stbir__get_filter_pixel_width(filter) + int ring_buffer_num_entries; // Total number of entries in the ring buffer. + + stbir_datatype input_type; + stbir_datatype output_type; + + stbir_input_callback * in_pixels_cb; + void * user_data; + stbir_output_callback * out_pixels_cb; + + stbir__extents scanline_extents; + + void * alloced_mem; + stbir__per_split_info * split_info; // by default 1, but there will be N of these allocated based on the thread init you did + + stbir__decode_pixels_func * decode_pixels; + stbir__alpha_weight_func * alpha_weight; + stbir__horizontal_gather_channels_func * horizontal_gather_channels; + stbir__alpha_unweight_func * alpha_unweight; + stbir__encode_pixels_func * encode_pixels; + + int alloc_ring_buffer_num_entries; // Number of entries in the ring buffer that will be allocated + int splits; // count of splits + + stbir_internal_pixel_layout input_pixel_layout_internal; + stbir_internal_pixel_layout output_pixel_layout_internal; + + int input_color_and_type; + int offset_x, offset_y; // offset within output_data + int vertical_first; + int channels; + int effective_channels; // same as channels, except on RGBA/ARGB (7), or XA/AX (3) + size_t alloced_total; +}; + + +#define stbir__max_uint8_as_float 255.0f +#define stbir__max_uint16_as_float 65535.0f +#define stbir__max_uint8_as_float_inverted (1.0f/255.0f) +#define stbir__max_uint16_as_float_inverted (1.0f/65535.0f) +#define stbir__small_float ((float)1 / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20)) + +// min/max friendly +#define STBIR_CLAMP(x, xmin, xmax) for(;;) { \ + if ( (x) < (xmin) ) (x) = (xmin); \ + if ( (x) > (xmax) ) (x) = (xmax); \ + break; \ +} + +static stbir__inline int stbir__min(int a, int b) +{ + return a < b ? a : b; +} + +static stbir__inline int stbir__max(int a, int b) +{ + return a > b ? a : b; +} + +static float stbir__srgb_uchar_to_linear_float[256] = { + 0.000000f, 0.000304f, 0.000607f, 0.000911f, 0.001214f, 0.001518f, 0.001821f, 0.002125f, 0.002428f, 0.002732f, 0.003035f, + 0.003347f, 0.003677f, 0.004025f, 0.004391f, 0.004777f, 0.005182f, 0.005605f, 0.006049f, 0.006512f, 0.006995f, 0.007499f, + 0.008023f, 0.008568f, 0.009134f, 0.009721f, 0.010330f, 0.010960f, 0.011612f, 0.012286f, 0.012983f, 0.013702f, 0.014444f, + 0.015209f, 0.015996f, 0.016807f, 0.017642f, 0.018500f, 0.019382f, 0.020289f, 0.021219f, 0.022174f, 0.023153f, 0.024158f, + 0.025187f, 0.026241f, 0.027321f, 0.028426f, 0.029557f, 0.030713f, 0.031896f, 0.033105f, 0.034340f, 0.035601f, 0.036889f, + 0.038204f, 0.039546f, 0.040915f, 0.042311f, 0.043735f, 0.045186f, 0.046665f, 0.048172f, 0.049707f, 0.051269f, 0.052861f, + 0.054480f, 0.056128f, 0.057805f, 0.059511f, 0.061246f, 0.063010f, 0.064803f, 0.066626f, 0.068478f, 0.070360f, 0.072272f, + 0.074214f, 0.076185f, 0.078187f, 0.080220f, 0.082283f, 0.084376f, 0.086500f, 0.088656f, 0.090842f, 0.093059f, 0.095307f, + 0.097587f, 0.099899f, 0.102242f, 0.104616f, 0.107023f, 0.109462f, 0.111932f, 0.114435f, 0.116971f, 0.119538f, 0.122139f, + 0.124772f, 0.127438f, 0.130136f, 0.132868f, 0.135633f, 0.138432f, 0.141263f, 0.144128f, 0.147027f, 0.149960f, 0.152926f, + 0.155926f, 0.158961f, 0.162029f, 0.165132f, 0.168269f, 0.171441f, 0.174647f, 0.177888f, 0.181164f, 0.184475f, 0.187821f, + 0.191202f, 0.194618f, 0.198069f, 0.201556f, 0.205079f, 0.208637f, 0.212231f, 0.215861f, 0.219526f, 0.223228f, 0.226966f, + 0.230740f, 0.234551f, 0.238398f, 0.242281f, 0.246201f, 0.250158f, 0.254152f, 0.258183f, 0.262251f, 0.266356f, 0.270498f, + 0.274677f, 0.278894f, 0.283149f, 0.287441f, 0.291771f, 0.296138f, 0.300544f, 0.304987f, 0.309469f, 0.313989f, 0.318547f, + 0.323143f, 0.327778f, 0.332452f, 0.337164f, 0.341914f, 0.346704f, 0.351533f, 0.356400f, 0.361307f, 0.366253f, 0.371238f, + 0.376262f, 0.381326f, 0.386430f, 0.391573f, 0.396755f, 0.401978f, 0.407240f, 0.412543f, 0.417885f, 0.423268f, 0.428691f, + 0.434154f, 0.439657f, 0.445201f, 0.450786f, 0.456411f, 0.462077f, 0.467784f, 0.473532f, 0.479320f, 0.485150f, 0.491021f, + 0.496933f, 0.502887f, 0.508881f, 0.514918f, 0.520996f, 0.527115f, 0.533276f, 0.539480f, 0.545725f, 0.552011f, 0.558340f, + 0.564712f, 0.571125f, 0.577581f, 0.584078f, 0.590619f, 0.597202f, 0.603827f, 0.610496f, 0.617207f, 0.623960f, 0.630757f, + 0.637597f, 0.644480f, 0.651406f, 0.658375f, 0.665387f, 0.672443f, 0.679543f, 0.686685f, 0.693872f, 0.701102f, 0.708376f, + 0.715694f, 0.723055f, 0.730461f, 0.737911f, 0.745404f, 0.752942f, 0.760525f, 0.768151f, 0.775822f, 0.783538f, 0.791298f, + 0.799103f, 0.806952f, 0.814847f, 0.822786f, 0.830770f, 0.838799f, 0.846873f, 0.854993f, 0.863157f, 0.871367f, 0.879622f, + 0.887923f, 0.896269f, 0.904661f, 0.913099f, 0.921582f, 0.930111f, 0.938686f, 0.947307f, 0.955974f, 0.964686f, 0.973445f, + 0.982251f, 0.991102f, 1.0f +}; + +typedef union +{ + unsigned int u; + float f; +} stbir__FP32; + +// From https://gist.github.com/rygorous/2203834 + +static const stbir_uint32 fp32_to_srgb8_tab4[104] = { + 0x0073000d, 0x007a000d, 0x0080000d, 0x0087000d, 0x008d000d, 0x0094000d, 0x009a000d, 0x00a1000d, + 0x00a7001a, 0x00b4001a, 0x00c1001a, 0x00ce001a, 0x00da001a, 0x00e7001a, 0x00f4001a, 0x0101001a, + 0x010e0033, 0x01280033, 0x01410033, 0x015b0033, 0x01750033, 0x018f0033, 0x01a80033, 0x01c20033, + 0x01dc0067, 0x020f0067, 0x02430067, 0x02760067, 0x02aa0067, 0x02dd0067, 0x03110067, 0x03440067, + 0x037800ce, 0x03df00ce, 0x044600ce, 0x04ad00ce, 0x051400ce, 0x057b00c5, 0x05dd00bc, 0x063b00b5, + 0x06970158, 0x07420142, 0x07e30130, 0x087b0120, 0x090b0112, 0x09940106, 0x0a1700fc, 0x0a9500f2, + 0x0b0f01cb, 0x0bf401ae, 0x0ccb0195, 0x0d950180, 0x0e56016e, 0x0f0d015e, 0x0fbc0150, 0x10630143, + 0x11070264, 0x1238023e, 0x1357021d, 0x14660201, 0x156601e9, 0x165a01d3, 0x174401c0, 0x182401af, + 0x18fe0331, 0x1a9602fe, 0x1c1502d2, 0x1d7e02ad, 0x1ed4028d, 0x201a0270, 0x21520256, 0x227d0240, + 0x239f0443, 0x25c003fe, 0x27bf03c4, 0x29a10392, 0x2b6a0367, 0x2d1d0341, 0x2ebe031f, 0x304d0300, + 0x31d105b0, 0x34a80555, 0x37520507, 0x39d504c5, 0x3c37048b, 0x3e7c0458, 0x40a8042a, 0x42bd0401, + 0x44c20798, 0x488e071e, 0x4c1c06b6, 0x4f76065d, 0x52a50610, 0x55ac05cc, 0x5892058f, 0x5b590559, + 0x5e0c0a23, 0x631c0980, 0x67db08f6, 0x6c55087f, 0x70940818, 0x74a007bd, 0x787d076c, 0x7c330723, +}; + +static stbir__inline stbir_uint8 stbir__linear_to_srgb_uchar(float in) +{ + static const stbir__FP32 almostone = { 0x3f7fffff }; // 1-eps + static const stbir__FP32 minval = { (127-13) << 23 }; + stbir_uint32 tab,bias,scale,t; + stbir__FP32 f; + + // Clamp to [2^(-13), 1-eps]; these two values map to 0 and 1, respectively. + // The tests are carefully written so that NaNs map to 0, same as in the reference + // implementation. + if (!(in > minval.f)) // written this way to catch NaNs + return 0; + if (in > almostone.f) + return 255; + + // Do the table lookup and unpack bias, scale + f.f = in; + tab = fp32_to_srgb8_tab4[(f.u - minval.u) >> 20]; + bias = (tab >> 16) << 9; + scale = tab & 0xffff; + + // Grab next-highest mantissa bits and perform linear interpolation + t = (f.u >> 12) & 0xff; + return (unsigned char) ((bias + scale*t) >> 16); +} + +#ifndef STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT +#define STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT 32 // when downsampling and <= 32 scanlines of buffering, use gather. gather used down to 1/8th scaling for 25% win. +#endif + +#ifndef STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS +#define STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS 4 // when threading, what is the minimum number of scanlines for a split? +#endif + +// restrict pointers for the output pointers, other loop and unroll control +#if defined( _MSC_VER ) && !defined(__clang__) + #define STBIR_STREAMOUT_PTR( star ) star __restrict + #define STBIR_NO_UNROLL( ptr ) __assume(ptr) // this oddly keeps msvc from unrolling a loop + #if _MSC_VER >= 1900 + #define STBIR_NO_UNROLL_LOOP_START __pragma(loop( no_vector )) + #else + #define STBIR_NO_UNROLL_LOOP_START + #endif +#elif defined( __clang__ ) + #define STBIR_STREAMOUT_PTR( star ) star __restrict__ + #define STBIR_NO_UNROLL( ptr ) __asm__ (""::"r"(ptr)) + #if ( __clang_major__ >= 4 ) || ( ( __clang_major__ >= 3 ) && ( __clang_minor__ >= 5 ) ) + #define STBIR_NO_UNROLL_LOOP_START _Pragma("clang loop unroll(disable)") _Pragma("clang loop vectorize(disable)") + #else + #define STBIR_NO_UNROLL_LOOP_START + #endif +#elif defined( __GNUC__ ) + #define STBIR_STREAMOUT_PTR( star ) star __restrict__ + #define STBIR_NO_UNROLL( ptr ) __asm__ (""::"r"(ptr)) + #if __GNUC__ >= 14 + #define STBIR_NO_UNROLL_LOOP_START _Pragma("GCC unroll 0") _Pragma("GCC novector") + #else + #define STBIR_NO_UNROLL_LOOP_START + #endif + #define STBIR_NO_UNROLL_LOOP_START_INF_FOR +#else + #define STBIR_STREAMOUT_PTR( star ) star + #define STBIR_NO_UNROLL( ptr ) + #define STBIR_NO_UNROLL_LOOP_START +#endif + +#ifndef STBIR_NO_UNROLL_LOOP_START_INF_FOR +#define STBIR_NO_UNROLL_LOOP_START_INF_FOR STBIR_NO_UNROLL_LOOP_START +#endif + +#ifdef STBIR_NO_SIMD // force simd off for whatever reason + +// force simd off overrides everything else, so clear it all + +#ifdef STBIR_SSE2 +#undef STBIR_SSE2 +#endif + +#ifdef STBIR_AVX +#undef STBIR_AVX +#endif + +#ifdef STBIR_NEON +#undef STBIR_NEON +#endif + +#ifdef STBIR_AVX2 +#undef STBIR_AVX2 +#endif + +#ifdef STBIR_FP16C +#undef STBIR_FP16C +#endif + +#ifdef STBIR_WASM +#undef STBIR_WASM +#endif + +#ifdef STBIR_SIMD +#undef STBIR_SIMD +#endif + +#else // STBIR_SIMD + +#ifdef STBIR_SSE2 + #include + + #define stbir__simdf __m128 + #define stbir__simdi __m128i + + #define stbir_simdi_castf( reg ) _mm_castps_si128(reg) + #define stbir_simdf_casti( reg ) _mm_castsi128_ps(reg) + + #define stbir__simdf_load( reg, ptr ) (reg) = _mm_loadu_ps( (float const*)(ptr) ) + #define stbir__simdi_load( reg, ptr ) (reg) = _mm_loadu_si128 ( (stbir__simdi const*)(ptr) ) + #define stbir__simdf_load1( out, ptr ) (out) = _mm_load_ss( (float const*)(ptr) ) // top values can be random (not denormal or nan for perf) + #define stbir__simdi_load1( out, ptr ) (out) = _mm_castps_si128( _mm_load_ss( (float const*)(ptr) )) + #define stbir__simdf_load1z( out, ptr ) (out) = _mm_load_ss( (float const*)(ptr) ) // top values must be zero + #define stbir__simdf_frep4( fvar ) _mm_set_ps1( fvar ) + #define stbir__simdf_load1frep4( out, fvar ) (out) = _mm_set_ps1( fvar ) + #define stbir__simdf_load2( out, ptr ) (out) = _mm_castsi128_ps( _mm_loadl_epi64( (__m128i*)(ptr)) ) // top values can be random (not denormal or nan for perf) + #define stbir__simdf_load2z( out, ptr ) (out) = _mm_castsi128_ps( _mm_loadl_epi64( (__m128i*)(ptr)) ) // top values must be zero + #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = _mm_castpd_ps(_mm_loadh_pd( _mm_castps_pd(reg), (double*)(ptr) )) + + #define stbir__simdf_zeroP() _mm_setzero_ps() + #define stbir__simdf_zero( reg ) (reg) = _mm_setzero_ps() + + #define stbir__simdf_store( ptr, reg ) _mm_storeu_ps( (float*)(ptr), reg ) + #define stbir__simdf_store1( ptr, reg ) _mm_store_ss( (float*)(ptr), reg ) + #define stbir__simdf_store2( ptr, reg ) _mm_storel_epi64( (__m128i*)(ptr), _mm_castps_si128(reg) ) + #define stbir__simdf_store2h( ptr, reg ) _mm_storeh_pd( (double*)(ptr), _mm_castps_pd(reg) ) + + #define stbir__simdi_store( ptr, reg ) _mm_storeu_si128( (__m128i*)(ptr), reg ) + #define stbir__simdi_store1( ptr, reg ) _mm_store_ss( (float*)(ptr), _mm_castsi128_ps(reg) ) + #define stbir__simdi_store2( ptr, reg ) _mm_storel_epi64( (__m128i*)(ptr), (reg) ) + + #define stbir__prefetch( ptr ) _mm_prefetch((char*)(ptr), _MM_HINT_T0 ) + + #define stbir__simdi_expand_u8_to_u32(out0,out1,out2,out3,ireg) \ + { \ + stbir__simdi zero = _mm_setzero_si128(); \ + out2 = _mm_unpacklo_epi8( ireg, zero ); \ + out3 = _mm_unpackhi_epi8( ireg, zero ); \ + out0 = _mm_unpacklo_epi16( out2, zero ); \ + out1 = _mm_unpackhi_epi16( out2, zero ); \ + out2 = _mm_unpacklo_epi16( out3, zero ); \ + out3 = _mm_unpackhi_epi16( out3, zero ); \ + } + +#define stbir__simdi_expand_u8_to_1u32(out,ireg) \ + { \ + stbir__simdi zero = _mm_setzero_si128(); \ + out = _mm_unpacklo_epi8( ireg, zero ); \ + out = _mm_unpacklo_epi16( out, zero ); \ + } + + #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \ + { \ + stbir__simdi zero = _mm_setzero_si128(); \ + out0 = _mm_unpacklo_epi16( ireg, zero ); \ + out1 = _mm_unpackhi_epi16( ireg, zero ); \ + } + + #define stbir__simdf_convert_float_to_i32( i, f ) (i) = _mm_cvttps_epi32(f) + #define stbir__simdf_convert_float_to_int( f ) _mm_cvtt_ss2si(f) + #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)_mm_cvtsi128_si32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(f,STBIR__CONSTF(STBIR_max_uint8_as_float)),_mm_setzero_ps())))) + #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)_mm_cvtsi128_si32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(f,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())))) + + #define stbir__simdi_to_int( i ) _mm_cvtsi128_si32(i) + #define stbir__simdi_convert_i32_to_float(out, ireg) (out) = _mm_cvtepi32_ps( ireg ) + #define stbir__simdf_add( out, reg0, reg1 ) (out) = _mm_add_ps( reg0, reg1 ) + #define stbir__simdf_mult( out, reg0, reg1 ) (out) = _mm_mul_ps( reg0, reg1 ) + #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = _mm_mul_ps( reg, _mm_loadu_ps( (float const*)(ptr) ) ) + #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = _mm_mul_ss( reg, _mm_load_ss( (float const*)(ptr) ) ) + #define stbir__simdf_add_mem( out, reg, ptr ) (out) = _mm_add_ps( reg, _mm_loadu_ps( (float const*)(ptr) ) ) + #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = _mm_add_ss( reg, _mm_load_ss( (float const*)(ptr) ) ) + + #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd + #include + #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = _mm_fmadd_ps( mul1, mul2, add ) + #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = _mm_fmadd_ss( mul1, mul2, add ) + #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = _mm_fmadd_ps( mul, _mm_loadu_ps( (float const*)(ptr) ), add ) + #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = _mm_fmadd_ss( mul, _mm_load_ss( (float const*)(ptr) ), add ) + #else + #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = _mm_add_ps( add, _mm_mul_ps( mul1, mul2 ) ) + #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = _mm_add_ss( add, _mm_mul_ss( mul1, mul2 ) ) + #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = _mm_add_ps( add, _mm_mul_ps( mul, _mm_loadu_ps( (float const*)(ptr) ) ) ) + #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = _mm_add_ss( add, _mm_mul_ss( mul, _mm_load_ss( (float const*)(ptr) ) ) ) + #endif + + #define stbir__simdf_add1( out, reg0, reg1 ) (out) = _mm_add_ss( reg0, reg1 ) + #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = _mm_mul_ss( reg0, reg1 ) + + #define stbir__simdf_and( out, reg0, reg1 ) (out) = _mm_and_ps( reg0, reg1 ) + #define stbir__simdf_or( out, reg0, reg1 ) (out) = _mm_or_ps( reg0, reg1 ) + + #define stbir__simdf_min( out, reg0, reg1 ) (out) = _mm_min_ps( reg0, reg1 ) + #define stbir__simdf_max( out, reg0, reg1 ) (out) = _mm_max_ps( reg0, reg1 ) + #define stbir__simdf_min1( out, reg0, reg1 ) (out) = _mm_min_ss( reg0, reg1 ) + #define stbir__simdf_max1( out, reg0, reg1 ) (out) = _mm_max_ss( reg0, reg1 ) + + #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_shuffle_ps( reg1,reg0, (0<<0) + (1<<2) + (2<<4) + (3<<6) )), (3<<0) + (0<<2) + (1<<4) + (2<<6) ) ) + #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_shuffle_ps( reg1,reg0, (0<<0) + (1<<2) + (2<<4) + (3<<6) )), (2<<0) + (3<<2) + (0<<4) + (1<<6) ) ) + + static const stbir__simdf STBIR_zeroones = { 0.0f,1.0f,0.0f,1.0f }; + static const stbir__simdf STBIR_onezeros = { 1.0f,0.0f,1.0f,0.0f }; + #define stbir__simdf_aaa1( out, alp, ones ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_movehl_ps( ones, alp ) ), (1<<0) + (1<<2) + (1<<4) + (2<<6) ) ) + #define stbir__simdf_1aaa( out, alp, ones ) (out)=_mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( _mm_movelh_ps( ones, alp ) ), (0<<0) + (2<<2) + (2<<4) + (2<<6) ) ) + #define stbir__simdf_a1a1( out, alp, ones) (out) = _mm_or_ps( _mm_castsi128_ps( _mm_srli_epi64( _mm_castps_si128(alp), 32 ) ), STBIR_zeroones ) + #define stbir__simdf_1a1a( out, alp, ones) (out) = _mm_or_ps( _mm_castsi128_ps( _mm_slli_epi64( _mm_castps_si128(alp), 32 ) ), STBIR_onezeros ) + + #define stbir__simdf_swiz( reg, one, two, three, four ) _mm_castsi128_ps( _mm_shuffle_epi32( _mm_castps_si128( reg ), (one<<0) + (two<<2) + (three<<4) + (four<<6) ) ) + + #define stbir__simdi_and( out, reg0, reg1 ) (out) = _mm_and_si128( reg0, reg1 ) + #define stbir__simdi_or( out, reg0, reg1 ) (out) = _mm_or_si128( reg0, reg1 ) + #define stbir__simdi_16madd( out, reg0, reg1 ) (out) = _mm_madd_epi16( reg0, reg1 ) + + #define stbir__simdf_pack_to_8bytes(out,aa,bb) \ + { \ + stbir__simdf af,bf; \ + stbir__simdi a,b; \ + af = _mm_min_ps( aa, STBIR_max_uint8_as_float ); \ + bf = _mm_min_ps( bb, STBIR_max_uint8_as_float ); \ + af = _mm_max_ps( af, _mm_setzero_ps() ); \ + bf = _mm_max_ps( bf, _mm_setzero_ps() ); \ + a = _mm_cvttps_epi32( af ); \ + b = _mm_cvttps_epi32( bf ); \ + a = _mm_packs_epi32( a, b ); \ + out = _mm_packus_epi16( a, a ); \ + } + + #define stbir__simdf_load4_transposed( o0, o1, o2, o3, ptr ) \ + stbir__simdf_load( o0, (ptr) ); \ + stbir__simdf_load( o1, (ptr)+4 ); \ + stbir__simdf_load( o2, (ptr)+8 ); \ + stbir__simdf_load( o3, (ptr)+12 ); \ + { \ + __m128 tmp0, tmp1, tmp2, tmp3; \ + tmp0 = _mm_unpacklo_ps(o0, o1); \ + tmp2 = _mm_unpacklo_ps(o2, o3); \ + tmp1 = _mm_unpackhi_ps(o0, o1); \ + tmp3 = _mm_unpackhi_ps(o2, o3); \ + o0 = _mm_movelh_ps(tmp0, tmp2); \ + o1 = _mm_movehl_ps(tmp2, tmp0); \ + o2 = _mm_movelh_ps(tmp1, tmp3); \ + o3 = _mm_movehl_ps(tmp3, tmp1); \ + } + + #define stbir__interleave_pack_and_store_16_u8( ptr, r0, r1, r2, r3 ) \ + r0 = _mm_packs_epi32( r0, r1 ); \ + r2 = _mm_packs_epi32( r2, r3 ); \ + r1 = _mm_unpacklo_epi16( r0, r2 ); \ + r3 = _mm_unpackhi_epi16( r0, r2 ); \ + r0 = _mm_unpacklo_epi16( r1, r3 ); \ + r2 = _mm_unpackhi_epi16( r1, r3 ); \ + r0 = _mm_packus_epi16( r0, r2 ); \ + stbir__simdi_store( ptr, r0 ); \ + + #define stbir__simdi_32shr( out, reg, imm ) out = _mm_srli_epi32( reg, imm ) + + #if defined(_MSC_VER) && !defined(__clang__) + // msvc inits with 8 bytes + #define STBIR__CONST_32_TO_8( v ) (char)(unsigned char)((v)&255),(char)(unsigned char)(((v)>>8)&255),(char)(unsigned char)(((v)>>16)&255),(char)(unsigned char)(((v)>>24)&255) + #define STBIR__CONST_4_32i( v ) STBIR__CONST_32_TO_8( v ), STBIR__CONST_32_TO_8( v ), STBIR__CONST_32_TO_8( v ), STBIR__CONST_32_TO_8( v ) + #define STBIR__CONST_4d_32i( v0, v1, v2, v3 ) STBIR__CONST_32_TO_8( v0 ), STBIR__CONST_32_TO_8( v1 ), STBIR__CONST_32_TO_8( v2 ), STBIR__CONST_32_TO_8( v3 ) + #else + // everything else inits with long long's + #define STBIR__CONST_4_32i( v ) (long long)((((stbir_uint64)(stbir_uint32)(v))<<32)|((stbir_uint64)(stbir_uint32)(v))),(long long)((((stbir_uint64)(stbir_uint32)(v))<<32)|((stbir_uint64)(stbir_uint32)(v))) + #define STBIR__CONST_4d_32i( v0, v1, v2, v3 ) (long long)((((stbir_uint64)(stbir_uint32)(v1))<<32)|((stbir_uint64)(stbir_uint32)(v0))),(long long)((((stbir_uint64)(stbir_uint32)(v3))<<32)|((stbir_uint64)(stbir_uint32)(v2))) + #endif + + #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = { x, x, x, x } + #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { STBIR__CONST_4_32i(x) } + #define STBIR__CONSTF(var) (var) + #define STBIR__CONSTI(var) (var) + + #if defined(STBIR_AVX) || defined(__SSE4_1__) + #include + #define stbir__simdf_pack_to_8words(out,reg0,reg1) out = _mm_packus_epi32(_mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg0,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())), _mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg1,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps()))) + #else + STBIR__SIMDI_CONST(stbir__s32_32768, 32768); + STBIR__SIMDI_CONST(stbir__s16_32768, ((32768<<16)|32768)); + + #define stbir__simdf_pack_to_8words(out,reg0,reg1) \ + { \ + stbir__simdi tmp0,tmp1; \ + tmp0 = _mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg0,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())); \ + tmp1 = _mm_cvttps_epi32(_mm_max_ps(_mm_min_ps(reg1,STBIR__CONSTF(STBIR_max_uint16_as_float)),_mm_setzero_ps())); \ + tmp0 = _mm_sub_epi32( tmp0, stbir__s32_32768 ); \ + tmp1 = _mm_sub_epi32( tmp1, stbir__s32_32768 ); \ + out = _mm_packs_epi32( tmp0, tmp1 ); \ + out = _mm_sub_epi16( out, stbir__s16_32768 ); \ + } + + #endif + + #define STBIR_SIMD + + // if we detect AVX, set the simd8 defines + #ifdef STBIR_AVX + #include + #define STBIR_SIMD8 + #define stbir__simdf8 __m256 + #define stbir__simdi8 __m256i + #define stbir__simdf8_load( out, ptr ) (out) = _mm256_loadu_ps( (float const *)(ptr) ) + #define stbir__simdi8_load( out, ptr ) (out) = _mm256_loadu_si256( (__m256i const *)(ptr) ) + #define stbir__simdf8_mult( out, a, b ) (out) = _mm256_mul_ps( (a), (b) ) + #define stbir__simdf8_store( ptr, out ) _mm256_storeu_ps( (float*)(ptr), out ) + #define stbir__simdi8_store( ptr, reg ) _mm256_storeu_si256( (__m256i*)(ptr), reg ) + #define stbir__simdf8_frep8( fval ) _mm256_set1_ps( fval ) + + #define stbir__simdf8_min( out, reg0, reg1 ) (out) = _mm256_min_ps( reg0, reg1 ) + #define stbir__simdf8_max( out, reg0, reg1 ) (out) = _mm256_max_ps( reg0, reg1 ) + + #define stbir__simdf8_add4halves( out, bot4, top8 ) (out) = _mm_add_ps( bot4, _mm256_extractf128_ps( top8, 1 ) ) + #define stbir__simdf8_mult_mem( out, reg, ptr ) (out) = _mm256_mul_ps( reg, _mm256_loadu_ps( (float const*)(ptr) ) ) + #define stbir__simdf8_add_mem( out, reg, ptr ) (out) = _mm256_add_ps( reg, _mm256_loadu_ps( (float const*)(ptr) ) ) + #define stbir__simdf8_add( out, a, b ) (out) = _mm256_add_ps( a, b ) + #define stbir__simdf8_load1b( out, ptr ) (out) = _mm256_broadcast_ss( ptr ) + #define stbir__simdf_load1rep4( out, ptr ) (out) = _mm_broadcast_ss( ptr ) // avx load instruction + + #define stbir__simdi8_convert_i32_to_float(out, ireg) (out) = _mm256_cvtepi32_ps( ireg ) + #define stbir__simdf8_convert_float_to_i32( i, f ) (i) = _mm256_cvttps_epi32(f) + + #define stbir__simdf8_bot4s( out, a, b ) (out) = _mm256_permute2f128_ps(a,b, (0<<0)+(2<<4) ) + #define stbir__simdf8_top4s( out, a, b ) (out) = _mm256_permute2f128_ps(a,b, (1<<0)+(3<<4) ) + + #define stbir__simdf8_gettop4( reg ) _mm256_extractf128_ps(reg,1) + + #ifdef STBIR_AVX2 + + #define stbir__simdi8_expand_u8_to_u32(out0,out1,ireg) \ + { \ + stbir__simdi8 a, zero =_mm256_setzero_si256();\ + a = _mm256_permute4x64_epi64( _mm256_unpacklo_epi8( _mm256_permute4x64_epi64(_mm256_castsi128_si256(ireg),(0<<0)+(2<<2)+(1<<4)+(3<<6)), zero ),(0<<0)+(2<<2)+(1<<4)+(3<<6)); \ + out0 = _mm256_unpacklo_epi16( a, zero ); \ + out1 = _mm256_unpackhi_epi16( a, zero ); \ + } + + #define stbir__simdf8_pack_to_16bytes(out,aa,bb) \ + { \ + stbir__simdi8 t; \ + stbir__simdf8 af,bf; \ + stbir__simdi8 a,b; \ + af = _mm256_min_ps( aa, STBIR_max_uint8_as_floatX ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint8_as_floatX ); \ + af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ + bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ + a = _mm256_cvttps_epi32( af ); \ + b = _mm256_cvttps_epi32( bf ); \ + t = _mm256_permute4x64_epi64( _mm256_packs_epi32( a, b ), (0<<0)+(2<<2)+(1<<4)+(3<<6) ); \ + out = _mm256_castsi256_si128( _mm256_permute4x64_epi64( _mm256_packus_epi16( t, t ), (0<<0)+(2<<2)+(1<<4)+(3<<6) ) ); \ + } + + #define stbir__simdi8_expand_u16_to_u32(out,ireg) out = _mm256_unpacklo_epi16( _mm256_permute4x64_epi64(_mm256_castsi128_si256(ireg),(0<<0)+(2<<2)+(1<<4)+(3<<6)), _mm256_setzero_si256() ); + + #define stbir__simdf8_pack_to_16words(out,aa,bb) \ + { \ + stbir__simdf8 af,bf; \ + stbir__simdi8 a,b; \ + af = _mm256_min_ps( aa, STBIR_max_uint16_as_floatX ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint16_as_floatX ); \ + af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ + bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ + a = _mm256_cvttps_epi32( af ); \ + b = _mm256_cvttps_epi32( bf ); \ + (out) = _mm256_permute4x64_epi64( _mm256_packus_epi32(a, b), (0<<0)+(2<<2)+(1<<4)+(3<<6) ); \ + } + + #else + + #define stbir__simdi8_expand_u8_to_u32(out0,out1,ireg) \ + { \ + stbir__simdi a,zero = _mm_setzero_si128(); \ + a = _mm_unpacklo_epi8( ireg, zero ); \ + out0 = _mm256_setr_m128i( _mm_unpacklo_epi16( a, zero ), _mm_unpackhi_epi16( a, zero ) ); \ + a = _mm_unpackhi_epi8( ireg, zero ); \ + out1 = _mm256_setr_m128i( _mm_unpacklo_epi16( a, zero ), _mm_unpackhi_epi16( a, zero ) ); \ + } + + #define stbir__simdf8_pack_to_16bytes(out,aa,bb) \ + { \ + stbir__simdi t; \ + stbir__simdf8 af,bf; \ + stbir__simdi8 a,b; \ + af = _mm256_min_ps( aa, STBIR_max_uint8_as_floatX ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint8_as_floatX ); \ + af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ + bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ + a = _mm256_cvttps_epi32( af ); \ + b = _mm256_cvttps_epi32( bf ); \ + out = _mm_packs_epi32( _mm256_castsi256_si128(a), _mm256_extractf128_si256( a, 1 ) ); \ + out = _mm_packus_epi16( out, out ); \ + t = _mm_packs_epi32( _mm256_castsi256_si128(b), _mm256_extractf128_si256( b, 1 ) ); \ + t = _mm_packus_epi16( t, t ); \ + out = _mm_castps_si128( _mm_shuffle_ps( _mm_castsi128_ps(out), _mm_castsi128_ps(t), (0<<0)+(1<<2)+(0<<4)+(1<<6) ) ); \ + } + + #define stbir__simdi8_expand_u16_to_u32(out,ireg) \ + { \ + stbir__simdi a,b,zero = _mm_setzero_si128(); \ + a = _mm_unpacklo_epi16( ireg, zero ); \ + b = _mm_unpackhi_epi16( ireg, zero ); \ + out = _mm256_insertf128_si256( _mm256_castsi128_si256( a ), b, 1 ); \ + } + + #define stbir__simdf8_pack_to_16words(out,aa,bb) \ + { \ + stbir__simdi t0,t1; \ + stbir__simdf8 af,bf; \ + stbir__simdi8 a,b; \ + af = _mm256_min_ps( aa, STBIR_max_uint16_as_floatX ); \ + bf = _mm256_min_ps( bb, STBIR_max_uint16_as_floatX ); \ + af = _mm256_max_ps( af, _mm256_setzero_ps() ); \ + bf = _mm256_max_ps( bf, _mm256_setzero_ps() ); \ + a = _mm256_cvttps_epi32( af ); \ + b = _mm256_cvttps_epi32( bf ); \ + t0 = _mm_packus_epi32( _mm256_castsi256_si128(a), _mm256_extractf128_si256( a, 1 ) ); \ + t1 = _mm_packus_epi32( _mm256_castsi256_si128(b), _mm256_extractf128_si256( b, 1 ) ); \ + out = _mm256_setr_m128i( t0, t1 ); \ + } + + #endif + + static __m256i stbir_00001111 = { STBIR__CONST_4d_32i( 0, 0, 0, 0 ), STBIR__CONST_4d_32i( 1, 1, 1, 1 ) }; + #define stbir__simdf8_0123to00001111( out, in ) (out) = _mm256_permutevar_ps ( in, stbir_00001111 ) + + static __m256i stbir_22223333 = { STBIR__CONST_4d_32i( 2, 2, 2, 2 ), STBIR__CONST_4d_32i( 3, 3, 3, 3 ) }; + #define stbir__simdf8_0123to22223333( out, in ) (out) = _mm256_permutevar_ps ( in, stbir_22223333 ) + + #define stbir__simdf8_0123to2222( out, in ) (out) = stbir__simdf_swiz(_mm256_castps256_ps128(in), 2,2,2,2 ) + + #define stbir__simdf8_load4b( out, ptr ) (out) = _mm256_broadcast_ps( (__m128 const *)(ptr) ) + + static __m256i stbir_00112233 = { STBIR__CONST_4d_32i( 0, 0, 1, 1 ), STBIR__CONST_4d_32i( 2, 2, 3, 3 ) }; + #define stbir__simdf8_0123to00112233( out, in ) (out) = _mm256_permutevar_ps ( in, stbir_00112233 ) + #define stbir__simdf8_add4( out, a8, b ) (out) = _mm256_add_ps( a8, _mm256_castps128_ps256( b ) ) + + static __m256i stbir_load6 = { STBIR__CONST_4_32i( 0x80000000 ), STBIR__CONST_4d_32i( 0x80000000, 0x80000000, 0, 0 ) }; + #define stbir__simdf8_load6z( out, ptr ) (out) = _mm256_maskload_ps( ptr, stbir_load6 ) + + #define stbir__simdf8_0123to00000000( out, in ) (out) = _mm256_shuffle_ps ( in, in, (0<<0)+(0<<2)+(0<<4)+(0<<6) ) + #define stbir__simdf8_0123to11111111( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(1<<2)+(1<<4)+(1<<6) ) + #define stbir__simdf8_0123to22222222( out, in ) (out) = _mm256_shuffle_ps ( in, in, (2<<0)+(2<<2)+(2<<4)+(2<<6) ) + #define stbir__simdf8_0123to33333333( out, in ) (out) = _mm256_shuffle_ps ( in, in, (3<<0)+(3<<2)+(3<<4)+(3<<6) ) + #define stbir__simdf8_0123to21032103( out, in ) (out) = _mm256_shuffle_ps ( in, in, (2<<0)+(1<<2)+(0<<4)+(3<<6) ) + #define stbir__simdf8_0123to32103210( out, in ) (out) = _mm256_shuffle_ps ( in, in, (3<<0)+(2<<2)+(1<<4)+(0<<6) ) + #define stbir__simdf8_0123to12301230( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(2<<2)+(3<<4)+(0<<6) ) + #define stbir__simdf8_0123to10321032( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(0<<2)+(3<<4)+(2<<6) ) + #define stbir__simdf8_0123to30123012( out, in ) (out) = _mm256_shuffle_ps ( in, in, (3<<0)+(0<<2)+(1<<4)+(2<<6) ) + + #define stbir__simdf8_0123to11331133( out, in ) (out) = _mm256_shuffle_ps ( in, in, (1<<0)+(1<<2)+(3<<4)+(3<<6) ) + #define stbir__simdf8_0123to00220022( out, in ) (out) = _mm256_shuffle_ps ( in, in, (0<<0)+(0<<2)+(2<<4)+(2<<6) ) + + #define stbir__simdf8_aaa1( out, alp, ones ) (out) = _mm256_blend_ps( alp, ones, (1<<0)+(1<<1)+(1<<2)+(0<<3)+(1<<4)+(1<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (3<<0) + (3<<2) + (3<<4) + (0<<6) ) + #define stbir__simdf8_1aaa( out, alp, ones ) (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(1<<2)+(1<<3)+(0<<4)+(1<<5)+(1<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (0<<4) + (0<<6) ) + #define stbir__simdf8_a1a1( out, alp, ones) (out) = _mm256_blend_ps( alp, ones, (1<<0)+(0<<1)+(1<<2)+(0<<3)+(1<<4)+(0<<5)+(1<<6)+(0<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) ) + #define stbir__simdf8_1a1a( out, alp, ones) (out) = _mm256_blend_ps( alp, ones, (0<<0)+(1<<1)+(0<<2)+(1<<3)+(0<<4)+(1<<5)+(0<<6)+(1<<7)); (out)=_mm256_shuffle_ps( out,out, (1<<0) + (0<<2) + (3<<4) + (2<<6) ) + + #define stbir__simdf8_zero( reg ) (reg) = _mm256_setzero_ps() + + #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd + #define stbir__simdf8_madd( out, add, mul1, mul2 ) (out) = _mm256_fmadd_ps( mul1, mul2, add ) + #define stbir__simdf8_madd_mem( out, add, mul, ptr ) (out) = _mm256_fmadd_ps( mul, _mm256_loadu_ps( (float const*)(ptr) ), add ) + #define stbir__simdf8_madd_mem4( out, add, mul, ptr )(out) = _mm256_fmadd_ps( _mm256_setr_m128( mul, _mm_setzero_ps() ), _mm256_setr_m128( _mm_loadu_ps( (float const*)(ptr) ), _mm_setzero_ps() ), add ) + #else + #define stbir__simdf8_madd( out, add, mul1, mul2 ) (out) = _mm256_add_ps( add, _mm256_mul_ps( mul1, mul2 ) ) + #define stbir__simdf8_madd_mem( out, add, mul, ptr ) (out) = _mm256_add_ps( add, _mm256_mul_ps( mul, _mm256_loadu_ps( (float const*)(ptr) ) ) ) + #define stbir__simdf8_madd_mem4( out, add, mul, ptr ) (out) = _mm256_add_ps( add, _mm256_setr_m128( _mm_mul_ps( mul, _mm_loadu_ps( (float const*)(ptr) ) ), _mm_setzero_ps() ) ) + #endif + #define stbir__if_simdf8_cast_to_simdf4( val ) _mm256_castps256_ps128( val ) + + #endif + + #ifdef STBIR_FLOORF + #undef STBIR_FLOORF + #endif + #define STBIR_FLOORF stbir_simd_floorf + static stbir__inline float stbir_simd_floorf(float x) // martins floorf + { + #if defined(STBIR_AVX) || defined(__SSE4_1__) || defined(STBIR_SSE41) + __m128 t = _mm_set_ss(x); + return _mm_cvtss_f32( _mm_floor_ss(t, t) ); + #else + __m128 f = _mm_set_ss(x); + __m128 t = _mm_cvtepi32_ps(_mm_cvttps_epi32(f)); + __m128 r = _mm_add_ss(t, _mm_and_ps(_mm_cmplt_ss(f, t), _mm_set_ss(-1.0f))); + return _mm_cvtss_f32(r); + #endif + } + + #ifdef STBIR_CEILF + #undef STBIR_CEILF + #endif + #define STBIR_CEILF stbir_simd_ceilf + static stbir__inline float stbir_simd_ceilf(float x) // martins ceilf + { + #if defined(STBIR_AVX) || defined(__SSE4_1__) || defined(STBIR_SSE41) + __m128 t = _mm_set_ss(x); + return _mm_cvtss_f32( _mm_ceil_ss(t, t) ); + #else + __m128 f = _mm_set_ss(x); + __m128 t = _mm_cvtepi32_ps(_mm_cvttps_epi32(f)); + __m128 r = _mm_add_ss(t, _mm_and_ps(_mm_cmplt_ss(t, f), _mm_set_ss(1.0f))); + return _mm_cvtss_f32(r); + #endif + } + +#elif defined(STBIR_NEON) + + #include + + #define stbir__simdf float32x4_t + #define stbir__simdi uint32x4_t + + #define stbir_simdi_castf( reg ) vreinterpretq_u32_f32(reg) + #define stbir_simdf_casti( reg ) vreinterpretq_f32_u32(reg) + + #define stbir__simdf_load( reg, ptr ) (reg) = vld1q_f32( (float const*)(ptr) ) + #define stbir__simdi_load( reg, ptr ) (reg) = vld1q_u32( (uint32_t const*)(ptr) ) + #define stbir__simdf_load1( out, ptr ) (out) = vld1q_dup_f32( (float const*)(ptr) ) // top values can be random (not denormal or nan for perf) + #define stbir__simdi_load1( out, ptr ) (out) = vld1q_dup_u32( (uint32_t const*)(ptr) ) + #define stbir__simdf_load1z( out, ptr ) (out) = vld1q_lane_f32( (float const*)(ptr), vdupq_n_f32(0), 0 ) // top values must be zero + #define stbir__simdf_frep4( fvar ) vdupq_n_f32( fvar ) + #define stbir__simdf_load1frep4( out, fvar ) (out) = vdupq_n_f32( fvar ) + #define stbir__simdf_load2( out, ptr ) (out) = vcombine_f32( vld1_f32( (float const*)(ptr) ), vcreate_f32(0) ) // top values can be random (not denormal or nan for perf) + #define stbir__simdf_load2z( out, ptr ) (out) = vcombine_f32( vld1_f32( (float const*)(ptr) ), vcreate_f32(0) ) // top values must be zero + #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = vcombine_f32( vget_low_f32(reg), vld1_f32( (float const*)(ptr) ) ) + + #define stbir__simdf_zeroP() vdupq_n_f32(0) + #define stbir__simdf_zero( reg ) (reg) = vdupq_n_f32(0) + + #define stbir__simdf_store( ptr, reg ) vst1q_f32( (float*)(ptr), reg ) + #define stbir__simdf_store1( ptr, reg ) vst1q_lane_f32( (float*)(ptr), reg, 0) + #define stbir__simdf_store2( ptr, reg ) vst1_f32( (float*)(ptr), vget_low_f32(reg) ) + #define stbir__simdf_store2h( ptr, reg ) vst1_f32( (float*)(ptr), vget_high_f32(reg) ) + + #define stbir__simdi_store( ptr, reg ) vst1q_u32( (uint32_t*)(ptr), reg ) + #define stbir__simdi_store1( ptr, reg ) vst1q_lane_u32( (uint32_t*)(ptr), reg, 0 ) + #define stbir__simdi_store2( ptr, reg ) vst1_u32( (uint32_t*)(ptr), vget_low_u32(reg) ) + + #define stbir__prefetch( ptr ) + + #define stbir__simdi_expand_u8_to_u32(out0,out1,out2,out3,ireg) \ + { \ + uint16x8_t l = vmovl_u8( vget_low_u8 ( vreinterpretq_u8_u32(ireg) ) ); \ + uint16x8_t h = vmovl_u8( vget_high_u8( vreinterpretq_u8_u32(ireg) ) ); \ + out0 = vmovl_u16( vget_low_u16 ( l ) ); \ + out1 = vmovl_u16( vget_high_u16( l ) ); \ + out2 = vmovl_u16( vget_low_u16 ( h ) ); \ + out3 = vmovl_u16( vget_high_u16( h ) ); \ + } + + #define stbir__simdi_expand_u8_to_1u32(out,ireg) \ + { \ + uint16x8_t tmp = vmovl_u8( vget_low_u8( vreinterpretq_u8_u32(ireg) ) ); \ + out = vmovl_u16( vget_low_u16( tmp ) ); \ + } + + #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \ + { \ + uint16x8_t tmp = vreinterpretq_u16_u32(ireg); \ + out0 = vmovl_u16( vget_low_u16 ( tmp ) ); \ + out1 = vmovl_u16( vget_high_u16( tmp ) ); \ + } + + #define stbir__simdf_convert_float_to_i32( i, f ) (i) = vreinterpretq_u32_s32( vcvtq_s32_f32(f) ) + #define stbir__simdf_convert_float_to_int( f ) vgetq_lane_s32(vcvtq_s32_f32(f), 0) + #define stbir__simdi_to_int( i ) (int)vgetq_lane_u32(i, 0) + #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)vgetq_lane_s32(vcvtq_s32_f32(vmaxq_f32(vminq_f32(f,STBIR__CONSTF(STBIR_max_uint8_as_float)),vdupq_n_f32(0))), 0)) + #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)vgetq_lane_s32(vcvtq_s32_f32(vmaxq_f32(vminq_f32(f,STBIR__CONSTF(STBIR_max_uint16_as_float)),vdupq_n_f32(0))), 0)) + #define stbir__simdi_convert_i32_to_float(out, ireg) (out) = vcvtq_f32_s32( vreinterpretq_s32_u32(ireg) ) + #define stbir__simdf_add( out, reg0, reg1 ) (out) = vaddq_f32( reg0, reg1 ) + #define stbir__simdf_mult( out, reg0, reg1 ) (out) = vmulq_f32( reg0, reg1 ) + #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = vmulq_f32( reg, vld1q_f32( (float const*)(ptr) ) ) + #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = vmulq_f32( reg, vld1q_dup_f32( (float const*)(ptr) ) ) + #define stbir__simdf_add_mem( out, reg, ptr ) (out) = vaddq_f32( reg, vld1q_f32( (float const*)(ptr) ) ) + #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = vaddq_f32( reg, vld1q_dup_f32( (float const*)(ptr) ) ) + + #ifdef STBIR_USE_FMA // not on by default to maintain bit identical simd to non-simd (and also x64 no madd to arm madd) + #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = vfmaq_f32( add, mul1, mul2 ) + #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = vfmaq_f32( add, mul1, mul2 ) + #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = vfmaq_f32( add, mul, vld1q_f32( (float const*)(ptr) ) ) + #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = vfmaq_f32( add, mul, vld1q_dup_f32( (float const*)(ptr) ) ) + #else + #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = vaddq_f32( add, vmulq_f32( mul1, mul2 ) ) + #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = vaddq_f32( add, vmulq_f32( mul1, mul2 ) ) + #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = vaddq_f32( add, vmulq_f32( mul, vld1q_f32( (float const*)(ptr) ) ) ) + #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = vaddq_f32( add, vmulq_f32( mul, vld1q_dup_f32( (float const*)(ptr) ) ) ) + #endif + + #define stbir__simdf_add1( out, reg0, reg1 ) (out) = vaddq_f32( reg0, reg1 ) + #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = vmulq_f32( reg0, reg1 ) + + #define stbir__simdf_and( out, reg0, reg1 ) (out) = vreinterpretq_f32_u32( vandq_u32( vreinterpretq_u32_f32(reg0), vreinterpretq_u32_f32(reg1) ) ) + #define stbir__simdf_or( out, reg0, reg1 ) (out) = vreinterpretq_f32_u32( vorrq_u32( vreinterpretq_u32_f32(reg0), vreinterpretq_u32_f32(reg1) ) ) + + #define stbir__simdf_min( out, reg0, reg1 ) (out) = vminq_f32( reg0, reg1 ) + #define stbir__simdf_max( out, reg0, reg1 ) (out) = vmaxq_f32( reg0, reg1 ) + #define stbir__simdf_min1( out, reg0, reg1 ) (out) = vminq_f32( reg0, reg1 ) + #define stbir__simdf_max1( out, reg0, reg1 ) (out) = vmaxq_f32( reg0, reg1 ) + + #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out) = vextq_f32( reg0, reg1, 3 ) + #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out) = vextq_f32( reg0, reg1, 2 ) + + #define stbir__simdf_a1a1( out, alp, ones ) (out) = vzipq_f32(vuzpq_f32(alp, alp).val[1], ones).val[0] + #define stbir__simdf_1a1a( out, alp, ones ) (out) = vzipq_f32(ones, vuzpq_f32(alp, alp).val[0]).val[0] + + #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) + + #define stbir__simdf_aaa1( out, alp, ones ) (out) = vcopyq_laneq_f32(vdupq_n_f32(vgetq_lane_f32(alp, 3)), 3, ones, 3) + #define stbir__simdf_1aaa( out, alp, ones ) (out) = vcopyq_laneq_f32(vdupq_n_f32(vgetq_lane_f32(alp, 0)), 0, ones, 0) + + #if defined( _MSC_VER ) && !defined(__clang__) + #define stbir_make16(a,b,c,d) vcombine_u8( \ + vcreate_u8( (4*a+0) | ((4*a+1)<<8) | ((4*a+2)<<16) | ((4*a+3)<<24) | \ + ((stbir_uint64)(4*b+0)<<32) | ((stbir_uint64)(4*b+1)<<40) | ((stbir_uint64)(4*b+2)<<48) | ((stbir_uint64)(4*b+3)<<56)), \ + vcreate_u8( (4*c+0) | ((4*c+1)<<8) | ((4*c+2)<<16) | ((4*c+3)<<24) | \ + ((stbir_uint64)(4*d+0)<<32) | ((stbir_uint64)(4*d+1)<<40) | ((stbir_uint64)(4*d+2)<<48) | ((stbir_uint64)(4*d+3)<<56) ) ) + + static stbir__inline uint8x16x2_t stbir_make16x2(float32x4_t rega,float32x4_t regb) + { + uint8x16x2_t r = { vreinterpretq_u8_f32(rega), vreinterpretq_u8_f32(regb) }; + return r; + } + #else + #define stbir_make16(a,b,c,d) (uint8x16_t){4*a+0,4*a+1,4*a+2,4*a+3,4*b+0,4*b+1,4*b+2,4*b+3,4*c+0,4*c+1,4*c+2,4*c+3,4*d+0,4*d+1,4*d+2,4*d+3} + #define stbir_make16x2(a,b) (uint8x16x2_t){{vreinterpretq_u8_f32(a),vreinterpretq_u8_f32(b)}} + #endif + + #define stbir__simdf_swiz( reg, one, two, three, four ) vreinterpretq_f32_u8( vqtbl1q_u8( vreinterpretq_u8_f32(reg), stbir_make16(one, two, three, four) ) ) + #define stbir__simdf_swiz2( rega, regb, one, two, three, four ) vreinterpretq_f32_u8( vqtbl2q_u8( stbir_make16x2(rega,regb), stbir_make16(one, two, three, four) ) ) + + #define stbir__simdi_16madd( out, reg0, reg1 ) \ + { \ + int16x8_t r0 = vreinterpretq_s16_u32(reg0); \ + int16x8_t r1 = vreinterpretq_s16_u32(reg1); \ + int32x4_t tmp0 = vmull_s16( vget_low_s16(r0), vget_low_s16(r1) ); \ + int32x4_t tmp1 = vmull_s16( vget_high_s16(r0), vget_high_s16(r1) ); \ + (out) = vreinterpretq_u32_s32( vpaddq_s32(tmp0, tmp1) ); \ + } + + #else + + #define stbir__simdf_aaa1( out, alp, ones ) (out) = vsetq_lane_f32(1.0f, vdupq_n_f32(vgetq_lane_f32(alp, 3)), 3) + #define stbir__simdf_1aaa( out, alp, ones ) (out) = vsetq_lane_f32(1.0f, vdupq_n_f32(vgetq_lane_f32(alp, 0)), 0) + + #if defined( _MSC_VER ) && !defined(__clang__) + static stbir__inline uint8x8x2_t stbir_make8x2(float32x4_t reg) + { + uint8x8x2_t r = { { vget_low_u8(vreinterpretq_u8_f32(reg)), vget_high_u8(vreinterpretq_u8_f32(reg)) } }; + return r; + } + #define stbir_make8(a,b) vcreate_u8( \ + (4*a+0) | ((4*a+1)<<8) | ((4*a+2)<<16) | ((4*a+3)<<24) | \ + ((stbir_uint64)(4*b+0)<<32) | ((stbir_uint64)(4*b+1)<<40) | ((stbir_uint64)(4*b+2)<<48) | ((stbir_uint64)(4*b+3)<<56) ) + #else + #define stbir_make8x2(reg) (uint8x8x2_t){ { vget_low_u8(vreinterpretq_u8_f32(reg)), vget_high_u8(vreinterpretq_u8_f32(reg)) } } + #define stbir_make8(a,b) (uint8x8_t){4*a+0,4*a+1,4*a+2,4*a+3,4*b+0,4*b+1,4*b+2,4*b+3} + #endif + + #define stbir__simdf_swiz( reg, one, two, three, four ) vreinterpretq_f32_u8( vcombine_u8( \ + vtbl2_u8( stbir_make8x2( reg ), stbir_make8( one, two ) ), \ + vtbl2_u8( stbir_make8x2( reg ), stbir_make8( three, four ) ) ) ) + + #define stbir__simdi_16madd( out, reg0, reg1 ) \ + { \ + int16x8_t r0 = vreinterpretq_s16_u32(reg0); \ + int16x8_t r1 = vreinterpretq_s16_u32(reg1); \ + int32x4_t tmp0 = vmull_s16( vget_low_s16(r0), vget_low_s16(r1) ); \ + int32x4_t tmp1 = vmull_s16( vget_high_s16(r0), vget_high_s16(r1) ); \ + int32x2_t out0 = vpadd_s32( vget_low_s32(tmp0), vget_high_s32(tmp0) ); \ + int32x2_t out1 = vpadd_s32( vget_low_s32(tmp1), vget_high_s32(tmp1) ); \ + (out) = vreinterpretq_u32_s32( vcombine_s32(out0, out1) ); \ + } + + #endif + + #define stbir__simdi_and( out, reg0, reg1 ) (out) = vandq_u32( reg0, reg1 ) + #define stbir__simdi_or( out, reg0, reg1 ) (out) = vorrq_u32( reg0, reg1 ) + + #define stbir__simdf_pack_to_8bytes(out,aa,bb) \ + { \ + float32x4_t af = vmaxq_f32( vminq_f32(aa,STBIR__CONSTF(STBIR_max_uint8_as_float) ), vdupq_n_f32(0) ); \ + float32x4_t bf = vmaxq_f32( vminq_f32(bb,STBIR__CONSTF(STBIR_max_uint8_as_float) ), vdupq_n_f32(0) ); \ + int16x4_t ai = vqmovn_s32( vcvtq_s32_f32( af ) ); \ + int16x4_t bi = vqmovn_s32( vcvtq_s32_f32( bf ) ); \ + uint8x8_t out8 = vqmovun_s16( vcombine_s16(ai, bi) ); \ + out = vreinterpretq_u32_u8( vcombine_u8(out8, out8) ); \ + } + + #define stbir__simdf_pack_to_8words(out,aa,bb) \ + { \ + float32x4_t af = vmaxq_f32( vminq_f32(aa,STBIR__CONSTF(STBIR_max_uint16_as_float) ), vdupq_n_f32(0) ); \ + float32x4_t bf = vmaxq_f32( vminq_f32(bb,STBIR__CONSTF(STBIR_max_uint16_as_float) ), vdupq_n_f32(0) ); \ + int32x4_t ai = vcvtq_s32_f32( af ); \ + int32x4_t bi = vcvtq_s32_f32( bf ); \ + out = vreinterpretq_u32_u16( vcombine_u16(vqmovun_s32(ai), vqmovun_s32(bi)) ); \ + } + + #define stbir__interleave_pack_and_store_16_u8( ptr, r0, r1, r2, r3 ) \ + { \ + int16x4x2_t tmp0 = vzip_s16( vqmovn_s32(vreinterpretq_s32_u32(r0)), vqmovn_s32(vreinterpretq_s32_u32(r2)) ); \ + int16x4x2_t tmp1 = vzip_s16( vqmovn_s32(vreinterpretq_s32_u32(r1)), vqmovn_s32(vreinterpretq_s32_u32(r3)) ); \ + uint8x8x2_t out = \ + { { \ + vqmovun_s16( vcombine_s16(tmp0.val[0], tmp0.val[1]) ), \ + vqmovun_s16( vcombine_s16(tmp1.val[0], tmp1.val[1]) ), \ + } }; \ + vst2_u8(ptr, out); \ + } + + #define stbir__simdf_load4_transposed( o0, o1, o2, o3, ptr ) \ + { \ + float32x4x4_t tmp = vld4q_f32(ptr); \ + o0 = tmp.val[0]; \ + o1 = tmp.val[1]; \ + o2 = tmp.val[2]; \ + o3 = tmp.val[3]; \ + } + + #define stbir__simdi_32shr( out, reg, imm ) out = vshrq_n_u32( reg, imm ) + + #if defined( _MSC_VER ) && !defined(__clang__) + #define STBIR__SIMDF_CONST(var, x) __declspec(align(8)) float var[] = { x, x, x, x } + #define STBIR__SIMDI_CONST(var, x) __declspec(align(8)) uint32_t var[] = { x, x, x, x } + #define STBIR__CONSTF(var) (*(const float32x4_t*)var) + #define STBIR__CONSTI(var) (*(const uint32x4_t*)var) + #else + #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = { x, x, x, x } + #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { x, x, x, x } + #define STBIR__CONSTF(var) (var) + #define STBIR__CONSTI(var) (var) + #endif + + #ifdef STBIR_FLOORF + #undef STBIR_FLOORF + #endif + #define STBIR_FLOORF stbir_simd_floorf + static stbir__inline float stbir_simd_floorf(float x) + { + #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) + return vget_lane_f32( vrndm_f32( vdup_n_f32(x) ), 0); + #else + float32x2_t f = vdup_n_f32(x); + float32x2_t t = vcvt_f32_s32(vcvt_s32_f32(f)); + uint32x2_t a = vclt_f32(f, t); + uint32x2_t b = vreinterpret_u32_f32(vdup_n_f32(-1.0f)); + float32x2_t r = vadd_f32(t, vreinterpret_f32_u32(vand_u32(a, b))); + return vget_lane_f32(r, 0); + #endif + } + + #ifdef STBIR_CEILF + #undef STBIR_CEILF + #endif + #define STBIR_CEILF stbir_simd_ceilf + static stbir__inline float stbir_simd_ceilf(float x) + { + #if defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) + return vget_lane_f32( vrndp_f32( vdup_n_f32(x) ), 0); + #else + float32x2_t f = vdup_n_f32(x); + float32x2_t t = vcvt_f32_s32(vcvt_s32_f32(f)); + uint32x2_t a = vclt_f32(t, f); + uint32x2_t b = vreinterpret_u32_f32(vdup_n_f32(1.0f)); + float32x2_t r = vadd_f32(t, vreinterpret_f32_u32(vand_u32(a, b))); + return vget_lane_f32(r, 0); + #endif + } + + #define STBIR_SIMD + +#elif defined(STBIR_WASM) + + #include + + #define stbir__simdf v128_t + #define stbir__simdi v128_t + + #define stbir_simdi_castf( reg ) (reg) + #define stbir_simdf_casti( reg ) (reg) + + #define stbir__simdf_load( reg, ptr ) (reg) = wasm_v128_load( (void const*)(ptr) ) + #define stbir__simdi_load( reg, ptr ) (reg) = wasm_v128_load( (void const*)(ptr) ) + #define stbir__simdf_load1( out, ptr ) (out) = wasm_v128_load32_splat( (void const*)(ptr) ) // top values can be random (not denormal or nan for perf) + #define stbir__simdi_load1( out, ptr ) (out) = wasm_v128_load32_splat( (void const*)(ptr) ) + #define stbir__simdf_load1z( out, ptr ) (out) = wasm_v128_load32_zero( (void const*)(ptr) ) // top values must be zero + #define stbir__simdf_frep4( fvar ) wasm_f32x4_splat( fvar ) + #define stbir__simdf_load1frep4( out, fvar ) (out) = wasm_f32x4_splat( fvar ) + #define stbir__simdf_load2( out, ptr ) (out) = wasm_v128_load64_splat( (void const*)(ptr) ) // top values can be random (not denormal or nan for perf) + #define stbir__simdf_load2z( out, ptr ) (out) = wasm_v128_load64_zero( (void const*)(ptr) ) // top values must be zero + #define stbir__simdf_load2hmerge( out, reg, ptr ) (out) = wasm_v128_load64_lane( (void const*)(ptr), reg, 1 ) + + #define stbir__simdf_zeroP() wasm_f32x4_const_splat(0) + #define stbir__simdf_zero( reg ) (reg) = wasm_f32x4_const_splat(0) + + #define stbir__simdf_store( ptr, reg ) wasm_v128_store( (void*)(ptr), reg ) + #define stbir__simdf_store1( ptr, reg ) wasm_v128_store32_lane( (void*)(ptr), reg, 0 ) + #define stbir__simdf_store2( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 0 ) + #define stbir__simdf_store2h( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 1 ) + + #define stbir__simdi_store( ptr, reg ) wasm_v128_store( (void*)(ptr), reg ) + #define stbir__simdi_store1( ptr, reg ) wasm_v128_store32_lane( (void*)(ptr), reg, 0 ) + #define stbir__simdi_store2( ptr, reg ) wasm_v128_store64_lane( (void*)(ptr), reg, 0 ) + + #define stbir__prefetch( ptr ) + + #define stbir__simdi_expand_u8_to_u32(out0,out1,out2,out3,ireg) \ + { \ + v128_t l = wasm_u16x8_extend_low_u8x16 ( ireg ); \ + v128_t h = wasm_u16x8_extend_high_u8x16( ireg ); \ + out0 = wasm_u32x4_extend_low_u16x8 ( l ); \ + out1 = wasm_u32x4_extend_high_u16x8( l ); \ + out2 = wasm_u32x4_extend_low_u16x8 ( h ); \ + out3 = wasm_u32x4_extend_high_u16x8( h ); \ + } + + #define stbir__simdi_expand_u8_to_1u32(out,ireg) \ + { \ + v128_t tmp = wasm_u16x8_extend_low_u8x16(ireg); \ + out = wasm_u32x4_extend_low_u16x8(tmp); \ + } + + #define stbir__simdi_expand_u16_to_u32(out0,out1,ireg) \ + { \ + out0 = wasm_u32x4_extend_low_u16x8 ( ireg ); \ + out1 = wasm_u32x4_extend_high_u16x8( ireg ); \ + } + + #define stbir__simdf_convert_float_to_i32( i, f ) (i) = wasm_i32x4_trunc_sat_f32x4(f) + #define stbir__simdf_convert_float_to_int( f ) wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(f), 0) + #define stbir__simdi_to_int( i ) wasm_i32x4_extract_lane(i, 0) + #define stbir__simdf_convert_float_to_uint8( f ) ((unsigned char)wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(wasm_f32x4_max(wasm_f32x4_min(f,STBIR_max_uint8_as_float),wasm_f32x4_const_splat(0))), 0)) + #define stbir__simdf_convert_float_to_short( f ) ((unsigned short)wasm_i32x4_extract_lane(wasm_i32x4_trunc_sat_f32x4(wasm_f32x4_max(wasm_f32x4_min(f,STBIR_max_uint16_as_float),wasm_f32x4_const_splat(0))), 0)) + #define stbir__simdi_convert_i32_to_float(out, ireg) (out) = wasm_f32x4_convert_i32x4(ireg) + #define stbir__simdf_add( out, reg0, reg1 ) (out) = wasm_f32x4_add( reg0, reg1 ) + #define stbir__simdf_mult( out, reg0, reg1 ) (out) = wasm_f32x4_mul( reg0, reg1 ) + #define stbir__simdf_mult_mem( out, reg, ptr ) (out) = wasm_f32x4_mul( reg, wasm_v128_load( (void const*)(ptr) ) ) + #define stbir__simdf_mult1_mem( out, reg, ptr ) (out) = wasm_f32x4_mul( reg, wasm_v128_load32_splat( (void const*)(ptr) ) ) + #define stbir__simdf_add_mem( out, reg, ptr ) (out) = wasm_f32x4_add( reg, wasm_v128_load( (void const*)(ptr) ) ) + #define stbir__simdf_add1_mem( out, reg, ptr ) (out) = wasm_f32x4_add( reg, wasm_v128_load32_splat( (void const*)(ptr) ) ) + + #define stbir__simdf_madd( out, add, mul1, mul2 ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul1, mul2 ) ) + #define stbir__simdf_madd1( out, add, mul1, mul2 ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul1, mul2 ) ) + #define stbir__simdf_madd_mem( out, add, mul, ptr ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul, wasm_v128_load( (void const*)(ptr) ) ) ) + #define stbir__simdf_madd1_mem( out, add, mul, ptr ) (out) = wasm_f32x4_add( add, wasm_f32x4_mul( mul, wasm_v128_load32_splat( (void const*)(ptr) ) ) ) + + #define stbir__simdf_add1( out, reg0, reg1 ) (out) = wasm_f32x4_add( reg0, reg1 ) + #define stbir__simdf_mult1( out, reg0, reg1 ) (out) = wasm_f32x4_mul( reg0, reg1 ) + + #define stbir__simdf_and( out, reg0, reg1 ) (out) = wasm_v128_and( reg0, reg1 ) + #define stbir__simdf_or( out, reg0, reg1 ) (out) = wasm_v128_or( reg0, reg1 ) + + #define stbir__simdf_min( out, reg0, reg1 ) (out) = wasm_f32x4_min( reg0, reg1 ) + #define stbir__simdf_max( out, reg0, reg1 ) (out) = wasm_f32x4_max( reg0, reg1 ) + #define stbir__simdf_min1( out, reg0, reg1 ) (out) = wasm_f32x4_min( reg0, reg1 ) + #define stbir__simdf_max1( out, reg0, reg1 ) (out) = wasm_f32x4_max( reg0, reg1 ) + + #define stbir__simdf_0123ABCDto3ABx( out, reg0, reg1 ) (out) = wasm_i32x4_shuffle( reg0, reg1, 3, 4, 5, -1 ) + #define stbir__simdf_0123ABCDto23Ax( out, reg0, reg1 ) (out) = wasm_i32x4_shuffle( reg0, reg1, 2, 3, 4, -1 ) + + #define stbir__simdf_aaa1(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 3, 3, 3, 4) + #define stbir__simdf_1aaa(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 4, 0, 0, 0) + #define stbir__simdf_a1a1(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 1, 4, 3, 4) + #define stbir__simdf_1a1a(out,alp,ones) (out) = wasm_i32x4_shuffle(alp, ones, 4, 0, 4, 2) + + #define stbir__simdf_swiz( reg, one, two, three, four ) wasm_i32x4_shuffle(reg, reg, one, two, three, four) + + #define stbir__simdi_and( out, reg0, reg1 ) (out) = wasm_v128_and( reg0, reg1 ) + #define stbir__simdi_or( out, reg0, reg1 ) (out) = wasm_v128_or( reg0, reg1 ) + #define stbir__simdi_16madd( out, reg0, reg1 ) (out) = wasm_i32x4_dot_i16x8( reg0, reg1 ) + + #define stbir__simdf_pack_to_8bytes(out,aa,bb) \ + { \ + v128_t af = wasm_f32x4_max( wasm_f32x4_min(aa, STBIR_max_uint8_as_float), wasm_f32x4_const_splat(0) ); \ + v128_t bf = wasm_f32x4_max( wasm_f32x4_min(bb, STBIR_max_uint8_as_float), wasm_f32x4_const_splat(0) ); \ + v128_t ai = wasm_i32x4_trunc_sat_f32x4( af ); \ + v128_t bi = wasm_i32x4_trunc_sat_f32x4( bf ); \ + v128_t out16 = wasm_i16x8_narrow_i32x4( ai, bi ); \ + out = wasm_u8x16_narrow_i16x8( out16, out16 ); \ + } + + #define stbir__simdf_pack_to_8words(out,aa,bb) \ + { \ + v128_t af = wasm_f32x4_max( wasm_f32x4_min(aa, STBIR_max_uint16_as_float), wasm_f32x4_const_splat(0)); \ + v128_t bf = wasm_f32x4_max( wasm_f32x4_min(bb, STBIR_max_uint16_as_float), wasm_f32x4_const_splat(0)); \ + v128_t ai = wasm_i32x4_trunc_sat_f32x4( af ); \ + v128_t bi = wasm_i32x4_trunc_sat_f32x4( bf ); \ + out = wasm_u16x8_narrow_i32x4( ai, bi ); \ + } + + #define stbir__interleave_pack_and_store_16_u8( ptr, r0, r1, r2, r3 ) \ + { \ + v128_t tmp0 = wasm_i16x8_narrow_i32x4(r0, r1); \ + v128_t tmp1 = wasm_i16x8_narrow_i32x4(r2, r3); \ + v128_t tmp = wasm_u8x16_narrow_i16x8(tmp0, tmp1); \ + tmp = wasm_i8x16_shuffle(tmp, tmp, 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); \ + wasm_v128_store( (void*)(ptr), tmp); \ + } + + #define stbir__simdf_load4_transposed( o0, o1, o2, o3, ptr ) \ + { \ + v128_t t0 = wasm_v128_load( ptr ); \ + v128_t t1 = wasm_v128_load( ptr+4 ); \ + v128_t t2 = wasm_v128_load( ptr+8 ); \ + v128_t t3 = wasm_v128_load( ptr+12 ); \ + v128_t s0 = wasm_i32x4_shuffle(t0, t1, 0, 4, 2, 6); \ + v128_t s1 = wasm_i32x4_shuffle(t0, t1, 1, 5, 3, 7); \ + v128_t s2 = wasm_i32x4_shuffle(t2, t3, 0, 4, 2, 6); \ + v128_t s3 = wasm_i32x4_shuffle(t2, t3, 1, 5, 3, 7); \ + o0 = wasm_i32x4_shuffle(s0, s2, 0, 1, 4, 5); \ + o1 = wasm_i32x4_shuffle(s1, s3, 0, 1, 4, 5); \ + o2 = wasm_i32x4_shuffle(s0, s2, 2, 3, 6, 7); \ + o3 = wasm_i32x4_shuffle(s1, s3, 2, 3, 6, 7); \ + } + + #define stbir__simdi_32shr( out, reg, imm ) out = wasm_u32x4_shr( reg, imm ) + + typedef float stbir__f32x4 __attribute__((__vector_size__(16), __aligned__(16))); + #define STBIR__SIMDF_CONST(var, x) stbir__simdf var = (v128_t)(stbir__f32x4){ x, x, x, x } + #define STBIR__SIMDI_CONST(var, x) stbir__simdi var = { x, x, x, x } + #define STBIR__CONSTF(var) (var) + #define STBIR__CONSTI(var) (var) + + #ifdef STBIR_FLOORF + #undef STBIR_FLOORF + #endif + #define STBIR_FLOORF stbir_simd_floorf + static stbir__inline float stbir_simd_floorf(float x) + { + return wasm_f32x4_extract_lane( wasm_f32x4_floor( wasm_f32x4_splat(x) ), 0); + } + + #ifdef STBIR_CEILF + #undef STBIR_CEILF + #endif + #define STBIR_CEILF stbir_simd_ceilf + static stbir__inline float stbir_simd_ceilf(float x) + { + return wasm_f32x4_extract_lane( wasm_f32x4_ceil( wasm_f32x4_splat(x) ), 0); + } + + #define STBIR_SIMD + +#endif // SSE2/NEON/WASM + +#endif // NO SIMD + +#ifdef STBIR_SIMD8 + #define stbir__simdfX stbir__simdf8 + #define stbir__simdiX stbir__simdi8 + #define stbir__simdfX_load stbir__simdf8_load + #define stbir__simdiX_load stbir__simdi8_load + #define stbir__simdfX_mult stbir__simdf8_mult + #define stbir__simdfX_add_mem stbir__simdf8_add_mem + #define stbir__simdfX_madd_mem stbir__simdf8_madd_mem + #define stbir__simdfX_store stbir__simdf8_store + #define stbir__simdiX_store stbir__simdi8_store + #define stbir__simdf_frepX stbir__simdf8_frep8 + #define stbir__simdfX_madd stbir__simdf8_madd + #define stbir__simdfX_min stbir__simdf8_min + #define stbir__simdfX_max stbir__simdf8_max + #define stbir__simdfX_aaa1 stbir__simdf8_aaa1 + #define stbir__simdfX_1aaa stbir__simdf8_1aaa + #define stbir__simdfX_a1a1 stbir__simdf8_a1a1 + #define stbir__simdfX_1a1a stbir__simdf8_1a1a + #define stbir__simdfX_convert_float_to_i32 stbir__simdf8_convert_float_to_i32 + #define stbir__simdfX_pack_to_words stbir__simdf8_pack_to_16words + #define stbir__simdfX_zero stbir__simdf8_zero + #define STBIR_onesX STBIR_ones8 + #define STBIR_max_uint8_as_floatX STBIR_max_uint8_as_float8 + #define STBIR_max_uint16_as_floatX STBIR_max_uint16_as_float8 + #define STBIR_simd_point5X STBIR_simd_point58 + #define stbir__simdfX_float_count 8 + #define stbir__simdfX_0123to1230 stbir__simdf8_0123to12301230 + #define stbir__simdfX_0123to2103 stbir__simdf8_0123to21032103 + static const stbir__simdf8 STBIR_max_uint16_as_float_inverted8 = { stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted,stbir__max_uint16_as_float_inverted }; + static const stbir__simdf8 STBIR_max_uint8_as_float_inverted8 = { stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted,stbir__max_uint8_as_float_inverted }; + static const stbir__simdf8 STBIR_ones8 = { 1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 }; + static const stbir__simdf8 STBIR_simd_point58 = { 0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5 }; + static const stbir__simdf8 STBIR_max_uint8_as_float8 = { stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float, stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float,stbir__max_uint8_as_float }; + static const stbir__simdf8 STBIR_max_uint16_as_float8 = { stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float, stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float,stbir__max_uint16_as_float }; +#else + #define stbir__simdfX stbir__simdf + #define stbir__simdiX stbir__simdi + #define stbir__simdfX_load stbir__simdf_load + #define stbir__simdiX_load stbir__simdi_load + #define stbir__simdfX_mult stbir__simdf_mult + #define stbir__simdfX_add_mem stbir__simdf_add_mem + #define stbir__simdfX_madd_mem stbir__simdf_madd_mem + #define stbir__simdfX_store stbir__simdf_store + #define stbir__simdiX_store stbir__simdi_store + #define stbir__simdf_frepX stbir__simdf_frep4 + #define stbir__simdfX_madd stbir__simdf_madd + #define stbir__simdfX_min stbir__simdf_min + #define stbir__simdfX_max stbir__simdf_max + #define stbir__simdfX_aaa1 stbir__simdf_aaa1 + #define stbir__simdfX_1aaa stbir__simdf_1aaa + #define stbir__simdfX_a1a1 stbir__simdf_a1a1 + #define stbir__simdfX_1a1a stbir__simdf_1a1a + #define stbir__simdfX_convert_float_to_i32 stbir__simdf_convert_float_to_i32 + #define stbir__simdfX_pack_to_words stbir__simdf_pack_to_8words + #define stbir__simdfX_zero stbir__simdf_zero + #define STBIR_onesX STBIR__CONSTF(STBIR_ones) + #define STBIR_simd_point5X STBIR__CONSTF(STBIR_simd_point5) + #define STBIR_max_uint8_as_floatX STBIR__CONSTF(STBIR_max_uint8_as_float) + #define STBIR_max_uint16_as_floatX STBIR__CONSTF(STBIR_max_uint16_as_float) + #define stbir__simdfX_float_count 4 + #define stbir__if_simdf8_cast_to_simdf4( val ) ( val ) + #define stbir__simdfX_0123to1230 stbir__simdf_0123to1230 + #define stbir__simdfX_0123to2103 stbir__simdf_0123to2103 +#endif + + +#if defined(STBIR_NEON) && !defined(_M_ARM) && !defined(__arm__) + + #if defined( _MSC_VER ) && !defined(__clang__) + typedef __int16 stbir__FP16; + #else + typedef float16_t stbir__FP16; + #endif + +#else // no NEON, or 32-bit ARM for MSVC + + typedef union stbir__FP16 + { + unsigned short u; + } stbir__FP16; + +#endif + +#if (!defined(STBIR_NEON) && !defined(STBIR_FP16C)) || (defined(STBIR_NEON) && defined(_M_ARM)) || (defined(STBIR_NEON) && defined(__arm__)) + + // Fabian's half float routines, see: https://gist.github.com/rygorous/2156668 + + static stbir__inline float stbir__half_to_float( stbir__FP16 h ) + { + static const stbir__FP32 magic = { (254 - 15) << 23 }; + static const stbir__FP32 was_infnan = { (127 + 16) << 23 }; + stbir__FP32 o; + + o.u = (h.u & 0x7fff) << 13; // exponent/mantissa bits + o.f *= magic.f; // exponent adjust + if (o.f >= was_infnan.f) // make sure Inf/NaN survive + o.u |= 255 << 23; + o.u |= (h.u & 0x8000) << 16; // sign bit + return o.f; + } + + static stbir__inline stbir__FP16 stbir__float_to_half(float val) + { + stbir__FP32 f32infty = { 255 << 23 }; + stbir__FP32 f16max = { (127 + 16) << 23 }; + stbir__FP32 denorm_magic = { ((127 - 15) + (23 - 10) + 1) << 23 }; + unsigned int sign_mask = 0x80000000u; + stbir__FP16 o = { 0 }; + stbir__FP32 f; + unsigned int sign; + + f.f = val; + sign = f.u & sign_mask; + f.u ^= sign; + + if (f.u >= f16max.u) // result is Inf or NaN (all exponent bits set) + o.u = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf + else // (De)normalized number or zero + { + if (f.u < (113 << 23)) // resulting FP16 is subnormal or zero + { + // use a magic value to align our 10 mantissa bits at the bottom of + // the float. as long as FP addition is round-to-nearest-even this + // just works. + f.f += denorm_magic.f; + // and one integer subtract of the bias later, we have our final float! + o.u = (unsigned short) ( f.u - denorm_magic.u ); + } + else + { + unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd + // update exponent, rounding bias part 1 + f.u = f.u + ((15u - 127) << 23) + 0xfff; + // rounding bias part 2 + f.u += mant_odd; + // take the bits! + o.u = (unsigned short) ( f.u >> 13 ); + } + } + + o.u |= sign >> 16; + return o; + } + +#endif + + +#if defined(STBIR_FP16C) + + #include + + static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) + { + _mm256_storeu_ps( (float*)output, _mm256_cvtph_ps( _mm_loadu_si128( (__m128i const* )input ) ) ); + } + + static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input) + { + _mm_storeu_si128( (__m128i*)output, _mm256_cvtps_ph( _mm256_loadu_ps( input ), 0 ) ); + } + + static stbir__inline float stbir__half_to_float( stbir__FP16 h ) + { + return _mm_cvtss_f32( _mm_cvtph_ps( _mm_cvtsi32_si128( (int)h.u ) ) ); + } + + static stbir__inline stbir__FP16 stbir__float_to_half( float f ) + { + stbir__FP16 h; + h.u = (unsigned short) _mm_cvtsi128_si32( _mm_cvtps_ph( _mm_set_ss( f ), 0 ) ); + return h; + } + +#elif defined(STBIR_SSE2) + + // Fabian's half float routines, see: https://gist.github.com/rygorous/2156668 + stbir__inline static void stbir__half_to_float_SIMD(float * output, void const * input) + { + static const STBIR__SIMDI_CONST(mask_nosign, 0x7fff); + static const STBIR__SIMDI_CONST(smallest_normal, 0x0400); + static const STBIR__SIMDI_CONST(infinity, 0x7c00); + static const STBIR__SIMDI_CONST(expadjust_normal, (127 - 15) << 23); + static const STBIR__SIMDI_CONST(magic_denorm, 113 << 23); + + __m128i i = _mm_loadu_si128 ( (__m128i const*)(input) ); + __m128i h = _mm_unpacklo_epi16 ( i, _mm_setzero_si128() ); + __m128i mnosign = STBIR__CONSTI(mask_nosign); + __m128i eadjust = STBIR__CONSTI(expadjust_normal); + __m128i smallest = STBIR__CONSTI(smallest_normal); + __m128i infty = STBIR__CONSTI(infinity); + __m128i expmant = _mm_and_si128(mnosign, h); + __m128i justsign = _mm_xor_si128(h, expmant); + __m128i b_notinfnan = _mm_cmpgt_epi32(infty, expmant); + __m128i b_isdenorm = _mm_cmpgt_epi32(smallest, expmant); + __m128i shifted = _mm_slli_epi32(expmant, 13); + __m128i adj_infnan = _mm_andnot_si128(b_notinfnan, eadjust); + __m128i adjusted = _mm_add_epi32(eadjust, shifted); + __m128i den1 = _mm_add_epi32(shifted, STBIR__CONSTI(magic_denorm)); + __m128i adjusted2 = _mm_add_epi32(adjusted, adj_infnan); + __m128 den2 = _mm_sub_ps(_mm_castsi128_ps(den1), *(const __m128 *)&magic_denorm); + __m128 adjusted3 = _mm_and_ps(den2, _mm_castsi128_ps(b_isdenorm)); + __m128 adjusted4 = _mm_andnot_ps(_mm_castsi128_ps(b_isdenorm), _mm_castsi128_ps(adjusted2)); + __m128 adjusted5 = _mm_or_ps(adjusted3, adjusted4); + __m128i sign = _mm_slli_epi32(justsign, 16); + __m128 final = _mm_or_ps(adjusted5, _mm_castsi128_ps(sign)); + stbir__simdf_store( output + 0, final ); + + h = _mm_unpackhi_epi16 ( i, _mm_setzero_si128() ); + expmant = _mm_and_si128(mnosign, h); + justsign = _mm_xor_si128(h, expmant); + b_notinfnan = _mm_cmpgt_epi32(infty, expmant); + b_isdenorm = _mm_cmpgt_epi32(smallest, expmant); + shifted = _mm_slli_epi32(expmant, 13); + adj_infnan = _mm_andnot_si128(b_notinfnan, eadjust); + adjusted = _mm_add_epi32(eadjust, shifted); + den1 = _mm_add_epi32(shifted, STBIR__CONSTI(magic_denorm)); + adjusted2 = _mm_add_epi32(adjusted, adj_infnan); + den2 = _mm_sub_ps(_mm_castsi128_ps(den1), *(const __m128 *)&magic_denorm); + adjusted3 = _mm_and_ps(den2, _mm_castsi128_ps(b_isdenorm)); + adjusted4 = _mm_andnot_ps(_mm_castsi128_ps(b_isdenorm), _mm_castsi128_ps(adjusted2)); + adjusted5 = _mm_or_ps(adjusted3, adjusted4); + sign = _mm_slli_epi32(justsign, 16); + final = _mm_or_ps(adjusted5, _mm_castsi128_ps(sign)); + stbir__simdf_store( output + 4, final ); + + // ~38 SSE2 ops for 8 values + } + + // Fabian's round-to-nearest-even float to half + // ~48 SSE2 ops for 8 output + stbir__inline static void stbir__float_to_half_SIMD(void * output, float const * input) + { + static const STBIR__SIMDI_CONST(mask_sign, 0x80000000u); + static const STBIR__SIMDI_CONST(c_f16max, (127 + 16) << 23); // all FP32 values >=this round to +inf + static const STBIR__SIMDI_CONST(c_nanbit, 0x200); + static const STBIR__SIMDI_CONST(c_infty_as_fp16, 0x7c00); + static const STBIR__SIMDI_CONST(c_min_normal, (127 - 14) << 23); // smallest FP32 that yields a normalized FP16 + static const STBIR__SIMDI_CONST(c_subnorm_magic, ((127 - 15) + (23 - 10) + 1) << 23); + static const STBIR__SIMDI_CONST(c_normal_bias, 0xfff - ((127 - 15) << 23)); // adjust exponent and add mantissa rounding + + __m128 f = _mm_loadu_ps(input); + __m128 msign = _mm_castsi128_ps(STBIR__CONSTI(mask_sign)); + __m128 justsign = _mm_and_ps(msign, f); + __m128 absf = _mm_xor_ps(f, justsign); + __m128i absf_int = _mm_castps_si128(absf); // the cast is "free" (extra bypass latency, but no thruput hit) + __m128i f16max = STBIR__CONSTI(c_f16max); + __m128 b_isnan = _mm_cmpunord_ps(absf, absf); // is this a NaN? + __m128i b_isregular = _mm_cmpgt_epi32(f16max, absf_int); // (sub)normalized or special? + __m128i nanbit = _mm_and_si128(_mm_castps_si128(b_isnan), STBIR__CONSTI(c_nanbit)); + __m128i inf_or_nan = _mm_or_si128(nanbit, STBIR__CONSTI(c_infty_as_fp16)); // output for specials + + __m128i min_normal = STBIR__CONSTI(c_min_normal); + __m128i b_issub = _mm_cmpgt_epi32(min_normal, absf_int); + + // "result is subnormal" path + __m128 subnorm1 = _mm_add_ps(absf, _mm_castsi128_ps(STBIR__CONSTI(c_subnorm_magic))); // magic value to round output mantissa + __m128i subnorm2 = _mm_sub_epi32(_mm_castps_si128(subnorm1), STBIR__CONSTI(c_subnorm_magic)); // subtract out bias + + // "result is normal" path + __m128i mantoddbit = _mm_slli_epi32(absf_int, 31 - 13); // shift bit 13 (mantissa LSB) to sign + __m128i mantodd = _mm_srai_epi32(mantoddbit, 31); // -1 if FP16 mantissa odd, else 0 + + __m128i round1 = _mm_add_epi32(absf_int, STBIR__CONSTI(c_normal_bias)); + __m128i round2 = _mm_sub_epi32(round1, mantodd); // if mantissa LSB odd, bias towards rounding up (RTNE) + __m128i normal = _mm_srli_epi32(round2, 13); // rounded result + + // combine the two non-specials + __m128i nonspecial = _mm_or_si128(_mm_and_si128(subnorm2, b_issub), _mm_andnot_si128(b_issub, normal)); + + // merge in specials as well + __m128i joined = _mm_or_si128(_mm_and_si128(nonspecial, b_isregular), _mm_andnot_si128(b_isregular, inf_or_nan)); + + __m128i sign_shift = _mm_srai_epi32(_mm_castps_si128(justsign), 16); + __m128i final2, final= _mm_or_si128(joined, sign_shift); + + f = _mm_loadu_ps(input+4); + justsign = _mm_and_ps(msign, f); + absf = _mm_xor_ps(f, justsign); + absf_int = _mm_castps_si128(absf); // the cast is "free" (extra bypass latency, but no thruput hit) + b_isnan = _mm_cmpunord_ps(absf, absf); // is this a NaN? + b_isregular = _mm_cmpgt_epi32(f16max, absf_int); // (sub)normalized or special? + nanbit = _mm_and_si128(_mm_castps_si128(b_isnan), c_nanbit); + inf_or_nan = _mm_or_si128(nanbit, STBIR__CONSTI(c_infty_as_fp16)); // output for specials + + b_issub = _mm_cmpgt_epi32(min_normal, absf_int); + + // "result is subnormal" path + subnorm1 = _mm_add_ps(absf, _mm_castsi128_ps(STBIR__CONSTI(c_subnorm_magic))); // magic value to round output mantissa + subnorm2 = _mm_sub_epi32(_mm_castps_si128(subnorm1), STBIR__CONSTI(c_subnorm_magic)); // subtract out bias + + // "result is normal" path + mantoddbit = _mm_slli_epi32(absf_int, 31 - 13); // shift bit 13 (mantissa LSB) to sign + mantodd = _mm_srai_epi32(mantoddbit, 31); // -1 if FP16 mantissa odd, else 0 + + round1 = _mm_add_epi32(absf_int, STBIR__CONSTI(c_normal_bias)); + round2 = _mm_sub_epi32(round1, mantodd); // if mantissa LSB odd, bias towards rounding up (RTNE) + normal = _mm_srli_epi32(round2, 13); // rounded result + + // combine the two non-specials + nonspecial = _mm_or_si128(_mm_and_si128(subnorm2, b_issub), _mm_andnot_si128(b_issub, normal)); + + // merge in specials as well + joined = _mm_or_si128(_mm_and_si128(nonspecial, b_isregular), _mm_andnot_si128(b_isregular, inf_or_nan)); + + sign_shift = _mm_srai_epi32(_mm_castps_si128(justsign), 16); + final2 = _mm_or_si128(joined, sign_shift); + final = _mm_packs_epi32(final, final2); + stbir__simdi_store( output,final ); + } + +#elif defined(STBIR_NEON) && defined(_MSC_VER) && defined(_M_ARM64) && !defined(__clang__) // 64-bit ARM on MSVC (not clang) + + static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) + { + float16x4_t in0 = vld1_f16(input + 0); + float16x4_t in1 = vld1_f16(input + 4); + vst1q_f32(output + 0, vcvt_f32_f16(in0)); + vst1q_f32(output + 4, vcvt_f32_f16(in1)); + } + + static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input) + { + float16x4_t out0 = vcvt_f16_f32(vld1q_f32(input + 0)); + float16x4_t out1 = vcvt_f16_f32(vld1q_f32(input + 4)); + vst1_f16(output+0, out0); + vst1_f16(output+4, out1); + } + + static stbir__inline float stbir__half_to_float( stbir__FP16 h ) + { + return vgetq_lane_f32(vcvt_f32_f16(vld1_dup_f16(&h)), 0); + } + + static stbir__inline stbir__FP16 stbir__float_to_half( float f ) + { + return vget_lane_f16(vcvt_f16_f32(vdupq_n_f32(f)), 0).n16_u16[0]; + } + +#elif defined(STBIR_NEON) && ( defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) ) // 64-bit ARM + + static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) + { + float16x8_t in = vld1q_f16(input); + vst1q_f32(output + 0, vcvt_f32_f16(vget_low_f16(in))); + vst1q_f32(output + 4, vcvt_f32_f16(vget_high_f16(in))); + } + + static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input) + { + float16x4_t out0 = vcvt_f16_f32(vld1q_f32(input + 0)); + float16x4_t out1 = vcvt_f16_f32(vld1q_f32(input + 4)); + vst1q_f16(output, vcombine_f16(out0, out1)); + } + + static stbir__inline float stbir__half_to_float( stbir__FP16 h ) + { + return vgetq_lane_f32(vcvt_f32_f16(vdup_n_f16(h)), 0); + } + + static stbir__inline stbir__FP16 stbir__float_to_half( float f ) + { + return vget_lane_f16(vcvt_f16_f32(vdupq_n_f32(f)), 0); + } + +#elif defined(STBIR_WASM) || (defined(STBIR_NEON) && (defined(_MSC_VER) || defined(_M_ARM) || defined(__arm__))) // WASM or 32-bit ARM on MSVC/clang + + static stbir__inline void stbir__half_to_float_SIMD(float * output, stbir__FP16 const * input) + { + for (int i=0; i<8; i++) + { + output[i] = stbir__half_to_float(input[i]); + } + } + static stbir__inline void stbir__float_to_half_SIMD(stbir__FP16 * output, float const * input) + { + for (int i=0; i<8; i++) + { + output[i] = stbir__float_to_half(input[i]); + } + } + +#endif + + +#ifdef STBIR_SIMD + +#define stbir__simdf_0123to3333( out, reg ) (out) = stbir__simdf_swiz( reg, 3,3,3,3 ) +#define stbir__simdf_0123to2222( out, reg ) (out) = stbir__simdf_swiz( reg, 2,2,2,2 ) +#define stbir__simdf_0123to1111( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,1,1 ) +#define stbir__simdf_0123to0000( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,0 ) +#define stbir__simdf_0123to0003( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,3 ) +#define stbir__simdf_0123to0001( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,0,1 ) +#define stbir__simdf_0123to1122( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,2,2 ) +#define stbir__simdf_0123to2333( out, reg ) (out) = stbir__simdf_swiz( reg, 2,3,3,3 ) +#define stbir__simdf_0123to0023( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,2,3 ) +#define stbir__simdf_0123to1230( out, reg ) (out) = stbir__simdf_swiz( reg, 1,2,3,0 ) +#define stbir__simdf_0123to2103( out, reg ) (out) = stbir__simdf_swiz( reg, 2,1,0,3 ) +#define stbir__simdf_0123to3210( out, reg ) (out) = stbir__simdf_swiz( reg, 3,2,1,0 ) +#define stbir__simdf_0123to2301( out, reg ) (out) = stbir__simdf_swiz( reg, 2,3,0,1 ) +#define stbir__simdf_0123to3012( out, reg ) (out) = stbir__simdf_swiz( reg, 3,0,1,2 ) +#define stbir__simdf_0123to0011( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,1,1 ) +#define stbir__simdf_0123to1100( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,0,0 ) +#define stbir__simdf_0123to2233( out, reg ) (out) = stbir__simdf_swiz( reg, 2,2,3,3 ) +#define stbir__simdf_0123to1133( out, reg ) (out) = stbir__simdf_swiz( reg, 1,1,3,3 ) +#define stbir__simdf_0123to0022( out, reg ) (out) = stbir__simdf_swiz( reg, 0,0,2,2 ) +#define stbir__simdf_0123to1032( out, reg ) (out) = stbir__simdf_swiz( reg, 1,0,3,2 ) + +typedef union stbir__simdi_u32 +{ + stbir_uint32 m128i_u32[4]; + int m128i_i32[4]; + stbir__simdi m128i_i128; +} stbir__simdi_u32; + +static const int STBIR_mask[9] = { 0,0,0,-1,-1,-1,0,0,0 }; + +static const STBIR__SIMDF_CONST(STBIR_max_uint8_as_float, stbir__max_uint8_as_float); +static const STBIR__SIMDF_CONST(STBIR_max_uint16_as_float, stbir__max_uint16_as_float); +static const STBIR__SIMDF_CONST(STBIR_max_uint8_as_float_inverted, stbir__max_uint8_as_float_inverted); +static const STBIR__SIMDF_CONST(STBIR_max_uint16_as_float_inverted, stbir__max_uint16_as_float_inverted); + +static const STBIR__SIMDF_CONST(STBIR_simd_point5, 0.5f); +static const STBIR__SIMDF_CONST(STBIR_ones, 1.0f); +static const STBIR__SIMDI_CONST(STBIR_almost_zero, (127 - 13) << 23); +static const STBIR__SIMDI_CONST(STBIR_almost_one, 0x3f7fffff); +static const STBIR__SIMDI_CONST(STBIR_mastissa_mask, 0xff); +static const STBIR__SIMDI_CONST(STBIR_topscale, 0x02000000); + +// Basically, in simd mode, we unroll the proper amount, and we don't want +// the non-simd remnant loops to be unroll because they only run a few times +// Adding this switch saves about 5K on clang which is Captain Unroll the 3rd. +#define STBIR_SIMD_STREAMOUT_PTR( star ) STBIR_STREAMOUT_PTR( star ) +#define STBIR_SIMD_NO_UNROLL(ptr) STBIR_NO_UNROLL(ptr) +#define STBIR_SIMD_NO_UNROLL_LOOP_START STBIR_NO_UNROLL_LOOP_START +#define STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR STBIR_NO_UNROLL_LOOP_START_INF_FOR + +#ifdef STBIR_MEMCPY +#undef STBIR_MEMCPY +#endif +#define STBIR_MEMCPY stbir_simd_memcpy + +// override normal use of memcpy with much simpler copy (faster and smaller with our sized copies) +static void stbir_simd_memcpy( void * dest, void const * src, size_t bytes ) +{ + char STBIR_SIMD_STREAMOUT_PTR (*) d = (char*) dest; + char STBIR_SIMD_STREAMOUT_PTR( * ) d_end = ((char*) dest) + bytes; + ptrdiff_t ofs_to_src = (char*)src - (char*)dest; + + // check overlaps + STBIR_ASSERT( ( ( d >= ( (char*)src) + bytes ) ) || ( ( d + bytes ) <= (char*)src ) ); + + if ( bytes < (16*stbir__simdfX_float_count) ) + { + if ( bytes < 16 ) + { + if ( bytes ) + { + STBIR_SIMD_NO_UNROLL_LOOP_START + do + { + STBIR_SIMD_NO_UNROLL(d); + d[ 0 ] = d[ ofs_to_src ]; + ++d; + } while ( d < d_end ); + } + } + else + { + stbir__simdf x; + // do one unaligned to get us aligned for the stream out below + stbir__simdf_load( x, ( d + ofs_to_src ) ); + stbir__simdf_store( d, x ); + d = (char*)( ( ( (size_t)d ) + 16 ) & ~15 ); + + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + STBIR_SIMD_NO_UNROLL(d); + + if ( d > ( d_end - 16 ) ) + { + if ( d == d_end ) + return; + d = d_end - 16; + } + + stbir__simdf_load( x, ( d + ofs_to_src ) ); + stbir__simdf_store( d, x ); + d += 16; + } + } + } + else + { + stbir__simdfX x0,x1,x2,x3; + + // do one unaligned to get us aligned for the stream out below + stbir__simdfX_load( x0, ( d + ofs_to_src ) + 0*stbir__simdfX_float_count ); + stbir__simdfX_load( x1, ( d + ofs_to_src ) + 4*stbir__simdfX_float_count ); + stbir__simdfX_load( x2, ( d + ofs_to_src ) + 8*stbir__simdfX_float_count ); + stbir__simdfX_load( x3, ( d + ofs_to_src ) + 12*stbir__simdfX_float_count ); + stbir__simdfX_store( d + 0*stbir__simdfX_float_count, x0 ); + stbir__simdfX_store( d + 4*stbir__simdfX_float_count, x1 ); + stbir__simdfX_store( d + 8*stbir__simdfX_float_count, x2 ); + stbir__simdfX_store( d + 12*stbir__simdfX_float_count, x3 ); + d = (char*)( ( ( (size_t)d ) + (16*stbir__simdfX_float_count) ) & ~((16*stbir__simdfX_float_count)-1) ); + + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + STBIR_SIMD_NO_UNROLL(d); + + if ( d > ( d_end - (16*stbir__simdfX_float_count) ) ) + { + if ( d == d_end ) + return; + d = d_end - (16*stbir__simdfX_float_count); + } + + stbir__simdfX_load( x0, ( d + ofs_to_src ) + 0*stbir__simdfX_float_count ); + stbir__simdfX_load( x1, ( d + ofs_to_src ) + 4*stbir__simdfX_float_count ); + stbir__simdfX_load( x2, ( d + ofs_to_src ) + 8*stbir__simdfX_float_count ); + stbir__simdfX_load( x3, ( d + ofs_to_src ) + 12*stbir__simdfX_float_count ); + stbir__simdfX_store( d + 0*stbir__simdfX_float_count, x0 ); + stbir__simdfX_store( d + 4*stbir__simdfX_float_count, x1 ); + stbir__simdfX_store( d + 8*stbir__simdfX_float_count, x2 ); + stbir__simdfX_store( d + 12*stbir__simdfX_float_count, x3 ); + d += (16*stbir__simdfX_float_count); + } + } +} + +// memcpy that is specically intentionally overlapping (src is smaller then dest, so can be +// a normal forward copy, bytes is divisible by 4 and bytes is greater than or equal to +// the diff between dest and src) +static void stbir_overlapping_memcpy( void * dest, void const * src, size_t bytes ) +{ + char STBIR_SIMD_STREAMOUT_PTR (*) sd = (char*) src; + char STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((char*) src) + bytes; + ptrdiff_t ofs_to_dest = (char*)dest - (char*)src; + + if ( ofs_to_dest >= 16 ) // is the overlap more than 16 away? + { + char STBIR_SIMD_STREAMOUT_PTR( * ) s_end16 = ((char*) src) + (bytes&~15); + STBIR_SIMD_NO_UNROLL_LOOP_START + do + { + stbir__simdf x; + STBIR_SIMD_NO_UNROLL(sd); + stbir__simdf_load( x, sd ); + stbir__simdf_store( ( sd + ofs_to_dest ), x ); + sd += 16; + } while ( sd < s_end16 ); + + if ( sd == s_end ) + return; + } + + do + { + STBIR_SIMD_NO_UNROLL(sd); + *(int*)( sd + ofs_to_dest ) = *(int*) sd; + sd += 4; + } while ( sd < s_end ); +} + +#else // no SSE2 + +// when in scalar mode, we let unrolling happen, so this macro just does the __restrict +#define STBIR_SIMD_STREAMOUT_PTR( star ) STBIR_STREAMOUT_PTR( star ) +#define STBIR_SIMD_NO_UNROLL(ptr) +#define STBIR_SIMD_NO_UNROLL_LOOP_START +#define STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + +#endif // SSE2 + + +#ifdef STBIR_PROFILE + +#ifndef STBIR_PROFILE_FUNC + +#if defined(_x86_64) || defined( __x86_64__ ) || defined( _M_X64 ) || defined(__x86_64) || defined(__SSE2__) || defined(STBIR_SSE) || defined( _M_IX86_FP ) || defined(__i386) || defined( __i386__ ) || defined( _M_IX86 ) || defined( _X86_ ) + +#ifdef _MSC_VER + + STBIRDEF stbir_uint64 __rdtsc(); + #define STBIR_PROFILE_FUNC() __rdtsc() + +#else // non msvc + + static stbir__inline stbir_uint64 STBIR_PROFILE_FUNC() + { + stbir_uint32 lo, hi; + asm volatile ("rdtsc" : "=a" (lo), "=d" (hi) ); + return ( ( (stbir_uint64) hi ) << 32 ) | ( (stbir_uint64) lo ); + } + +#endif // msvc + +#elif defined( _M_ARM64 ) || defined( __aarch64__ ) || defined( __arm64__ ) || defined(__ARM_NEON__) + +#if defined( _MSC_VER ) && !defined(__clang__) + + #define STBIR_PROFILE_FUNC() _ReadStatusReg(ARM64_CNTVCT) + +#else + + static stbir__inline stbir_uint64 STBIR_PROFILE_FUNC() + { + stbir_uint64 tsc; + asm volatile("mrs %0, cntvct_el0" : "=r" (tsc)); + return tsc; + } + +#endif + +#else // x64, arm + +#error Unknown platform for profiling. + +#endif // x64, arm + +#endif // STBIR_PROFILE_FUNC + +#define STBIR_ONLY_PROFILE_GET_SPLIT_INFO ,stbir__per_split_info * split_info +#define STBIR_ONLY_PROFILE_SET_SPLIT_INFO ,split_info + +#define STBIR_ONLY_PROFILE_BUILD_GET_INFO ,stbir__info * profile_info +#define STBIR_ONLY_PROFILE_BUILD_SET_INFO ,profile_info + +// super light-weight micro profiler +#define STBIR_PROFILE_START_ll( info, wh ) { stbir_uint64 wh##thiszonetime = STBIR_PROFILE_FUNC(); stbir_uint64 * wh##save_parent_excluded_ptr = info->current_zone_excluded_ptr; stbir_uint64 wh##current_zone_excluded = 0; info->current_zone_excluded_ptr = &wh##current_zone_excluded; +#define STBIR_PROFILE_END_ll( info, wh ) wh##thiszonetime = STBIR_PROFILE_FUNC() - wh##thiszonetime; info->profile.named.wh += wh##thiszonetime - wh##current_zone_excluded; *wh##save_parent_excluded_ptr += wh##thiszonetime; info->current_zone_excluded_ptr = wh##save_parent_excluded_ptr; } +#define STBIR_PROFILE_FIRST_START_ll( info, wh ) { int i; info->current_zone_excluded_ptr = &info->profile.named.total; for(i=0;iprofile.array);i++) info->profile.array[i]=0; } STBIR_PROFILE_START_ll( info, wh ); +#define STBIR_PROFILE_CLEAR_EXTRAS_ll( info, num ) { int extra; for(extra=1;extra<(num);extra++) { int i; for(i=0;iprofile.array);i++) (info)[extra].profile.array[i]=0; } } + +// for thread data +#define STBIR_PROFILE_START( wh ) STBIR_PROFILE_START_ll( split_info, wh ) +#define STBIR_PROFILE_END( wh ) STBIR_PROFILE_END_ll( split_info, wh ) +#define STBIR_PROFILE_FIRST_START( wh ) STBIR_PROFILE_FIRST_START_ll( split_info, wh ) +#define STBIR_PROFILE_CLEAR_EXTRAS() STBIR_PROFILE_CLEAR_EXTRAS_ll( split_info, split_count ) + +// for build data +#define STBIR_PROFILE_BUILD_START( wh ) STBIR_PROFILE_START_ll( profile_info, wh ) +#define STBIR_PROFILE_BUILD_END( wh ) STBIR_PROFILE_END_ll( profile_info, wh ) +#define STBIR_PROFILE_BUILD_FIRST_START( wh ) STBIR_PROFILE_FIRST_START_ll( profile_info, wh ) +#define STBIR_PROFILE_BUILD_CLEAR( info ) { int i; for(i=0;iprofile.array);i++) info->profile.array[i]=0; } + +#else // no profile + +#define STBIR_ONLY_PROFILE_GET_SPLIT_INFO +#define STBIR_ONLY_PROFILE_SET_SPLIT_INFO + +#define STBIR_ONLY_PROFILE_BUILD_GET_INFO +#define STBIR_ONLY_PROFILE_BUILD_SET_INFO + +#define STBIR_PROFILE_START( wh ) +#define STBIR_PROFILE_END( wh ) +#define STBIR_PROFILE_FIRST_START( wh ) +#define STBIR_PROFILE_CLEAR_EXTRAS( ) + +#define STBIR_PROFILE_BUILD_START( wh ) +#define STBIR_PROFILE_BUILD_END( wh ) +#define STBIR_PROFILE_BUILD_FIRST_START( wh ) +#define STBIR_PROFILE_BUILD_CLEAR( info ) + +#endif // stbir_profile + +#ifndef STBIR_CEILF +#include +#if _MSC_VER <= 1200 // support VC6 for Sean +#define STBIR_CEILF(x) ((float)ceil((float)(x))) +#define STBIR_FLOORF(x) ((float)floor((float)(x))) +#else +#define STBIR_CEILF(x) ceilf(x) +#define STBIR_FLOORF(x) floorf(x) +#endif +#endif + +#ifndef STBIR_MEMCPY +// For memcpy +#include +#define STBIR_MEMCPY( dest, src, len ) memcpy( dest, src, len ) +#endif + +#ifndef STBIR_SIMD + +// memcpy that is specifically intentionally overlapping (src is smaller then dest, so can be +// a normal forward copy, bytes is divisible by 4 and bytes is greater than or equal to +// the diff between dest and src) +static void stbir_overlapping_memcpy( void * dest, void const * src, size_t bytes ) +{ + char STBIR_SIMD_STREAMOUT_PTR (*) sd = (char*) src; + char STBIR_SIMD_STREAMOUT_PTR( * ) s_end = ((char*) src) + bytes; + ptrdiff_t ofs_to_dest = (char*)dest - (char*)src; + + if ( ofs_to_dest >= 8 ) // is the overlap more than 8 away? + { + char STBIR_SIMD_STREAMOUT_PTR( * ) s_end8 = ((char*) src) + (bytes&~7); + STBIR_NO_UNROLL_LOOP_START + do + { + STBIR_NO_UNROLL(sd); + *(stbir_uint64*)( sd + ofs_to_dest ) = *(stbir_uint64*) sd; + sd += 8; + } while ( sd < s_end8 ); + + if ( sd == s_end ) + return; + } + + STBIR_NO_UNROLL_LOOP_START + do + { + STBIR_NO_UNROLL(sd); + *(int*)( sd + ofs_to_dest ) = *(int*) sd; + sd += 4; + } while ( sd < s_end ); +} + +#endif + +static float stbir__filter_trapezoid(float x, float scale, void * user_data) +{ + float halfscale = scale / 2; + float t = 0.5f + halfscale; + STBIR_ASSERT(scale <= 1); + STBIR__UNUSED(user_data); + + if ( x < 0.0f ) x = -x; + + if (x >= t) + return 0.0f; + else + { + float r = 0.5f - halfscale; + if (x <= r) + return 1.0f; + else + return (t - x) / scale; + } +} + +static float stbir__support_trapezoid(float scale, void * user_data) +{ + STBIR__UNUSED(user_data); + return 0.5f + scale / 2.0f; +} + +static float stbir__filter_triangle(float x, float s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + if ( x < 0.0f ) x = -x; + + if (x <= 1.0f) + return 1.0f - x; + else + return 0.0f; +} + +static float stbir__filter_point(float x, float s, void * user_data) +{ + STBIR__UNUSED(x); + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + return 1.0f; +} + +static float stbir__filter_cubic(float x, float s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + if ( x < 0.0f ) x = -x; + + if (x < 1.0f) + return (4.0f + x*x*(3.0f*x - 6.0f))/6.0f; + else if (x < 2.0f) + return (8.0f + x*(-12.0f + x*(6.0f - x)))/6.0f; + + return (0.0f); +} + +static float stbir__filter_catmullrom(float x, float s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + if ( x < 0.0f ) x = -x; + + if (x < 1.0f) + return 1.0f - x*x*(2.5f - 1.5f*x); + else if (x < 2.0f) + return 2.0f - x*(4.0f + x*(0.5f*x - 2.5f)); + + return (0.0f); +} + +static float stbir__filter_mitchell(float x, float s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + + if ( x < 0.0f ) x = -x; + + if (x < 1.0f) + return (16.0f + x*x*(21.0f * x - 36.0f))/18.0f; + else if (x < 2.0f) + return (32.0f + x*(-60.0f + x*(36.0f - 7.0f*x)))/18.0f; + + return (0.0f); +} + +static float stbir__support_zeropoint5(float s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + return 0.5f; +} + +static float stbir__support_one(float s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + return 1; +} + +static float stbir__support_two(float s, void * user_data) +{ + STBIR__UNUSED(s); + STBIR__UNUSED(user_data); + return 2; +} + +// This is the maximum number of input samples that can affect an output sample +// with the given filter from the output pixel's perspective +static int stbir__get_filter_pixel_width(stbir__support_callback * support, float scale, void * user_data) +{ + STBIR_ASSERT(support != 0); + + if ( scale >= ( 1.0f-stbir__small_float ) ) // upscale + return (int)STBIR_CEILF(support(1.0f/scale,user_data) * 2.0f); + else + return (int)STBIR_CEILF(support(scale,user_data) * 2.0f / scale); +} + +// this is how many coefficents per run of the filter (which is different +// from the filter_pixel_width depending on if we are scattering or gathering) +static int stbir__get_coefficient_width(stbir__sampler * samp, int is_gather, void * user_data) +{ + float scale = samp->scale_info.scale; + stbir__support_callback * support = samp->filter_support; + + switch( is_gather ) + { + case 1: + return (int)STBIR_CEILF(support(1.0f / scale, user_data) * 2.0f); + case 2: + return (int)STBIR_CEILF(support(scale, user_data) * 2.0f / scale); + case 0: + return (int)STBIR_CEILF(support(scale, user_data) * 2.0f); + default: + STBIR_ASSERT( (is_gather >= 0 ) && (is_gather <= 2 ) ); + return 0; + } +} + +static int stbir__get_contributors(stbir__sampler * samp, int is_gather) +{ + if (is_gather) + return samp->scale_info.output_sub_size; + else + return (samp->scale_info.input_full_size + samp->filter_pixel_margin * 2); +} + +static int stbir__edge_zero_full( int n, int max ) +{ + STBIR__UNUSED(n); + STBIR__UNUSED(max); + return 0; // NOTREACHED +} + +static int stbir__edge_clamp_full( int n, int max ) +{ + if (n < 0) + return 0; + + if (n >= max) + return max - 1; + + return n; // NOTREACHED +} + +static int stbir__edge_reflect_full( int n, int max ) +{ + if (n < 0) + { + if (n > -max) + return -n; + else + return max - 1; + } + + if (n >= max) + { + int max2 = max * 2; + if (n >= max2) + return 0; + else + return max2 - n - 1; + } + + return n; // NOTREACHED +} + +static int stbir__edge_wrap_full( int n, int max ) +{ + if (n >= 0) + return (n % max); + else + { + int m = (-n) % max; + + if (m != 0) + m = max - m; + + return (m); + } +} + +typedef int stbir__edge_wrap_func( int n, int max ); +static stbir__edge_wrap_func * stbir__edge_wrap_slow[] = +{ + stbir__edge_clamp_full, // STBIR_EDGE_CLAMP + stbir__edge_reflect_full, // STBIR_EDGE_REFLECT + stbir__edge_wrap_full, // STBIR_EDGE_WRAP + stbir__edge_zero_full, // STBIR_EDGE_ZERO +}; + +stbir__inline static int stbir__edge_wrap(stbir_edge edge, int n, int max) +{ + // avoid per-pixel switch + if (n >= 0 && n < max) + return n; + return stbir__edge_wrap_slow[edge]( n, max ); +} + +#define STBIR__MERGE_RUNS_PIXEL_THRESHOLD 16 + +// get information on the extents of a sampler +static void stbir__get_extents( stbir__sampler * samp, stbir__extents * scanline_extents ) +{ + int j, stop; + int left_margin, right_margin; + int min_n = 0x7fffffff, max_n = -0x7fffffff; + int min_left = 0x7fffffff, max_left = -0x7fffffff; + int min_right = 0x7fffffff, max_right = -0x7fffffff; + stbir_edge edge = samp->edge; + stbir__contributors* contributors = samp->contributors; + int output_sub_size = samp->scale_info.output_sub_size; + int input_full_size = samp->scale_info.input_full_size; + int filter_pixel_margin = samp->filter_pixel_margin; + + STBIR_ASSERT( samp->is_gather ); + + stop = output_sub_size; + for (j = 0; j < stop; j++ ) + { + STBIR_ASSERT( contributors[j].n1 >= contributors[j].n0 ); + if ( contributors[j].n0 < min_n ) + { + min_n = contributors[j].n0; + stop = j + filter_pixel_margin; // if we find a new min, only scan another filter width + if ( stop > output_sub_size ) stop = output_sub_size; + } + } + + stop = 0; + for (j = output_sub_size - 1; j >= stop; j-- ) + { + STBIR_ASSERT( contributors[j].n1 >= contributors[j].n0 ); + if ( contributors[j].n1 > max_n ) + { + max_n = contributors[j].n1; + stop = j - filter_pixel_margin; // if we find a new max, only scan another filter width + if (stop<0) stop = 0; + } + } + + STBIR_ASSERT( scanline_extents->conservative.n0 <= min_n ); + STBIR_ASSERT( scanline_extents->conservative.n1 >= max_n ); + + // now calculate how much into the margins we really read + left_margin = 0; + if ( min_n < 0 ) + { + left_margin = -min_n; + min_n = 0; + } + + right_margin = 0; + if ( max_n >= input_full_size ) + { + right_margin = max_n - input_full_size + 1; + max_n = input_full_size - 1; + } + + // index 1 is margin pixel extents (how many pixels we hang over the edge) + scanline_extents->edge_sizes[0] = left_margin; + scanline_extents->edge_sizes[1] = right_margin; + + // index 2 is pixels read from the input + scanline_extents->spans[0].n0 = min_n; + scanline_extents->spans[0].n1 = max_n; + scanline_extents->spans[0].pixel_offset_for_input = min_n; + + // default to no other input range + scanline_extents->spans[1].n0 = 0; + scanline_extents->spans[1].n1 = -1; + scanline_extents->spans[1].pixel_offset_for_input = 0; + + // don't have to do edge calc for zero clamp + if ( edge == STBIR_EDGE_ZERO ) + return; + + // convert margin pixels to the pixels within the input (min and max) + for( j = -left_margin ; j < 0 ; j++ ) + { + int p = stbir__edge_wrap( edge, j, input_full_size ); + if ( p < min_left ) + min_left = p; + if ( p > max_left ) + max_left = p; + } + + for( j = input_full_size ; j < (input_full_size + right_margin) ; j++ ) + { + int p = stbir__edge_wrap( edge, j, input_full_size ); + if ( p < min_right ) + min_right = p; + if ( p > max_right ) + max_right = p; + } + + // merge the left margin pixel region if it connects within 4 pixels of main pixel region + if ( min_left != 0x7fffffff ) + { + if ( ( ( min_left <= min_n ) && ( ( max_left + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= min_n ) ) || + ( ( min_n <= min_left ) && ( ( max_n + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= max_left ) ) ) + { + scanline_extents->spans[0].n0 = min_n = stbir__min( min_n, min_left ); + scanline_extents->spans[0].n1 = max_n = stbir__max( max_n, max_left ); + scanline_extents->spans[0].pixel_offset_for_input = min_n; + left_margin = 0; + } + } + + // merge the right margin pixel region if it connects within 4 pixels of main pixel region + if ( min_right != 0x7fffffff ) + { + if ( ( ( min_right <= min_n ) && ( ( max_right + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= min_n ) ) || + ( ( min_n <= min_right ) && ( ( max_n + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= max_right ) ) ) + { + scanline_extents->spans[0].n0 = min_n = stbir__min( min_n, min_right ); + scanline_extents->spans[0].n1 = max_n = stbir__max( max_n, max_right ); + scanline_extents->spans[0].pixel_offset_for_input = min_n; + right_margin = 0; + } + } + + STBIR_ASSERT( scanline_extents->conservative.n0 <= min_n ); + STBIR_ASSERT( scanline_extents->conservative.n1 >= max_n ); + + // you get two ranges when you have the WRAP edge mode and you are doing just the a piece of the resize + // so you need to get a second run of pixels from the opposite side of the scanline (which you + // wouldn't need except for WRAP) + + + // if we can't merge the min_left range, add it as a second range + if ( ( left_margin ) && ( min_left != 0x7fffffff ) ) + { + stbir__span * newspan = scanline_extents->spans + 1; + STBIR_ASSERT( right_margin == 0 ); + if ( min_left < scanline_extents->spans[0].n0 ) + { + scanline_extents->spans[1].pixel_offset_for_input = scanline_extents->spans[0].n0; + scanline_extents->spans[1].n0 = scanline_extents->spans[0].n0; + scanline_extents->spans[1].n1 = scanline_extents->spans[0].n1; + --newspan; + } + newspan->pixel_offset_for_input = min_left; + newspan->n0 = -left_margin; + newspan->n1 = ( max_left - min_left ) - left_margin; + scanline_extents->edge_sizes[0] = 0; // don't need to copy the left margin, since we are directly decoding into the margin + return; + } + + // if we can't merge the min_left range, add it as a second range + if ( ( right_margin ) && ( min_right != 0x7fffffff ) ) + { + stbir__span * newspan = scanline_extents->spans + 1; + if ( min_right < scanline_extents->spans[0].n0 ) + { + scanline_extents->spans[1].pixel_offset_for_input = scanline_extents->spans[0].n0; + scanline_extents->spans[1].n0 = scanline_extents->spans[0].n0; + scanline_extents->spans[1].n1 = scanline_extents->spans[0].n1; + --newspan; + } + newspan->pixel_offset_for_input = min_right; + newspan->n0 = scanline_extents->spans[1].n1 + 1; + newspan->n1 = scanline_extents->spans[1].n1 + 1 + ( max_right - min_right ); + scanline_extents->edge_sizes[1] = 0; // don't need to copy the right margin, since we are directly decoding into the margin + return; + } +} + +static void stbir__calculate_in_pixel_range( int * first_pixel, int * last_pixel, float out_pixel_center, float out_filter_radius, float inv_scale, float out_shift, int input_size, stbir_edge edge ) +{ + int first, last; + float out_pixel_influence_lowerbound = out_pixel_center - out_filter_radius; + float out_pixel_influence_upperbound = out_pixel_center + out_filter_radius; + + float in_pixel_influence_lowerbound = (out_pixel_influence_lowerbound + out_shift) * inv_scale; + float in_pixel_influence_upperbound = (out_pixel_influence_upperbound + out_shift) * inv_scale; + + first = (int)(STBIR_FLOORF(in_pixel_influence_lowerbound + 0.5f)); + last = (int)(STBIR_FLOORF(in_pixel_influence_upperbound - 0.5f)); + if ( last < first ) last = first; // point sample mode can span a value *right* at 0.5, and cause these to cross + + if ( edge == STBIR_EDGE_WRAP ) + { + if ( first < -input_size ) + first = -input_size; + if ( last >= (input_size*2)) + last = (input_size*2) - 1; + } + + *first_pixel = first; + *last_pixel = last; +} + +static void stbir__calculate_coefficients_for_gather_upsample( float out_filter_radius, stbir__kernel_callback * kernel, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float* coefficient_group, int coefficient_width, stbir_edge edge, void * user_data ) +{ + int n, end; + float inv_scale = scale_info->inv_scale; + float out_shift = scale_info->pixel_shift; + int input_size = scale_info->input_full_size; + int numerator = scale_info->scale_numerator; + int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < num_contributors ) ); + + // Looping through out pixels + end = num_contributors; if ( polyphase ) end = numerator; + for (n = 0; n < end; n++) + { + int i; + int last_non_zero; + float out_pixel_center = (float)n + 0.5f; + float in_center_of_out = (out_pixel_center + out_shift) * inv_scale; + + int in_first_pixel, in_last_pixel; + + stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, out_pixel_center, out_filter_radius, inv_scale, out_shift, input_size, edge ); + + // make sure we never generate a range larger than our precalculated coeff width + // this only happens in point sample mode, but it's a good safe thing to do anyway + if ( ( in_last_pixel - in_first_pixel + 1 ) > coefficient_width ) + in_last_pixel = in_first_pixel + coefficient_width - 1; + + last_non_zero = -1; + for (i = 0; i <= in_last_pixel - in_first_pixel; i++) + { + float in_pixel_center = (float)(i + in_first_pixel) + 0.5f; + float coeff = kernel(in_center_of_out - in_pixel_center, inv_scale, user_data); + + // kill denormals + if ( ( ( coeff < stbir__small_float ) && ( coeff > -stbir__small_float ) ) ) + { + if ( i == 0 ) // if we're at the front, just eat zero contributors + { + STBIR_ASSERT ( ( in_last_pixel - in_first_pixel ) != 0 ); // there should be at least one contrib + ++in_first_pixel; + i--; + continue; + } + coeff = 0; // make sure is fully zero (should keep denormals away) + } + else + last_non_zero = i; + + coefficient_group[i] = coeff; + } + + in_last_pixel = last_non_zero+in_first_pixel; // kills trailing zeros + contributors->n0 = in_first_pixel; + contributors->n1 = in_last_pixel; + + STBIR_ASSERT(contributors->n1 >= contributors->n0); + + ++contributors; + coefficient_group += coefficient_width; + } +} + +static void stbir__insert_coeff( stbir__contributors * contribs, float * coeffs, int new_pixel, float new_coeff, int max_width ) +{ + if ( new_pixel <= contribs->n1 ) // before the end + { + if ( new_pixel < contribs->n0 ) // before the front? + { + if ( ( contribs->n1 - new_pixel + 1 ) <= max_width ) + { + int j, o = contribs->n0 - new_pixel; + for ( j = contribs->n1 - contribs->n0 ; j <= 0 ; j-- ) + coeffs[ j + o ] = coeffs[ j ]; + for ( j = 1 ; j < o ; j-- ) + coeffs[ j ] = coeffs[ 0 ]; + coeffs[ 0 ] = new_coeff; + contribs->n0 = new_pixel; + } + } + else + { + coeffs[ new_pixel - contribs->n0 ] += new_coeff; + } + } + else + { + if ( ( new_pixel - contribs->n0 + 1 ) <= max_width ) + { + int j, e = new_pixel - contribs->n0; + for( j = ( contribs->n1 - contribs->n0 ) + 1 ; j < e ; j++ ) // clear in-betweens coeffs if there are any + coeffs[j] = 0; + + coeffs[ e ] = new_coeff; + contribs->n1 = new_pixel; + } + } +} + +static void stbir__calculate_out_pixel_range( int * first_pixel, int * last_pixel, float in_pixel_center, float in_pixels_radius, float scale, float out_shift, int out_size ) +{ + float in_pixel_influence_lowerbound = in_pixel_center - in_pixels_radius; + float in_pixel_influence_upperbound = in_pixel_center + in_pixels_radius; + float out_pixel_influence_lowerbound = in_pixel_influence_lowerbound * scale - out_shift; + float out_pixel_influence_upperbound = in_pixel_influence_upperbound * scale - out_shift; + int out_first_pixel = (int)(STBIR_FLOORF(out_pixel_influence_lowerbound + 0.5f)); + int out_last_pixel = (int)(STBIR_FLOORF(out_pixel_influence_upperbound - 0.5f)); + + if ( out_first_pixel < 0 ) + out_first_pixel = 0; + if ( out_last_pixel >= out_size ) + out_last_pixel = out_size - 1; + *first_pixel = out_first_pixel; + *last_pixel = out_last_pixel; +} + +static void stbir__calculate_coefficients_for_gather_downsample( int start, int end, float in_pixels_radius, stbir__kernel_callback * kernel, stbir__scale_info * scale_info, int coefficient_width, int num_contributors, stbir__contributors * contributors, float * coefficient_group, void * user_data ) +{ + int in_pixel; + int i; + int first_out_inited = -1; + float scale = scale_info->scale; + float out_shift = scale_info->pixel_shift; + int out_size = scale_info->output_sub_size; + int numerator = scale_info->scale_numerator; + int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < out_size ) ); + + STBIR__UNUSED(num_contributors); + + // Loop through the input pixels + for (in_pixel = start; in_pixel < end; in_pixel++) + { + float in_pixel_center = (float)in_pixel + 0.5f; + float out_center_of_in = in_pixel_center * scale - out_shift; + int out_first_pixel, out_last_pixel; + + stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, in_pixel_center, in_pixels_radius, scale, out_shift, out_size ); + + if ( out_first_pixel > out_last_pixel ) + continue; + + // clamp or exit if we are using polyphase filtering, and the limit is up + if ( polyphase ) + { + // when polyphase, you only have to do coeffs up to the numerator count + if ( out_first_pixel == numerator ) + break; + + // don't do any extra work, clamp last pixel at numerator too + if ( out_last_pixel >= numerator ) + out_last_pixel = numerator - 1; + } + + for (i = 0; i <= out_last_pixel - out_first_pixel; i++) + { + float out_pixel_center = (float)(i + out_first_pixel) + 0.5f; + float x = out_pixel_center - out_center_of_in; + float coeff = kernel(x, scale, user_data) * scale; + + // kill the coeff if it's too small (avoid denormals) + if ( ( ( coeff < stbir__small_float ) && ( coeff > -stbir__small_float ) ) ) + coeff = 0.0f; + + { + int out = i + out_first_pixel; + float * coeffs = coefficient_group + out * coefficient_width; + stbir__contributors * contribs = contributors + out; + + // is this the first time this output pixel has been seen? Init it. + if ( out > first_out_inited ) + { + STBIR_ASSERT( out == ( first_out_inited + 1 ) ); // ensure we have only advanced one at time + first_out_inited = out; + contribs->n0 = in_pixel; + contribs->n1 = in_pixel; + coeffs[0] = coeff; + } + else + { + // insert on end (always in order) + if ( coeffs[0] == 0.0f ) // if the first coefficent is zero, then zap it for this coeffs + { + STBIR_ASSERT( ( in_pixel - contribs->n0 ) == 1 ); // ensure that when we zap, we're at the 2nd pos + contribs->n0 = in_pixel; + } + contribs->n1 = in_pixel; + STBIR_ASSERT( ( in_pixel - contribs->n0 ) < coefficient_width ); + coeffs[in_pixel - contribs->n0] = coeff; + } + } + } + } +} + +#ifdef STBIR_RENORMALIZE_IN_FLOAT +#define STBIR_RENORM_TYPE float +#else +#define STBIR_RENORM_TYPE double +#endif + +static void stbir__cleanup_gathered_coefficients( stbir_edge edge, stbir__filter_extent_info* filter_info, stbir__scale_info * scale_info, int num_contributors, stbir__contributors* contributors, float * coefficient_group, int coefficient_width ) +{ + int input_size = scale_info->input_full_size; + int input_last_n1 = input_size - 1; + int n, end; + int lowest = 0x7fffffff; + int highest = -0x7fffffff; + int widest = -1; + int numerator = scale_info->scale_numerator; + int denominator = scale_info->scale_denominator; + int polyphase = ( ( scale_info->scale_is_rational ) && ( numerator < num_contributors ) ); + float * coeffs; + stbir__contributors * contribs; + + // weight all the coeffs for each sample + coeffs = coefficient_group; + contribs = contributors; + end = num_contributors; if ( polyphase ) end = numerator; + for (n = 0; n < end; n++) + { + int i; + STBIR_RENORM_TYPE filter_scale, total_filter = 0; + int e; + + // add all contribs + e = contribs->n1 - contribs->n0; + for( i = 0 ; i <= e ; i++ ) + { + total_filter += (STBIR_RENORM_TYPE) coeffs[i]; + STBIR_ASSERT( ( coeffs[i] >= -2.0f ) && ( coeffs[i] <= 2.0f ) ); // check for wonky weights + } + + // rescale + if ( ( total_filter < stbir__small_float ) && ( total_filter > -stbir__small_float ) ) + { + // all coeffs are extremely small, just zero it + contribs->n1 = contribs->n0; + coeffs[0] = 0.0f; + } + else + { + // if the total isn't 1.0, rescale everything + if ( ( total_filter < (1.0f-stbir__small_float) ) || ( total_filter > (1.0f+stbir__small_float) ) ) + { + filter_scale = ((STBIR_RENORM_TYPE)1.0) / total_filter; + + // scale them all + for (i = 0; i <= e; i++) + coeffs[i] = (float) ( coeffs[i] * filter_scale ); + } + } + ++contribs; + coeffs += coefficient_width; + } + + // if we have a rational for the scale, we can exploit the polyphaseness to not calculate + // most of the coefficients, so we copy them here + if ( polyphase ) + { + stbir__contributors * prev_contribs = contributors; + stbir__contributors * cur_contribs = contributors + numerator; + + for( n = numerator ; n < num_contributors ; n++ ) + { + cur_contribs->n0 = prev_contribs->n0 + denominator; + cur_contribs->n1 = prev_contribs->n1 + denominator; + ++cur_contribs; + ++prev_contribs; + } + stbir_overlapping_memcpy( coefficient_group + numerator * coefficient_width, coefficient_group, ( num_contributors - numerator ) * coefficient_width * sizeof( coeffs[ 0 ] ) ); + } + + coeffs = coefficient_group; + contribs = contributors; + + for (n = 0; n < num_contributors; n++) + { + int i; + + // in zero edge mode, just remove out of bounds contribs completely (since their weights are accounted for now) + if ( edge == STBIR_EDGE_ZERO ) + { + // shrink the right side if necessary + if ( contribs->n1 > input_last_n1 ) + contribs->n1 = input_last_n1; + + // shrink the left side + if ( contribs->n0 < 0 ) + { + int j, left, skips = 0; + + skips = -contribs->n0; + contribs->n0 = 0; + + // now move down the weights + left = contribs->n1 - contribs->n0 + 1; + if ( left > 0 ) + { + for( j = 0 ; j < left ; j++ ) + coeffs[ j ] = coeffs[ j + skips ]; + } + } + } + else if ( ( edge == STBIR_EDGE_CLAMP ) || ( edge == STBIR_EDGE_REFLECT ) ) + { + // for clamp and reflect, calculate the true inbounds position (based on edge type) and just add that to the existing weight + + // right hand side first + if ( contribs->n1 > input_last_n1 ) + { + int start = contribs->n0; + int endi = contribs->n1; + contribs->n1 = input_last_n1; + for( i = input_size; i <= endi; i++ ) + stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( i, input_size ), coeffs[i-start], coefficient_width ); + } + + // now check left hand edge + if ( contribs->n0 < 0 ) + { + int save_n0; + float save_n0_coeff; + float * c = coeffs - ( contribs->n0 + 1 ); + + // reinsert the coeffs with it reflected or clamped (insert accumulates, if the coeffs exist) + for( i = -1 ; i > contribs->n0 ; i-- ) + stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( i, input_size ), *c--, coefficient_width ); + save_n0 = contribs->n0; + save_n0_coeff = c[0]; // save it, since we didn't do the final one (i==n0), because there might be too many coeffs to hold (before we resize)! + + // now slide all the coeffs down (since we have accumulated them in the positive contribs) and reset the first contrib + contribs->n0 = 0; + for(i = 0 ; i <= contribs->n1 ; i++ ) + coeffs[i] = coeffs[i-save_n0]; + + // now that we have shrunk down the contribs, we insert the first one safely + stbir__insert_coeff( contribs, coeffs, stbir__edge_wrap_slow[edge]( save_n0, input_size ), save_n0_coeff, coefficient_width ); + } + } + + if ( contribs->n0 <= contribs->n1 ) + { + int diff = contribs->n1 - contribs->n0 + 1; + while ( diff && ( coeffs[ diff-1 ] == 0.0f ) ) + --diff; + + contribs->n1 = contribs->n0 + diff - 1; + + if ( contribs->n0 <= contribs->n1 ) + { + if ( contribs->n0 < lowest ) + lowest = contribs->n0; + if ( contribs->n1 > highest ) + highest = contribs->n1; + if ( diff > widest ) + widest = diff; + } + + // re-zero out unused coefficients (if any) + for( i = diff ; i < coefficient_width ; i++ ) + coeffs[i] = 0.0f; + } + + ++contribs; + coeffs += coefficient_width; + } + filter_info->lowest = lowest; + filter_info->highest = highest; + filter_info->widest = widest; +} + +#undef STBIR_RENORM_TYPE + +static int stbir__pack_coefficients( int num_contributors, stbir__contributors* contributors, float * coefficents, int coefficient_width, int widest, int row0, int row1 ) +{ + #define STBIR_MOVE_1( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint32*)(dest))[0] = ((stbir_uint32*)(src))[0]; } + #define STBIR_MOVE_2( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint64*)(dest))[0] = ((stbir_uint64*)(src))[0]; } + #ifdef STBIR_SIMD + #define STBIR_MOVE_4( dest, src ) { stbir__simdf t; STBIR_NO_UNROLL(dest); stbir__simdf_load( t, src ); stbir__simdf_store( dest, t ); } + #else + #define STBIR_MOVE_4( dest, src ) { STBIR_NO_UNROLL(dest); ((stbir_uint64*)(dest))[0] = ((stbir_uint64*)(src))[0]; ((stbir_uint64*)(dest))[1] = ((stbir_uint64*)(src))[1]; } + #endif + + int row_end = row1 + 1; + STBIR__UNUSED( row0 ); // only used in an assert + + if ( coefficient_width != widest ) + { + float * pc = coefficents; + float * coeffs = coefficents; + float * pc_end = coefficents + num_contributors * widest; + switch( widest ) + { + case 1: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_1( pc, coeffs ); + ++pc; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 2: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_2( pc, coeffs ); + pc += 2; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 3: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_2( pc, coeffs ); + STBIR_MOVE_1( pc+2, coeffs+2 ); + pc += 3; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 4: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + pc += 4; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 5: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_1( pc+4, coeffs+4 ); + pc += 5; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 6: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_2( pc+4, coeffs+4 ); + pc += 6; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 7: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_2( pc+4, coeffs+4 ); + STBIR_MOVE_1( pc+6, coeffs+6 ); + pc += 7; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 8: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_4( pc+4, coeffs+4 ); + pc += 8; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 9: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_4( pc+4, coeffs+4 ); + STBIR_MOVE_1( pc+8, coeffs+8 ); + pc += 9; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 10: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_4( pc+4, coeffs+4 ); + STBIR_MOVE_2( pc+8, coeffs+8 ); + pc += 10; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 11: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_4( pc+4, coeffs+4 ); + STBIR_MOVE_2( pc+8, coeffs+8 ); + STBIR_MOVE_1( pc+10, coeffs+10 ); + pc += 11; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + case 12: + STBIR_NO_UNROLL_LOOP_START + do { + STBIR_MOVE_4( pc, coeffs ); + STBIR_MOVE_4( pc+4, coeffs+4 ); + STBIR_MOVE_4( pc+8, coeffs+8 ); + pc += 12; + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + default: + STBIR_NO_UNROLL_LOOP_START + do { + float * copy_end = pc + widest - 4; + float * c = coeffs; + do { + STBIR_NO_UNROLL( pc ); + STBIR_MOVE_4( pc, c ); + pc += 4; + c += 4; + } while ( pc <= copy_end ); + copy_end += 4; + STBIR_NO_UNROLL_LOOP_START + while ( pc < copy_end ) + { + STBIR_MOVE_1( pc, c ); + ++pc; ++c; + } + coeffs += coefficient_width; + } while ( pc < pc_end ); + break; + } + } + + // some horizontal routines read one float off the end (which is then masked off), so put in a sentinal so we don't read an snan or denormal + coefficents[ widest * num_contributors ] = 8888.0f; + + // the minimum we might read for unrolled filters widths is 12. So, we need to + // make sure we never read outside the decode buffer, by possibly moving + // the sample area back into the scanline, and putting zeros weights first. + // we start on the right edge and check until we're well past the possible + // clip area (2*widest). + { + stbir__contributors * contribs = contributors + num_contributors - 1; + float * coeffs = coefficents + widest * ( num_contributors - 1 ); + + // go until no chance of clipping (this is usually less than 8 lops) + while ( ( contribs >= contributors ) && ( ( contribs->n0 + widest*2 ) >= row_end ) ) + { + // might we clip?? + if ( ( contribs->n0 + widest ) > row_end ) + { + int stop_range = widest; + + // if range is larger than 12, it will be handled by generic loops that can terminate on the exact length + // of this contrib n1, instead of a fixed widest amount - so calculate this + if ( widest > 12 ) + { + int mod; + + // how far will be read in the n_coeff loop (which depends on the widest count mod4); + mod = widest & 3; + stop_range = ( ( ( contribs->n1 - contribs->n0 + 1 ) - mod + 3 ) & ~3 ) + mod; + + // the n_coeff loops do a minimum amount of coeffs, so factor that in! + if ( stop_range < ( 8 + mod ) ) stop_range = 8 + mod; + } + + // now see if we still clip with the refined range + if ( ( contribs->n0 + stop_range ) > row_end ) + { + int new_n0 = row_end - stop_range; + int num = contribs->n1 - contribs->n0 + 1; + int backup = contribs->n0 - new_n0; + float * from_co = coeffs + num - 1; + float * to_co = from_co + backup; + + STBIR_ASSERT( ( new_n0 >= row0 ) && ( new_n0 < contribs->n0 ) ); + + // move the coeffs over + while( num ) + { + *to_co-- = *from_co--; + --num; + } + // zero new positions + while ( to_co >= coeffs ) + *to_co-- = 0; + // set new start point + contribs->n0 = new_n0; + if ( widest > 12 ) + { + int mod; + + // how far will be read in the n_coeff loop (which depends on the widest count mod4); + mod = widest & 3; + stop_range = ( ( ( contribs->n1 - contribs->n0 + 1 ) - mod + 3 ) & ~3 ) + mod; + + // the n_coeff loops do a minimum amount of coeffs, so factor that in! + if ( stop_range < ( 8 + mod ) ) stop_range = 8 + mod; + } + } + } + --contribs; + coeffs -= widest; + } + } + + return widest; + #undef STBIR_MOVE_1 + #undef STBIR_MOVE_2 + #undef STBIR_MOVE_4 +} + +static void stbir__calculate_filters( stbir__sampler * samp, stbir__sampler * other_axis_for_pivot, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO ) +{ + int n; + float scale = samp->scale_info.scale; + stbir__kernel_callback * kernel = samp->filter_kernel; + stbir__support_callback * support = samp->filter_support; + float inv_scale = samp->scale_info.inv_scale; + int input_full_size = samp->scale_info.input_full_size; + int gather_num_contributors = samp->num_contributors; + stbir__contributors* gather_contributors = samp->contributors; + float * gather_coeffs = samp->coefficients; + int gather_coefficient_width = samp->coefficient_width; + + switch ( samp->is_gather ) + { + case 1: // gather upsample + { + float out_pixels_radius = support(inv_scale,user_data) * scale; + + stbir__calculate_coefficients_for_gather_upsample( out_pixels_radius, kernel, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width, samp->edge, user_data ); + + STBIR_PROFILE_BUILD_START( cleanup ); + stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width ); + STBIR_PROFILE_BUILD_END( cleanup ); + } + break; + + case 0: // scatter downsample (only on vertical) + case 2: // gather downsample + { + float in_pixels_radius = support(scale,user_data) * inv_scale; + int filter_pixel_margin = samp->filter_pixel_margin; + int input_end = input_full_size + filter_pixel_margin; + + // if this is a scatter, we do a downsample gather to get the coeffs, and then pivot after + if ( !samp->is_gather ) + { + // check if we are using the same gather downsample on the horizontal as this vertical, + // if so, then we don't have to generate them, we can just pivot from the horizontal. + if ( other_axis_for_pivot ) + { + gather_contributors = other_axis_for_pivot->contributors; + gather_coeffs = other_axis_for_pivot->coefficients; + gather_coefficient_width = other_axis_for_pivot->coefficient_width; + gather_num_contributors = other_axis_for_pivot->num_contributors; + samp->extent_info.lowest = other_axis_for_pivot->extent_info.lowest; + samp->extent_info.highest = other_axis_for_pivot->extent_info.highest; + samp->extent_info.widest = other_axis_for_pivot->extent_info.widest; + goto jump_right_to_pivot; + } + + gather_contributors = samp->gather_prescatter_contributors; + gather_coeffs = samp->gather_prescatter_coefficients; + gather_coefficient_width = samp->gather_prescatter_coefficient_width; + gather_num_contributors = samp->gather_prescatter_num_contributors; + } + + stbir__calculate_coefficients_for_gather_downsample( -filter_pixel_margin, input_end, in_pixels_radius, kernel, &samp->scale_info, gather_coefficient_width, gather_num_contributors, gather_contributors, gather_coeffs, user_data ); + + STBIR_PROFILE_BUILD_START( cleanup ); + stbir__cleanup_gathered_coefficients( samp->edge, &samp->extent_info, &samp->scale_info, gather_num_contributors, gather_contributors, gather_coeffs, gather_coefficient_width ); + STBIR_PROFILE_BUILD_END( cleanup ); + + if ( !samp->is_gather ) + { + // if this is a scatter (vertical only), then we need to pivot the coeffs + stbir__contributors * scatter_contributors; + int highest_set; + + jump_right_to_pivot: + + STBIR_PROFILE_BUILD_START( pivot ); + + highest_set = (-filter_pixel_margin) - 1; + for (n = 0; n < gather_num_contributors; n++) + { + int k; + int gn0 = gather_contributors->n0, gn1 = gather_contributors->n1; + int scatter_coefficient_width = samp->coefficient_width; + float * scatter_coeffs = samp->coefficients + ( gn0 + filter_pixel_margin ) * scatter_coefficient_width; + float * g_coeffs = gather_coeffs; + scatter_contributors = samp->contributors + ( gn0 + filter_pixel_margin ); + + for (k = gn0 ; k <= gn1 ; k++ ) + { + float gc = *g_coeffs++; + + // skip zero and denormals - must skip zeros to avoid adding coeffs beyond scatter_coefficient_width + // (which happens when pivoting from horizontal, which might have dummy zeros) + if ( ( ( gc >= stbir__small_float ) || ( gc <= -stbir__small_float ) ) ) + { + if ( ( k > highest_set ) || ( scatter_contributors->n0 > scatter_contributors->n1 ) ) + { + { + // if we are skipping over several contributors, we need to clear the skipped ones + stbir__contributors * clear_contributors = samp->contributors + ( highest_set + filter_pixel_margin + 1); + while ( clear_contributors < scatter_contributors ) + { + clear_contributors->n0 = 0; + clear_contributors->n1 = -1; + ++clear_contributors; + } + } + scatter_contributors->n0 = n; + scatter_contributors->n1 = n; + scatter_coeffs[0] = gc; + highest_set = k; + } + else + { + stbir__insert_coeff( scatter_contributors, scatter_coeffs, n, gc, scatter_coefficient_width ); + } + STBIR_ASSERT( ( scatter_contributors->n1 - scatter_contributors->n0 + 1 ) <= scatter_coefficient_width ); + } + ++scatter_contributors; + scatter_coeffs += scatter_coefficient_width; + } + + ++gather_contributors; + gather_coeffs += gather_coefficient_width; + } + + // now clear any unset contribs + { + stbir__contributors * clear_contributors = samp->contributors + ( highest_set + filter_pixel_margin + 1); + stbir__contributors * end_contributors = samp->contributors + samp->num_contributors; + while ( clear_contributors < end_contributors ) + { + clear_contributors->n0 = 0; + clear_contributors->n1 = -1; + ++clear_contributors; + } + } + + STBIR_PROFILE_BUILD_END( pivot ); + } + } + break; + } +} + + +//======================================================================================================== +// scanline decoders and encoders + +#define stbir__coder_min_num 1 +#define STB_IMAGE_RESIZE_DO_CODERS +#include STBIR__HEADER_FILENAME + +#define stbir__decode_suffix BGRA +#define stbir__decode_swizzle +#define stbir__decode_order0 2 +#define stbir__decode_order1 1 +#define stbir__decode_order2 0 +#define stbir__decode_order3 3 +#define stbir__encode_order0 2 +#define stbir__encode_order1 1 +#define stbir__encode_order2 0 +#define stbir__encode_order3 3 +#define stbir__coder_min_num 4 +#define STB_IMAGE_RESIZE_DO_CODERS +#include STBIR__HEADER_FILENAME + +#define stbir__decode_suffix ARGB +#define stbir__decode_swizzle +#define stbir__decode_order0 1 +#define stbir__decode_order1 2 +#define stbir__decode_order2 3 +#define stbir__decode_order3 0 +#define stbir__encode_order0 3 +#define stbir__encode_order1 0 +#define stbir__encode_order2 1 +#define stbir__encode_order3 2 +#define stbir__coder_min_num 4 +#define STB_IMAGE_RESIZE_DO_CODERS +#include STBIR__HEADER_FILENAME + +#define stbir__decode_suffix ABGR +#define stbir__decode_swizzle +#define stbir__decode_order0 3 +#define stbir__decode_order1 2 +#define stbir__decode_order2 1 +#define stbir__decode_order3 0 +#define stbir__encode_order0 3 +#define stbir__encode_order1 2 +#define stbir__encode_order2 1 +#define stbir__encode_order3 0 +#define stbir__coder_min_num 4 +#define STB_IMAGE_RESIZE_DO_CODERS +#include STBIR__HEADER_FILENAME + +#define stbir__decode_suffix AR +#define stbir__decode_swizzle +#define stbir__decode_order0 1 +#define stbir__decode_order1 0 +#define stbir__decode_order2 3 +#define stbir__decode_order3 2 +#define stbir__encode_order0 1 +#define stbir__encode_order1 0 +#define stbir__encode_order2 3 +#define stbir__encode_order3 2 +#define stbir__coder_min_num 2 +#define STB_IMAGE_RESIZE_DO_CODERS +#include STBIR__HEADER_FILENAME + + +// fancy alpha means we expand to keep both premultipied and non-premultiplied color channels +static void stbir__fancy_alpha_weight_4ch( float * out_buffer, int width_times_channels ) +{ + float STBIR_STREAMOUT_PTR(*) out = out_buffer; + float const * end_decode = out_buffer + ( width_times_channels / 4 ) * 7; // decode buffer aligned to end of out_buffer + float STBIR_STREAMOUT_PTR(*) decode = (float*)end_decode - width_times_channels; + + // fancy alpha is stored internally as R G B A Rpm Gpm Bpm + + #ifdef STBIR_SIMD + + #ifdef STBIR_SIMD8 + decode += 16; + STBIR_NO_UNROLL_LOOP_START + while ( decode <= end_decode ) + { + stbir__simdf8 d0,d1,a0,a1,p0,p1; + STBIR_NO_UNROLL(decode); + stbir__simdf8_load( d0, decode-16 ); + stbir__simdf8_load( d1, decode-16+8 ); + stbir__simdf8_0123to33333333( a0, d0 ); + stbir__simdf8_0123to33333333( a1, d1 ); + stbir__simdf8_mult( p0, a0, d0 ); + stbir__simdf8_mult( p1, a1, d1 ); + stbir__simdf8_bot4s( a0, d0, p0 ); + stbir__simdf8_bot4s( a1, d1, p1 ); + stbir__simdf8_top4s( d0, d0, p0 ); + stbir__simdf8_top4s( d1, d1, p1 ); + stbir__simdf8_store ( out, a0 ); + stbir__simdf8_store ( out+7, d0 ); + stbir__simdf8_store ( out+14, a1 ); + stbir__simdf8_store ( out+21, d1 ); + decode += 16; + out += 28; + } + decode -= 16; + #else + decode += 8; + STBIR_NO_UNROLL_LOOP_START + while ( decode <= end_decode ) + { + stbir__simdf d0,a0,d1,a1,p0,p1; + STBIR_NO_UNROLL(decode); + stbir__simdf_load( d0, decode-8 ); + stbir__simdf_load( d1, decode-8+4 ); + stbir__simdf_0123to3333( a0, d0 ); + stbir__simdf_0123to3333( a1, d1 ); + stbir__simdf_mult( p0, a0, d0 ); + stbir__simdf_mult( p1, a1, d1 ); + stbir__simdf_store ( out, d0 ); + stbir__simdf_store ( out+4, p0 ); + stbir__simdf_store ( out+7, d1 ); + stbir__simdf_store ( out+7+4, p1 ); + decode += 8; + out += 14; + } + decode -= 8; + #endif + + // might be one last odd pixel + #ifdef STBIR_SIMD8 + STBIR_NO_UNROLL_LOOP_START + while ( decode < end_decode ) + #else + if ( decode < end_decode ) + #endif + { + stbir__simdf d,a,p; + STBIR_NO_UNROLL(decode); + stbir__simdf_load( d, decode ); + stbir__simdf_0123to3333( a, d ); + stbir__simdf_mult( p, a, d ); + stbir__simdf_store ( out, d ); + stbir__simdf_store ( out+4, p ); + decode += 4; + out += 7; + } + + #else + + while( decode < end_decode ) + { + float r = decode[0], g = decode[1], b = decode[2], alpha = decode[3]; + out[0] = r; + out[1] = g; + out[2] = b; + out[3] = alpha; + out[4] = r * alpha; + out[5] = g * alpha; + out[6] = b * alpha; + out += 7; + decode += 4; + } + + #endif +} + +static void stbir__fancy_alpha_weight_2ch( float * out_buffer, int width_times_channels ) +{ + float STBIR_STREAMOUT_PTR(*) out = out_buffer; + float const * end_decode = out_buffer + ( width_times_channels / 2 ) * 3; + float STBIR_STREAMOUT_PTR(*) decode = (float*)end_decode - width_times_channels; + + // for fancy alpha, turns into: [X A Xpm][X A Xpm],etc + + #ifdef STBIR_SIMD + + decode += 8; + if ( decode <= end_decode ) + { + STBIR_NO_UNROLL_LOOP_START + do { + #ifdef STBIR_SIMD8 + stbir__simdf8 d0,a0,p0; + STBIR_NO_UNROLL(decode); + stbir__simdf8_load( d0, decode-8 ); + stbir__simdf8_0123to11331133( p0, d0 ); + stbir__simdf8_0123to00220022( a0, d0 ); + stbir__simdf8_mult( p0, p0, a0 ); + + stbir__simdf_store2( out, stbir__if_simdf8_cast_to_simdf4( d0 ) ); + stbir__simdf_store( out+2, stbir__if_simdf8_cast_to_simdf4( p0 ) ); + stbir__simdf_store2h( out+3, stbir__if_simdf8_cast_to_simdf4( d0 ) ); + + stbir__simdf_store2( out+6, stbir__simdf8_gettop4( d0 ) ); + stbir__simdf_store( out+8, stbir__simdf8_gettop4( p0 ) ); + stbir__simdf_store2h( out+9, stbir__simdf8_gettop4( d0 ) ); + #else + stbir__simdf d0,a0,d1,a1,p0,p1; + STBIR_NO_UNROLL(decode); + stbir__simdf_load( d0, decode-8 ); + stbir__simdf_load( d1, decode-8+4 ); + stbir__simdf_0123to1133( p0, d0 ); + stbir__simdf_0123to1133( p1, d1 ); + stbir__simdf_0123to0022( a0, d0 ); + stbir__simdf_0123to0022( a1, d1 ); + stbir__simdf_mult( p0, p0, a0 ); + stbir__simdf_mult( p1, p1, a1 ); + + stbir__simdf_store2( out, d0 ); + stbir__simdf_store( out+2, p0 ); + stbir__simdf_store2h( out+3, d0 ); + + stbir__simdf_store2( out+6, d1 ); + stbir__simdf_store( out+8, p1 ); + stbir__simdf_store2h( out+9, d1 ); + #endif + decode += 8; + out += 12; + } while ( decode <= end_decode ); + } + decode -= 8; + #endif + + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode < end_decode ) + { + float x = decode[0], y = decode[1]; + STBIR_SIMD_NO_UNROLL(decode); + out[0] = x; + out[1] = y; + out[2] = x * y; + out += 3; + decode += 2; + } +} + +static void stbir__fancy_alpha_unweight_4ch( float * encode_buffer, int width_times_channels ) +{ + float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; + float STBIR_SIMD_STREAMOUT_PTR(*) input = encode_buffer; + float const * end_output = encode_buffer + width_times_channels; + + // fancy RGBA is stored internally as R G B A Rpm Gpm Bpm + + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float alpha = input[3]; +#ifdef STBIR_SIMD + stbir__simdf i,ia; + STBIR_SIMD_NO_UNROLL(encode); + if ( alpha < stbir__small_float ) + { + stbir__simdf_load( i, input ); + stbir__simdf_store( encode, i ); + } + else + { + stbir__simdf_load1frep4( ia, 1.0f / alpha ); + stbir__simdf_load( i, input+4 ); + stbir__simdf_mult( i, i, ia ); + stbir__simdf_store( encode, i ); + encode[3] = alpha; + } +#else + if ( alpha < stbir__small_float ) + { + encode[0] = input[0]; + encode[1] = input[1]; + encode[2] = input[2]; + } + else + { + float ialpha = 1.0f / alpha; + encode[0] = input[4] * ialpha; + encode[1] = input[5] * ialpha; + encode[2] = input[6] * ialpha; + } + encode[3] = alpha; +#endif + + input += 7; + encode += 4; + } while ( encode < end_output ); +} + +// format: [X A Xpm][X A Xpm] etc +static void stbir__fancy_alpha_unweight_2ch( float * encode_buffer, int width_times_channels ) +{ + float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; + float STBIR_SIMD_STREAMOUT_PTR(*) input = encode_buffer; + float const * end_output = encode_buffer + width_times_channels; + + do { + float alpha = input[1]; + encode[0] = input[0]; + if ( alpha >= stbir__small_float ) + encode[0] = input[2] / alpha; + encode[1] = alpha; + + input += 3; + encode += 2; + } while ( encode < end_output ); +} + +static void stbir__simple_alpha_weight_4ch( float * decode_buffer, int width_times_channels ) +{ + float STBIR_STREAMOUT_PTR(*) decode = decode_buffer; + float const * end_decode = decode_buffer + width_times_channels; + + #ifdef STBIR_SIMD + { + decode += 2 * stbir__simdfX_float_count; + STBIR_NO_UNROLL_LOOP_START + while ( decode <= end_decode ) + { + stbir__simdfX d0,a0,d1,a1; + STBIR_NO_UNROLL(decode); + stbir__simdfX_load( d0, decode-2*stbir__simdfX_float_count ); + stbir__simdfX_load( d1, decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count ); + stbir__simdfX_aaa1( a0, d0, STBIR_onesX ); + stbir__simdfX_aaa1( a1, d1, STBIR_onesX ); + stbir__simdfX_mult( d0, d0, a0 ); + stbir__simdfX_mult( d1, d1, a1 ); + stbir__simdfX_store ( decode-2*stbir__simdfX_float_count, d0 ); + stbir__simdfX_store ( decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count, d1 ); + decode += 2 * stbir__simdfX_float_count; + } + decode -= 2 * stbir__simdfX_float_count; + + // few last pixels remnants + #ifdef STBIR_SIMD8 + STBIR_NO_UNROLL_LOOP_START + while ( decode < end_decode ) + #else + if ( decode < end_decode ) + #endif + { + stbir__simdf d,a; + stbir__simdf_load( d, decode ); + stbir__simdf_aaa1( a, d, STBIR__CONSTF(STBIR_ones) ); + stbir__simdf_mult( d, d, a ); + stbir__simdf_store ( decode, d ); + decode += 4; + } + } + + #else + + while( decode < end_decode ) + { + float alpha = decode[3]; + decode[0] *= alpha; + decode[1] *= alpha; + decode[2] *= alpha; + decode += 4; + } + + #endif +} + +static void stbir__simple_alpha_weight_2ch( float * decode_buffer, int width_times_channels ) +{ + float STBIR_STREAMOUT_PTR(*) decode = decode_buffer; + float const * end_decode = decode_buffer + width_times_channels; + + #ifdef STBIR_SIMD + decode += 2 * stbir__simdfX_float_count; + STBIR_NO_UNROLL_LOOP_START + while ( decode <= end_decode ) + { + stbir__simdfX d0,a0,d1,a1; + STBIR_NO_UNROLL(decode); + stbir__simdfX_load( d0, decode-2*stbir__simdfX_float_count ); + stbir__simdfX_load( d1, decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count ); + stbir__simdfX_a1a1( a0, d0, STBIR_onesX ); + stbir__simdfX_a1a1( a1, d1, STBIR_onesX ); + stbir__simdfX_mult( d0, d0, a0 ); + stbir__simdfX_mult( d1, d1, a1 ); + stbir__simdfX_store ( decode-2*stbir__simdfX_float_count, d0 ); + stbir__simdfX_store ( decode-2*stbir__simdfX_float_count+stbir__simdfX_float_count, d1 ); + decode += 2 * stbir__simdfX_float_count; + } + decode -= 2 * stbir__simdfX_float_count; + #endif + + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode < end_decode ) + { + float alpha = decode[1]; + STBIR_SIMD_NO_UNROLL(decode); + decode[0] *= alpha; + decode += 2; + } +} + +static void stbir__simple_alpha_unweight_4ch( float * encode_buffer, int width_times_channels ) +{ + float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; + float const * end_output = encode_buffer + width_times_channels; + + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float alpha = encode[3]; + +#ifdef STBIR_SIMD + stbir__simdf i,ia; + STBIR_SIMD_NO_UNROLL(encode); + if ( alpha >= stbir__small_float ) + { + stbir__simdf_load1frep4( ia, 1.0f / alpha ); + stbir__simdf_load( i, encode ); + stbir__simdf_mult( i, i, ia ); + stbir__simdf_store( encode, i ); + encode[3] = alpha; + } +#else + if ( alpha >= stbir__small_float ) + { + float ialpha = 1.0f / alpha; + encode[0] *= ialpha; + encode[1] *= ialpha; + encode[2] *= ialpha; + } +#endif + encode += 4; + } while ( encode < end_output ); +} + +static void stbir__simple_alpha_unweight_2ch( float * encode_buffer, int width_times_channels ) +{ + float STBIR_SIMD_STREAMOUT_PTR(*) encode = encode_buffer; + float const * end_output = encode_buffer + width_times_channels; + + do { + float alpha = encode[1]; + if ( alpha >= stbir__small_float ) + encode[0] /= alpha; + encode += 2; + } while ( encode < end_output ); +} + + +// only used in RGB->BGR or BGR->RGB +static void stbir__simple_flip_3ch( float * decode_buffer, int width_times_channels ) +{ + float STBIR_STREAMOUT_PTR(*) decode = decode_buffer; + float const * end_decode = decode_buffer + width_times_channels; + +#ifdef STBIR_SIMD + #ifdef stbir__simdf_swiz2 // do we have two argument swizzles? + end_decode -= 12; + STBIR_NO_UNROLL_LOOP_START + while( decode <= end_decode ) + { + // on arm64 8 instructions, no overlapping stores + stbir__simdf a,b,c,na,nb; + STBIR_SIMD_NO_UNROLL(decode); + stbir__simdf_load( a, decode ); + stbir__simdf_load( b, decode+4 ); + stbir__simdf_load( c, decode+8 ); + + na = stbir__simdf_swiz2( a, b, 2, 1, 0, 5 ); + b = stbir__simdf_swiz2( a, b, 4, 3, 6, 7 ); + nb = stbir__simdf_swiz2( b, c, 0, 1, 4, 3 ); + c = stbir__simdf_swiz2( b, c, 2, 7, 6, 5 ); + + stbir__simdf_store( decode, na ); + stbir__simdf_store( decode+4, nb ); + stbir__simdf_store( decode+8, c ); + decode += 12; + } + end_decode += 12; + #else + end_decode -= 24; + STBIR_NO_UNROLL_LOOP_START + while( decode <= end_decode ) + { + // 26 instructions on x64 + stbir__simdf a,b,c,d,e,f,g; + float i21, i23; + STBIR_SIMD_NO_UNROLL(decode); + stbir__simdf_load( a, decode ); + stbir__simdf_load( b, decode+3 ); + stbir__simdf_load( c, decode+6 ); + stbir__simdf_load( d, decode+9 ); + stbir__simdf_load( e, decode+12 ); + stbir__simdf_load( f, decode+15 ); + stbir__simdf_load( g, decode+18 ); + + a = stbir__simdf_swiz( a, 2, 1, 0, 3 ); + b = stbir__simdf_swiz( b, 2, 1, 0, 3 ); + c = stbir__simdf_swiz( c, 2, 1, 0, 3 ); + d = stbir__simdf_swiz( d, 2, 1, 0, 3 ); + e = stbir__simdf_swiz( e, 2, 1, 0, 3 ); + f = stbir__simdf_swiz( f, 2, 1, 0, 3 ); + g = stbir__simdf_swiz( g, 2, 1, 0, 3 ); + + // stores overlap, need to be in order, + stbir__simdf_store( decode, a ); + i21 = decode[21]; + stbir__simdf_store( decode+3, b ); + i23 = decode[23]; + stbir__simdf_store( decode+6, c ); + stbir__simdf_store( decode+9, d ); + stbir__simdf_store( decode+12, e ); + stbir__simdf_store( decode+15, f ); + stbir__simdf_store( decode+18, g ); + decode[21] = i23; + decode[23] = i21; + decode += 24; + } + end_decode += 24; + #endif +#else + end_decode -= 12; + STBIR_NO_UNROLL_LOOP_START + while( decode <= end_decode ) + { + // 16 instructions + float t0,t1,t2,t3; + STBIR_NO_UNROLL(decode); + t0 = decode[0]; t1 = decode[3]; t2 = decode[6]; t3 = decode[9]; + decode[0] = decode[2]; decode[3] = decode[5]; decode[6] = decode[8]; decode[9] = decode[11]; + decode[2] = t0; decode[5] = t1; decode[8] = t2; decode[11] = t3; + decode += 12; + } + end_decode += 12; +#endif + + STBIR_NO_UNROLL_LOOP_START + while( decode < end_decode ) + { + float t = decode[0]; + STBIR_NO_UNROLL(decode); + decode[0] = decode[2]; + decode[2] = t; + decode += 3; + } +} + + + +static void stbir__decode_scanline(stbir__info const * stbir_info, int n, float * output_buffer STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) +{ + int channels = stbir_info->channels; + int effective_channels = stbir_info->effective_channels; + int input_sample_in_bytes = stbir__type_size[stbir_info->input_type] * channels; + stbir_edge edge_horizontal = stbir_info->horizontal.edge; + stbir_edge edge_vertical = stbir_info->vertical.edge; + int row = stbir__edge_wrap(edge_vertical, n, stbir_info->vertical.scale_info.input_full_size); + const void* input_plane_data = ( (char *) stbir_info->input_data ) + (size_t)row * (size_t) stbir_info->input_stride_bytes; + stbir__span const * spans = stbir_info->scanline_extents.spans; + float* full_decode_buffer = output_buffer - stbir_info->scanline_extents.conservative.n0 * effective_channels; + + // if we are on edge_zero, and we get in here with an out of bounds n, then the calculate filters has failed + STBIR_ASSERT( !(edge_vertical == STBIR_EDGE_ZERO && (n < 0 || n >= stbir_info->vertical.scale_info.input_full_size)) ); + + do + { + float * decode_buffer; + void const * input_data; + float * end_decode; + int width_times_channels; + int width; + + if ( spans->n1 < spans->n0 ) + break; + + width = spans->n1 + 1 - spans->n0; + decode_buffer = full_decode_buffer + spans->n0 * effective_channels; + end_decode = full_decode_buffer + ( spans->n1 + 1 ) * effective_channels; + width_times_channels = width * channels; + + // read directly out of input plane by default + input_data = ( (char*)input_plane_data ) + spans->pixel_offset_for_input * input_sample_in_bytes; + + // if we have an input callback, call it to get the input data + if ( stbir_info->in_pixels_cb ) + { + // call the callback with a temp buffer (that they can choose to use or not). the temp is just right aligned memory in the decode_buffer itself + input_data = stbir_info->in_pixels_cb( ( (char*) end_decode ) - ( width * input_sample_in_bytes ), input_plane_data, width, spans->pixel_offset_for_input, row, stbir_info->user_data ); + } + + STBIR_PROFILE_START( decode ); + // convert the pixels info the float decode_buffer, (we index from end_decode, so that when channelsdecode_pixels( (float*)end_decode - width_times_channels, width_times_channels, input_data ); + STBIR_PROFILE_END( decode ); + + if (stbir_info->alpha_weight) + { + STBIR_PROFILE_START( alpha ); + stbir_info->alpha_weight( decode_buffer, width_times_channels ); + STBIR_PROFILE_END( alpha ); + } + + ++spans; + } while ( spans <= ( &stbir_info->scanline_extents.spans[1] ) ); + + // handle the edge_wrap filter (all other types are handled back out at the calculate_filter stage) + // basically the idea here is that if we have the whole scanline in memory, we don't redecode the + // wrapped edge pixels, and instead just memcpy them from the scanline into the edge positions + if ( ( edge_horizontal == STBIR_EDGE_WRAP ) && ( stbir_info->scanline_extents.edge_sizes[0] | stbir_info->scanline_extents.edge_sizes[1] ) ) + { + // this code only runs if we're in edge_wrap, and we're doing the entire scanline + int e, start_x[2]; + int input_full_size = stbir_info->horizontal.scale_info.input_full_size; + + start_x[0] = -stbir_info->scanline_extents.edge_sizes[0]; // left edge start x + start_x[1] = input_full_size; // right edge + + for( e = 0; e < 2 ; e++ ) + { + // do each margin + int margin = stbir_info->scanline_extents.edge_sizes[e]; + if ( margin ) + { + int x = start_x[e]; + float * marg = full_decode_buffer + x * effective_channels; + float const * src = full_decode_buffer + stbir__edge_wrap(edge_horizontal, x, input_full_size) * effective_channels; + STBIR_MEMCPY( marg, src, margin * effective_channels * sizeof(float) ); + } + } + } +} + + +//================= +// Do 1 channel horizontal routines + +#ifdef STBIR_SIMD + +#define stbir__1_coeff_only() \ + stbir__simdf tot,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1( c, hc ); \ + stbir__simdf_mult1_mem( tot, c, decode ); + +#define stbir__2_coeff_only() \ + stbir__simdf tot,c,d; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2z( c, hc ); \ + stbir__simdf_load2( d, decode ); \ + stbir__simdf_mult( tot, c, d ); \ + stbir__simdf_0123to1230( c, tot ); \ + stbir__simdf_add1( tot, tot, c ); + +#define stbir__3_coeff_only() \ + stbir__simdf tot,c,t; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( c, hc ); \ + stbir__simdf_mult_mem( tot, c, decode ); \ + stbir__simdf_0123to1230( c, tot ); \ + stbir__simdf_0123to2301( t, tot ); \ + stbir__simdf_add1( tot, tot, c ); \ + stbir__simdf_add1( tot, tot, t ); + +#define stbir__store_output_tiny() \ + stbir__simdf_store1( output, tot ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 1; + +#define stbir__4_coeff_start() \ + stbir__simdf tot,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( c, hc ); \ + stbir__simdf_mult_mem( tot, c, decode ); \ + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( c, hc + (ofs) ); \ + stbir__simdf_madd_mem( tot, tot, c, decode+(ofs) ); + +#define stbir__1_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + stbir__simdf_load1z( c, hc + (ofs) ); \ + stbir__simdf_load1( d, decode + (ofs) ); \ + stbir__simdf_madd( tot, tot, d, c ); } + +#define stbir__2_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + stbir__simdf_load2z( c, hc+(ofs) ); \ + stbir__simdf_load2( d, decode+(ofs) ); \ + stbir__simdf_madd( tot, tot, d, c ); } + +#define stbir__3_coeff_setup() \ + stbir__simdf mask; \ + stbir__simdf_load( mask, STBIR_mask + 3 ); + +#define stbir__3_coeff_remnant( ofs ) \ + stbir__simdf_load( c, hc+(ofs) ); \ + stbir__simdf_and( c, c, mask ); \ + stbir__simdf_madd_mem( tot, tot, c, decode+(ofs) ); + +#define stbir__store_output() \ + stbir__simdf_0123to2301( c, tot ); \ + stbir__simdf_add( tot, tot, c ); \ + stbir__simdf_0123to1230( c, tot ); \ + stbir__simdf_add1( tot, tot, c ); \ + stbir__simdf_store1( output, tot ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 1; + +#else + +#define stbir__1_coeff_only() \ + float tot; \ + tot = decode[0]*hc[0]; + +#define stbir__2_coeff_only() \ + float tot; \ + tot = decode[0] * hc[0]; \ + tot += decode[1] * hc[1]; + +#define stbir__3_coeff_only() \ + float tot; \ + tot = decode[0] * hc[0]; \ + tot += decode[1] * hc[1]; \ + tot += decode[2] * hc[2]; + +#define stbir__store_output_tiny() \ + output[0] = tot; \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 1; + +#define stbir__4_coeff_start() \ + float tot0,tot1,tot2,tot3; \ + tot0 = decode[0] * hc[0]; \ + tot1 = decode[1] * hc[1]; \ + tot2 = decode[2] * hc[2]; \ + tot3 = decode[3] * hc[3]; + +#define stbir__4_coeff_continue_from_4( ofs ) \ + tot0 += decode[0+(ofs)] * hc[0+(ofs)]; \ + tot1 += decode[1+(ofs)] * hc[1+(ofs)]; \ + tot2 += decode[2+(ofs)] * hc[2+(ofs)]; \ + tot3 += decode[3+(ofs)] * hc[3+(ofs)]; + +#define stbir__1_coeff_remnant( ofs ) \ + tot0 += decode[0+(ofs)] * hc[0+(ofs)]; + +#define stbir__2_coeff_remnant( ofs ) \ + tot0 += decode[0+(ofs)] * hc[0+(ofs)]; \ + tot1 += decode[1+(ofs)] * hc[1+(ofs)]; \ + +#define stbir__3_coeff_remnant( ofs ) \ + tot0 += decode[0+(ofs)] * hc[0+(ofs)]; \ + tot1 += decode[1+(ofs)] * hc[1+(ofs)]; \ + tot2 += decode[2+(ofs)] * hc[2+(ofs)]; + +#define stbir__store_output() \ + output[0] = (tot0+tot2)+(tot1+tot3); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 1; + +#endif + +#define STBIR__horizontal_channels 1 +#define STB_IMAGE_RESIZE_DO_HORIZONTALS +#include STBIR__HEADER_FILENAME + + +//================= +// Do 2 channel horizontal routines + +#ifdef STBIR_SIMD + +#define stbir__1_coeff_only() \ + stbir__simdf tot,c,d; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1z( c, hc ); \ + stbir__simdf_0123to0011( c, c ); \ + stbir__simdf_load2( d, decode ); \ + stbir__simdf_mult( tot, d, c ); + +#define stbir__2_coeff_only() \ + stbir__simdf tot,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2( c, hc ); \ + stbir__simdf_0123to0011( c, c ); \ + stbir__simdf_mult_mem( tot, c, decode ); + +#define stbir__3_coeff_only() \ + stbir__simdf tot,c,cs,d; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0011( c, cs ); \ + stbir__simdf_mult_mem( tot, c, decode ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_load2z( d, decode+4 ); \ + stbir__simdf_madd( tot, tot, d, c ); + +#define stbir__store_output_tiny() \ + stbir__simdf_0123to2301( c, tot ); \ + stbir__simdf_add( tot, tot, c ); \ + stbir__simdf_store2( output, tot ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 2; + +#ifdef STBIR_SIMD8 + +#define stbir__4_coeff_start() \ + stbir__simdf8 tot0,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc ); \ + stbir__simdf8_0123to00112233( c, cs ); \ + stbir__simdf8_mult_mem( tot0, c, decode ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00112233( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*2 ); + +#define stbir__1_coeff_remnant( ofs ) \ + { stbir__simdf t,d; \ + stbir__simdf_load1z( t, hc + (ofs) ); \ + stbir__simdf_load2( d, decode + (ofs) * 2 ); \ + stbir__simdf_0123to0011( t, t ); \ + stbir__simdf_mult( t, t, d ); \ + stbir__simdf8_add4( tot0, tot0, t ); } + +#define stbir__2_coeff_remnant( ofs ) \ + { stbir__simdf t; \ + stbir__simdf_load2( t, hc + (ofs) ); \ + stbir__simdf_0123to0011( t, t ); \ + stbir__simdf_mult_mem( t, t, decode+(ofs)*2 ); \ + stbir__simdf8_add4( tot0, tot0, t ); } + +#define stbir__3_coeff_remnant( ofs ) \ + { stbir__simdf8 d; \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00112233( c, cs ); \ + stbir__simdf8_load6z( d, decode+(ofs)*2 ); \ + stbir__simdf8_madd( tot0, tot0, c, d ); } + +#define stbir__store_output() \ + { stbir__simdf t,d; \ + stbir__simdf8_add4halves( t, stbir__if_simdf8_cast_to_simdf4(tot0), tot0 ); \ + stbir__simdf_0123to2301( d, t ); \ + stbir__simdf_add( t, t, d ); \ + stbir__simdf_store2( output, t ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 2; } + +#else + +#define stbir__4_coeff_start() \ + stbir__simdf tot0,tot1,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0011( c, cs ); \ + stbir__simdf_mult_mem( tot0, c, decode ); \ + stbir__simdf_0123to2233( c, cs ); \ + stbir__simdf_mult_mem( tot1, c, decode+4 ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0011( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*2 ); \ + stbir__simdf_0123to2233( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*2+4 ); + +#define stbir__1_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + stbir__simdf_load1z( cs, hc + (ofs) ); \ + stbir__simdf_0123to0011( c, cs ); \ + stbir__simdf_load2( d, decode + (ofs) * 2 ); \ + stbir__simdf_madd( tot0, tot0, d, c ); } + +#define stbir__2_coeff_remnant( ofs ) \ + stbir__simdf_load2( cs, hc + (ofs) ); \ + stbir__simdf_0123to0011( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*2 ); + +#define stbir__3_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0011( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*2 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_load2z( d, decode + (ofs) * 2 + 4 ); \ + stbir__simdf_madd( tot1, tot1, d, c ); } + +#define stbir__store_output() \ + stbir__simdf_add( tot0, tot0, tot1 ); \ + stbir__simdf_0123to2301( c, tot0 ); \ + stbir__simdf_add( tot0, tot0, c ); \ + stbir__simdf_store2( output, tot0 ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 2; + +#endif + +#else + +#define stbir__1_coeff_only() \ + float tota,totb,c; \ + c = hc[0]; \ + tota = decode[0]*c; \ + totb = decode[1]*c; + +#define stbir__2_coeff_only() \ + float tota,totb,c; \ + c = hc[0]; \ + tota = decode[0]*c; \ + totb = decode[1]*c; \ + c = hc[1]; \ + tota += decode[2]*c; \ + totb += decode[3]*c; + +// this weird order of add matches the simd +#define stbir__3_coeff_only() \ + float tota,totb,c; \ + c = hc[0]; \ + tota = decode[0]*c; \ + totb = decode[1]*c; \ + c = hc[2]; \ + tota += decode[4]*c; \ + totb += decode[5]*c; \ + c = hc[1]; \ + tota += decode[2]*c; \ + totb += decode[3]*c; + +#define stbir__store_output_tiny() \ + output[0] = tota; \ + output[1] = totb; \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 2; + +#define stbir__4_coeff_start() \ + float tota0,tota1,tota2,tota3,totb0,totb1,totb2,totb3,c; \ + c = hc[0]; \ + tota0 = decode[0]*c; \ + totb0 = decode[1]*c; \ + c = hc[1]; \ + tota1 = decode[2]*c; \ + totb1 = decode[3]*c; \ + c = hc[2]; \ + tota2 = decode[4]*c; \ + totb2 = decode[5]*c; \ + c = hc[3]; \ + tota3 = decode[6]*c; \ + totb3 = decode[7]*c; + +#define stbir__4_coeff_continue_from_4( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*2]*c; \ + totb0 += decode[1+(ofs)*2]*c; \ + c = hc[1+(ofs)]; \ + tota1 += decode[2+(ofs)*2]*c; \ + totb1 += decode[3+(ofs)*2]*c; \ + c = hc[2+(ofs)]; \ + tota2 += decode[4+(ofs)*2]*c; \ + totb2 += decode[5+(ofs)*2]*c; \ + c = hc[3+(ofs)]; \ + tota3 += decode[6+(ofs)*2]*c; \ + totb3 += decode[7+(ofs)*2]*c; + +#define stbir__1_coeff_remnant( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*2] * c; \ + totb0 += decode[1+(ofs)*2] * c; + +#define stbir__2_coeff_remnant( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*2] * c; \ + totb0 += decode[1+(ofs)*2] * c; \ + c = hc[1+(ofs)]; \ + tota1 += decode[2+(ofs)*2] * c; \ + totb1 += decode[3+(ofs)*2] * c; + +#define stbir__3_coeff_remnant( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*2] * c; \ + totb0 += decode[1+(ofs)*2] * c; \ + c = hc[1+(ofs)]; \ + tota1 += decode[2+(ofs)*2] * c; \ + totb1 += decode[3+(ofs)*2] * c; \ + c = hc[2+(ofs)]; \ + tota2 += decode[4+(ofs)*2] * c; \ + totb2 += decode[5+(ofs)*2] * c; + +#define stbir__store_output() \ + output[0] = (tota0+tota2)+(tota1+tota3); \ + output[1] = (totb0+totb2)+(totb1+totb3); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 2; + +#endif + +#define STBIR__horizontal_channels 2 +#define STB_IMAGE_RESIZE_DO_HORIZONTALS +#include STBIR__HEADER_FILENAME + + +//================= +// Do 3 channel horizontal routines + +#ifdef STBIR_SIMD + +#define stbir__1_coeff_only() \ + stbir__simdf tot,c,d; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1z( c, hc ); \ + stbir__simdf_0123to0001( c, c ); \ + stbir__simdf_load( d, decode ); \ + stbir__simdf_mult( tot, d, c ); + +#define stbir__2_coeff_only() \ + stbir__simdf tot,c,cs,d; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_load( d, decode ); \ + stbir__simdf_mult( tot, d, c ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_load( d, decode+3 ); \ + stbir__simdf_madd( tot, tot, d, c ); + +#define stbir__3_coeff_only() \ + stbir__simdf tot,c,d,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_load( d, decode ); \ + stbir__simdf_mult( tot, d, c ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_load( d, decode+3 ); \ + stbir__simdf_madd( tot, tot, d, c ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_load( d, decode+6 ); \ + stbir__simdf_madd( tot, tot, d, c ); + +#define stbir__store_output_tiny() \ + stbir__simdf_store2( output, tot ); \ + stbir__simdf_0123to2301( tot, tot ); \ + stbir__simdf_store1( output+2, tot ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 3; + +#ifdef STBIR_SIMD8 + +// we're loading from the XXXYYY decode by -1 to get the XXXYYY into different halves of the AVX reg fyi +#define stbir__4_coeff_start() \ + stbir__simdf8 tot0,tot1,c,cs; stbir__simdf t; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc ); \ + stbir__simdf8_0123to00001111( c, cs ); \ + stbir__simdf8_mult_mem( tot0, c, decode - 1 ); \ + stbir__simdf8_0123to22223333( c, cs ); \ + stbir__simdf8_mult_mem( tot1, c, decode+6 - 1 ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00001111( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*3 - 1 ); \ + stbir__simdf8_0123to22223333( c, cs ); \ + stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*3 + 6 - 1 ); + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1rep4( t, hc + (ofs) ); \ + stbir__simdf8_madd_mem4( tot0, tot0, t, decode+(ofs)*3 - 1 ); + +#define stbir__2_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) - 2 ); \ + stbir__simdf8_0123to22223333( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*3 - 1 ); + + #define stbir__3_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00001111( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*3 - 1 ); \ + stbir__simdf8_0123to2222( t, cs ); \ + stbir__simdf8_madd_mem4( tot1, tot1, t, decode+(ofs)*3 + 6 - 1 ); + +#define stbir__store_output() \ + stbir__simdf8_add( tot0, tot0, tot1 ); \ + stbir__simdf_0123to1230( t, stbir__if_simdf8_cast_to_simdf4( tot0 ) ); \ + stbir__simdf8_add4halves( t, t, tot0 ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 3; \ + if ( output < output_end ) \ + { \ + stbir__simdf_store( output-3, t ); \ + continue; \ + } \ + { stbir__simdf tt; stbir__simdf_0123to2301( tt, t ); \ + stbir__simdf_store2( output-3, t ); \ + stbir__simdf_store1( output+2-3, tt ); } \ + break; + + +#else + +#define stbir__4_coeff_start() \ + stbir__simdf tot0,tot1,tot2,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0001( c, cs ); \ + stbir__simdf_mult_mem( tot0, c, decode ); \ + stbir__simdf_0123to1122( c, cs ); \ + stbir__simdf_mult_mem( tot1, c, decode+4 ); \ + stbir__simdf_0123to2333( c, cs ); \ + stbir__simdf_mult_mem( tot2, c, decode+8 ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0001( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 ); \ + stbir__simdf_0123to1122( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*3+4 ); \ + stbir__simdf_0123to2333( c, cs ); \ + stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*3+8 ); + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1z( c, hc + (ofs) ); \ + stbir__simdf_0123to0001( c, c ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 ); + +#define stbir__2_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2z( cs, hc + (ofs) ); \ + stbir__simdf_0123to0001( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 ); \ + stbir__simdf_0123to1122( c, cs ); \ + stbir__simdf_load2z( d, decode+(ofs)*3+4 ); \ + stbir__simdf_madd( tot1, tot1, c, d ); } + +#define stbir__3_coeff_remnant( ofs ) \ + { stbir__simdf d; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0001( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*3 ); \ + stbir__simdf_0123to1122( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*3+4 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_load1z( d, decode+(ofs)*3+8 ); \ + stbir__simdf_madd( tot2, tot2, c, d ); } + +#define stbir__store_output() \ + stbir__simdf_0123ABCDto3ABx( c, tot0, tot1 ); \ + stbir__simdf_0123ABCDto23Ax( cs, tot1, tot2 ); \ + stbir__simdf_0123to1230( tot2, tot2 ); \ + stbir__simdf_add( tot0, tot0, cs ); \ + stbir__simdf_add( c, c, tot2 ); \ + stbir__simdf_add( tot0, tot0, c ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 3; \ + if ( output < output_end ) \ + { \ + stbir__simdf_store( output-3, tot0 ); \ + continue; \ + } \ + stbir__simdf_0123to2301( tot1, tot0 ); \ + stbir__simdf_store2( output-3, tot0 ); \ + stbir__simdf_store1( output+2-3, tot1 ); \ + break; + +#endif + +#else + +#define stbir__1_coeff_only() \ + float tot0, tot1, tot2, c; \ + c = hc[0]; \ + tot0 = decode[0]*c; \ + tot1 = decode[1]*c; \ + tot2 = decode[2]*c; + +#define stbir__2_coeff_only() \ + float tot0, tot1, tot2, c; \ + c = hc[0]; \ + tot0 = decode[0]*c; \ + tot1 = decode[1]*c; \ + tot2 = decode[2]*c; \ + c = hc[1]; \ + tot0 += decode[3]*c; \ + tot1 += decode[4]*c; \ + tot2 += decode[5]*c; + +#define stbir__3_coeff_only() \ + float tot0, tot1, tot2, c; \ + c = hc[0]; \ + tot0 = decode[0]*c; \ + tot1 = decode[1]*c; \ + tot2 = decode[2]*c; \ + c = hc[1]; \ + tot0 += decode[3]*c; \ + tot1 += decode[4]*c; \ + tot2 += decode[5]*c; \ + c = hc[2]; \ + tot0 += decode[6]*c; \ + tot1 += decode[7]*c; \ + tot2 += decode[8]*c; + +#define stbir__store_output_tiny() \ + output[0] = tot0; \ + output[1] = tot1; \ + output[2] = tot2; \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 3; + +#define stbir__4_coeff_start() \ + float tota0,tota1,tota2,totb0,totb1,totb2,totc0,totc1,totc2,totd0,totd1,totd2,c; \ + c = hc[0]; \ + tota0 = decode[0]*c; \ + tota1 = decode[1]*c; \ + tota2 = decode[2]*c; \ + c = hc[1]; \ + totb0 = decode[3]*c; \ + totb1 = decode[4]*c; \ + totb2 = decode[5]*c; \ + c = hc[2]; \ + totc0 = decode[6]*c; \ + totc1 = decode[7]*c; \ + totc2 = decode[8]*c; \ + c = hc[3]; \ + totd0 = decode[9]*c; \ + totd1 = decode[10]*c; \ + totd2 = decode[11]*c; + +#define stbir__4_coeff_continue_from_4( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*3]*c; \ + tota1 += decode[1+(ofs)*3]*c; \ + tota2 += decode[2+(ofs)*3]*c; \ + c = hc[1+(ofs)]; \ + totb0 += decode[3+(ofs)*3]*c; \ + totb1 += decode[4+(ofs)*3]*c; \ + totb2 += decode[5+(ofs)*3]*c; \ + c = hc[2+(ofs)]; \ + totc0 += decode[6+(ofs)*3]*c; \ + totc1 += decode[7+(ofs)*3]*c; \ + totc2 += decode[8+(ofs)*3]*c; \ + c = hc[3+(ofs)]; \ + totd0 += decode[9+(ofs)*3]*c; \ + totd1 += decode[10+(ofs)*3]*c; \ + totd2 += decode[11+(ofs)*3]*c; + +#define stbir__1_coeff_remnant( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*3]*c; \ + tota1 += decode[1+(ofs)*3]*c; \ + tota2 += decode[2+(ofs)*3]*c; + +#define stbir__2_coeff_remnant( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*3]*c; \ + tota1 += decode[1+(ofs)*3]*c; \ + tota2 += decode[2+(ofs)*3]*c; \ + c = hc[1+(ofs)]; \ + totb0 += decode[3+(ofs)*3]*c; \ + totb1 += decode[4+(ofs)*3]*c; \ + totb2 += decode[5+(ofs)*3]*c; \ + +#define stbir__3_coeff_remnant( ofs ) \ + c = hc[0+(ofs)]; \ + tota0 += decode[0+(ofs)*3]*c; \ + tota1 += decode[1+(ofs)*3]*c; \ + tota2 += decode[2+(ofs)*3]*c; \ + c = hc[1+(ofs)]; \ + totb0 += decode[3+(ofs)*3]*c; \ + totb1 += decode[4+(ofs)*3]*c; \ + totb2 += decode[5+(ofs)*3]*c; \ + c = hc[2+(ofs)]; \ + totc0 += decode[6+(ofs)*3]*c; \ + totc1 += decode[7+(ofs)*3]*c; \ + totc2 += decode[8+(ofs)*3]*c; + +#define stbir__store_output() \ + output[0] = (tota0+totc0)+(totb0+totd0); \ + output[1] = (tota1+totc1)+(totb1+totd1); \ + output[2] = (tota2+totc2)+(totb2+totd2); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 3; + +#endif + +#define STBIR__horizontal_channels 3 +#define STB_IMAGE_RESIZE_DO_HORIZONTALS +#include STBIR__HEADER_FILENAME + +//================= +// Do 4 channel horizontal routines + +#ifdef STBIR_SIMD + +#define stbir__1_coeff_only() \ + stbir__simdf tot,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1( c, hc ); \ + stbir__simdf_0123to0000( c, c ); \ + stbir__simdf_mult_mem( tot, c, decode ); + +#define stbir__2_coeff_only() \ + stbir__simdf tot,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_mult_mem( tot, c, decode ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot, tot, c, decode+4 ); + +#define stbir__3_coeff_only() \ + stbir__simdf tot,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_mult_mem( tot, c, decode ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot, tot, c, decode+4 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot, tot, c, decode+8 ); + +#define stbir__store_output_tiny() \ + stbir__simdf_store( output, tot ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 4; + +#ifdef STBIR_SIMD8 + +#define stbir__4_coeff_start() \ + stbir__simdf8 tot0,c,cs; stbir__simdf t; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc ); \ + stbir__simdf8_0123to00001111( c, cs ); \ + stbir__simdf8_mult_mem( tot0, c, decode ); \ + stbir__simdf8_0123to22223333( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+8 ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00001111( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \ + stbir__simdf8_0123to22223333( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4+8 ); + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1rep4( t, hc + (ofs) ); \ + stbir__simdf8_madd_mem4( tot0, tot0, t, decode+(ofs)*4 ); + +#define stbir__2_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) - 2 ); \ + stbir__simdf8_0123to22223333( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); + + #define stbir__3_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00001111( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \ + stbir__simdf8_0123to2222( t, cs ); \ + stbir__simdf8_madd_mem4( tot0, tot0, t, decode+(ofs)*4+8 ); + +#define stbir__store_output() \ + stbir__simdf8_add4halves( t, stbir__if_simdf8_cast_to_simdf4(tot0), tot0 ); \ + stbir__simdf_store( output, t ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 4; + +#else + +#define stbir__4_coeff_start() \ + stbir__simdf tot0,tot1,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_mult_mem( tot0, c, decode ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_mult_mem( tot1, c, decode+4 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+8 ); \ + stbir__simdf_0123to3333( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+12 ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+4 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4+8 ); \ + stbir__simdf_0123to3333( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+12 ); + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1( c, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, c ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); + +#define stbir__2_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2( cs, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+4 ); + +#define stbir__3_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*4+4 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*4+8 ); + +#define stbir__store_output() \ + stbir__simdf_add( tot0, tot0, tot1 ); \ + stbir__simdf_store( output, tot0 ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 4; + +#endif + +#else + +#define stbir__1_coeff_only() \ + float p0,p1,p2,p3,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0]; \ + p0 = decode[0] * c; \ + p1 = decode[1] * c; \ + p2 = decode[2] * c; \ + p3 = decode[3] * c; + +#define stbir__2_coeff_only() \ + float p0,p1,p2,p3,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0]; \ + p0 = decode[0] * c; \ + p1 = decode[1] * c; \ + p2 = decode[2] * c; \ + p3 = decode[3] * c; \ + c = hc[1]; \ + p0 += decode[4] * c; \ + p1 += decode[5] * c; \ + p2 += decode[6] * c; \ + p3 += decode[7] * c; + +#define stbir__3_coeff_only() \ + float p0,p1,p2,p3,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0]; \ + p0 = decode[0] * c; \ + p1 = decode[1] * c; \ + p2 = decode[2] * c; \ + p3 = decode[3] * c; \ + c = hc[1]; \ + p0 += decode[4] * c; \ + p1 += decode[5] * c; \ + p2 += decode[6] * c; \ + p3 += decode[7] * c; \ + c = hc[2]; \ + p0 += decode[8] * c; \ + p1 += decode[9] * c; \ + p2 += decode[10] * c; \ + p3 += decode[11] * c; + +#define stbir__store_output_tiny() \ + output[0] = p0; \ + output[1] = p1; \ + output[2] = p2; \ + output[3] = p3; \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 4; + +#define stbir__4_coeff_start() \ + float x0,x1,x2,x3,y0,y1,y2,y3,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0]; \ + x0 = decode[0] * c; \ + x1 = decode[1] * c; \ + x2 = decode[2] * c; \ + x3 = decode[3] * c; \ + c = hc[1]; \ + y0 = decode[4] * c; \ + y1 = decode[5] * c; \ + y2 = decode[6] * c; \ + y3 = decode[7] * c; \ + c = hc[2]; \ + x0 += decode[8] * c; \ + x1 += decode[9] * c; \ + x2 += decode[10] * c; \ + x3 += decode[11] * c; \ + c = hc[3]; \ + y0 += decode[12] * c; \ + y1 += decode[13] * c; \ + y2 += decode[14] * c; \ + y3 += decode[15] * c; + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*4] * c; \ + x1 += decode[1+(ofs)*4] * c; \ + x2 += decode[2+(ofs)*4] * c; \ + x3 += decode[3+(ofs)*4] * c; \ + c = hc[1+(ofs)]; \ + y0 += decode[4+(ofs)*4] * c; \ + y1 += decode[5+(ofs)*4] * c; \ + y2 += decode[6+(ofs)*4] * c; \ + y3 += decode[7+(ofs)*4] * c; \ + c = hc[2+(ofs)]; \ + x0 += decode[8+(ofs)*4] * c; \ + x1 += decode[9+(ofs)*4] * c; \ + x2 += decode[10+(ofs)*4] * c; \ + x3 += decode[11+(ofs)*4] * c; \ + c = hc[3+(ofs)]; \ + y0 += decode[12+(ofs)*4] * c; \ + y1 += decode[13+(ofs)*4] * c; \ + y2 += decode[14+(ofs)*4] * c; \ + y3 += decode[15+(ofs)*4] * c; + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*4] * c; \ + x1 += decode[1+(ofs)*4] * c; \ + x2 += decode[2+(ofs)*4] * c; \ + x3 += decode[3+(ofs)*4] * c; + +#define stbir__2_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*4] * c; \ + x1 += decode[1+(ofs)*4] * c; \ + x2 += decode[2+(ofs)*4] * c; \ + x3 += decode[3+(ofs)*4] * c; \ + c = hc[1+(ofs)]; \ + y0 += decode[4+(ofs)*4] * c; \ + y1 += decode[5+(ofs)*4] * c; \ + y2 += decode[6+(ofs)*4] * c; \ + y3 += decode[7+(ofs)*4] * c; + +#define stbir__3_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*4] * c; \ + x1 += decode[1+(ofs)*4] * c; \ + x2 += decode[2+(ofs)*4] * c; \ + x3 += decode[3+(ofs)*4] * c; \ + c = hc[1+(ofs)]; \ + y0 += decode[4+(ofs)*4] * c; \ + y1 += decode[5+(ofs)*4] * c; \ + y2 += decode[6+(ofs)*4] * c; \ + y3 += decode[7+(ofs)*4] * c; \ + c = hc[2+(ofs)]; \ + x0 += decode[8+(ofs)*4] * c; \ + x1 += decode[9+(ofs)*4] * c; \ + x2 += decode[10+(ofs)*4] * c; \ + x3 += decode[11+(ofs)*4] * c; + +#define stbir__store_output() \ + output[0] = x0 + y0; \ + output[1] = x1 + y1; \ + output[2] = x2 + y2; \ + output[3] = x3 + y3; \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 4; + +#endif + +#define STBIR__horizontal_channels 4 +#define STB_IMAGE_RESIZE_DO_HORIZONTALS +#include STBIR__HEADER_FILENAME + + + +//================= +// Do 7 channel horizontal routines + +#ifdef STBIR_SIMD + +#define stbir__1_coeff_only() \ + stbir__simdf tot0,tot1,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1( c, hc ); \ + stbir__simdf_0123to0000( c, c ); \ + stbir__simdf_mult_mem( tot0, c, decode ); \ + stbir__simdf_mult_mem( tot1, c, decode+3 ); + +#define stbir__2_coeff_only() \ + stbir__simdf tot0,tot1,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_mult_mem( tot0, c, decode ); \ + stbir__simdf_mult_mem( tot1, c, decode+3 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+7 ); \ + stbir__simdf_madd_mem( tot1, tot1, c,decode+10 ); + +#define stbir__3_coeff_only() \ + stbir__simdf tot0,tot1,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_mult_mem( tot0, c, decode ); \ + stbir__simdf_mult_mem( tot1, c, decode+3 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+7 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+10 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+14 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+17 ); + +#define stbir__store_output_tiny() \ + stbir__simdf_store( output+3, tot1 ); \ + stbir__simdf_store( output, tot0 ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 7; + +#ifdef STBIR_SIMD8 + +#define stbir__4_coeff_start() \ + stbir__simdf8 tot0,tot1,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc ); \ + stbir__simdf8_0123to00000000( c, cs ); \ + stbir__simdf8_mult_mem( tot0, c, decode ); \ + stbir__simdf8_0123to11111111( c, cs ); \ + stbir__simdf8_mult_mem( tot1, c, decode+7 ); \ + stbir__simdf8_0123to22222222( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+14 ); \ + stbir__simdf8_0123to33333333( c, cs ); \ + stbir__simdf8_madd_mem( tot1, tot1, c, decode+21 ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00000000( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \ + stbir__simdf8_0123to11111111( c, cs ); \ + stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+7 ); \ + stbir__simdf8_0123to22222222( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); \ + stbir__simdf8_0123to33333333( c, cs ); \ + stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+21 ); + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load1b( c, hc + (ofs) ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); + +#define stbir__2_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load1b( c, hc + (ofs) ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \ + stbir__simdf8_load1b( c, hc + (ofs)+1 ); \ + stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+7 ); + +#define stbir__3_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf8_load4b( cs, hc + (ofs) ); \ + stbir__simdf8_0123to00000000( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \ + stbir__simdf8_0123to11111111( c, cs ); \ + stbir__simdf8_madd_mem( tot1, tot1, c, decode+(ofs)*7+7 ); \ + stbir__simdf8_0123to22222222( c, cs ); \ + stbir__simdf8_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); + +#define stbir__store_output() \ + stbir__simdf8_add( tot0, tot0, tot1 ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 7; \ + if ( output < output_end ) \ + { \ + stbir__simdf8_store( output-7, tot0 ); \ + continue; \ + } \ + stbir__simdf_store( output-7+3, stbir__simdf_swiz(stbir__simdf8_gettop4(tot0),0,0,1,2) ); \ + stbir__simdf_store( output-7, stbir__if_simdf8_cast_to_simdf4(tot0) ); \ + break; + +#else + +#define stbir__4_coeff_start() \ + stbir__simdf tot0,tot1,tot2,tot3,c,cs; \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_mult_mem( tot0, c, decode ); \ + stbir__simdf_mult_mem( tot1, c, decode+3 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_mult_mem( tot2, c, decode+7 ); \ + stbir__simdf_mult_mem( tot3, c, decode+10 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+14 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+17 ); \ + stbir__simdf_0123to3333( c, cs ); \ + stbir__simdf_madd_mem( tot2, tot2, c, decode+21 ); \ + stbir__simdf_madd_mem( tot3, tot3, c, decode+24 ); + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+7 ); \ + stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+10 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+17 ); \ + stbir__simdf_0123to3333( c, cs ); \ + stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+21 ); \ + stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+24 ); + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load1( c, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, c ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \ + +#define stbir__2_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load2( cs, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+7 ); \ + stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+10 ); + +#define stbir__3_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + stbir__simdf_load( cs, hc + (ofs) ); \ + stbir__simdf_0123to0000( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+3 ); \ + stbir__simdf_0123to1111( c, cs ); \ + stbir__simdf_madd_mem( tot2, tot2, c, decode+(ofs)*7+7 ); \ + stbir__simdf_madd_mem( tot3, tot3, c, decode+(ofs)*7+10 ); \ + stbir__simdf_0123to2222( c, cs ); \ + stbir__simdf_madd_mem( tot0, tot0, c, decode+(ofs)*7+14 ); \ + stbir__simdf_madd_mem( tot1, tot1, c, decode+(ofs)*7+17 ); + +#define stbir__store_output() \ + stbir__simdf_add( tot0, tot0, tot2 ); \ + stbir__simdf_add( tot1, tot1, tot3 ); \ + stbir__simdf_store( output+3, tot1 ); \ + stbir__simdf_store( output, tot0 ); \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 7; + +#endif + +#else + +#define stbir__1_coeff_only() \ + float tot0, tot1, tot2, tot3, tot4, tot5, tot6, c; \ + c = hc[0]; \ + tot0 = decode[0]*c; \ + tot1 = decode[1]*c; \ + tot2 = decode[2]*c; \ + tot3 = decode[3]*c; \ + tot4 = decode[4]*c; \ + tot5 = decode[5]*c; \ + tot6 = decode[6]*c; + +#define stbir__2_coeff_only() \ + float tot0, tot1, tot2, tot3, tot4, tot5, tot6, c; \ + c = hc[0]; \ + tot0 = decode[0]*c; \ + tot1 = decode[1]*c; \ + tot2 = decode[2]*c; \ + tot3 = decode[3]*c; \ + tot4 = decode[4]*c; \ + tot5 = decode[5]*c; \ + tot6 = decode[6]*c; \ + c = hc[1]; \ + tot0 += decode[7]*c; \ + tot1 += decode[8]*c; \ + tot2 += decode[9]*c; \ + tot3 += decode[10]*c; \ + tot4 += decode[11]*c; \ + tot5 += decode[12]*c; \ + tot6 += decode[13]*c; \ + +#define stbir__3_coeff_only() \ + float tot0, tot1, tot2, tot3, tot4, tot5, tot6, c; \ + c = hc[0]; \ + tot0 = decode[0]*c; \ + tot1 = decode[1]*c; \ + tot2 = decode[2]*c; \ + tot3 = decode[3]*c; \ + tot4 = decode[4]*c; \ + tot5 = decode[5]*c; \ + tot6 = decode[6]*c; \ + c = hc[1]; \ + tot0 += decode[7]*c; \ + tot1 += decode[8]*c; \ + tot2 += decode[9]*c; \ + tot3 += decode[10]*c; \ + tot4 += decode[11]*c; \ + tot5 += decode[12]*c; \ + tot6 += decode[13]*c; \ + c = hc[2]; \ + tot0 += decode[14]*c; \ + tot1 += decode[15]*c; \ + tot2 += decode[16]*c; \ + tot3 += decode[17]*c; \ + tot4 += decode[18]*c; \ + tot5 += decode[19]*c; \ + tot6 += decode[20]*c; \ + +#define stbir__store_output_tiny() \ + output[0] = tot0; \ + output[1] = tot1; \ + output[2] = tot2; \ + output[3] = tot3; \ + output[4] = tot4; \ + output[5] = tot5; \ + output[6] = tot6; \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 7; + +#define stbir__4_coeff_start() \ + float x0,x1,x2,x3,x4,x5,x6,y0,y1,y2,y3,y4,y5,y6,c; \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0]; \ + x0 = decode[0] * c; \ + x1 = decode[1] * c; \ + x2 = decode[2] * c; \ + x3 = decode[3] * c; \ + x4 = decode[4] * c; \ + x5 = decode[5] * c; \ + x6 = decode[6] * c; \ + c = hc[1]; \ + y0 = decode[7] * c; \ + y1 = decode[8] * c; \ + y2 = decode[9] * c; \ + y3 = decode[10] * c; \ + y4 = decode[11] * c; \ + y5 = decode[12] * c; \ + y6 = decode[13] * c; \ + c = hc[2]; \ + x0 += decode[14] * c; \ + x1 += decode[15] * c; \ + x2 += decode[16] * c; \ + x3 += decode[17] * c; \ + x4 += decode[18] * c; \ + x5 += decode[19] * c; \ + x6 += decode[20] * c; \ + c = hc[3]; \ + y0 += decode[21] * c; \ + y1 += decode[22] * c; \ + y2 += decode[23] * c; \ + y3 += decode[24] * c; \ + y4 += decode[25] * c; \ + y5 += decode[26] * c; \ + y6 += decode[27] * c; + +#define stbir__4_coeff_continue_from_4( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*7] * c; \ + x1 += decode[1+(ofs)*7] * c; \ + x2 += decode[2+(ofs)*7] * c; \ + x3 += decode[3+(ofs)*7] * c; \ + x4 += decode[4+(ofs)*7] * c; \ + x5 += decode[5+(ofs)*7] * c; \ + x6 += decode[6+(ofs)*7] * c; \ + c = hc[1+(ofs)]; \ + y0 += decode[7+(ofs)*7] * c; \ + y1 += decode[8+(ofs)*7] * c; \ + y2 += decode[9+(ofs)*7] * c; \ + y3 += decode[10+(ofs)*7] * c; \ + y4 += decode[11+(ofs)*7] * c; \ + y5 += decode[12+(ofs)*7] * c; \ + y6 += decode[13+(ofs)*7] * c; \ + c = hc[2+(ofs)]; \ + x0 += decode[14+(ofs)*7] * c; \ + x1 += decode[15+(ofs)*7] * c; \ + x2 += decode[16+(ofs)*7] * c; \ + x3 += decode[17+(ofs)*7] * c; \ + x4 += decode[18+(ofs)*7] * c; \ + x5 += decode[19+(ofs)*7] * c; \ + x6 += decode[20+(ofs)*7] * c; \ + c = hc[3+(ofs)]; \ + y0 += decode[21+(ofs)*7] * c; \ + y1 += decode[22+(ofs)*7] * c; \ + y2 += decode[23+(ofs)*7] * c; \ + y3 += decode[24+(ofs)*7] * c; \ + y4 += decode[25+(ofs)*7] * c; \ + y5 += decode[26+(ofs)*7] * c; \ + y6 += decode[27+(ofs)*7] * c; + +#define stbir__1_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*7] * c; \ + x1 += decode[1+(ofs)*7] * c; \ + x2 += decode[2+(ofs)*7] * c; \ + x3 += decode[3+(ofs)*7] * c; \ + x4 += decode[4+(ofs)*7] * c; \ + x5 += decode[5+(ofs)*7] * c; \ + x6 += decode[6+(ofs)*7] * c; \ + +#define stbir__2_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*7] * c; \ + x1 += decode[1+(ofs)*7] * c; \ + x2 += decode[2+(ofs)*7] * c; \ + x3 += decode[3+(ofs)*7] * c; \ + x4 += decode[4+(ofs)*7] * c; \ + x5 += decode[5+(ofs)*7] * c; \ + x6 += decode[6+(ofs)*7] * c; \ + c = hc[1+(ofs)]; \ + y0 += decode[7+(ofs)*7] * c; \ + y1 += decode[8+(ofs)*7] * c; \ + y2 += decode[9+(ofs)*7] * c; \ + y3 += decode[10+(ofs)*7] * c; \ + y4 += decode[11+(ofs)*7] * c; \ + y5 += decode[12+(ofs)*7] * c; \ + y6 += decode[13+(ofs)*7] * c; \ + +#define stbir__3_coeff_remnant( ofs ) \ + STBIR_SIMD_NO_UNROLL(decode); \ + c = hc[0+(ofs)]; \ + x0 += decode[0+(ofs)*7] * c; \ + x1 += decode[1+(ofs)*7] * c; \ + x2 += decode[2+(ofs)*7] * c; \ + x3 += decode[3+(ofs)*7] * c; \ + x4 += decode[4+(ofs)*7] * c; \ + x5 += decode[5+(ofs)*7] * c; \ + x6 += decode[6+(ofs)*7] * c; \ + c = hc[1+(ofs)]; \ + y0 += decode[7+(ofs)*7] * c; \ + y1 += decode[8+(ofs)*7] * c; \ + y2 += decode[9+(ofs)*7] * c; \ + y3 += decode[10+(ofs)*7] * c; \ + y4 += decode[11+(ofs)*7] * c; \ + y5 += decode[12+(ofs)*7] * c; \ + y6 += decode[13+(ofs)*7] * c; \ + c = hc[2+(ofs)]; \ + x0 += decode[14+(ofs)*7] * c; \ + x1 += decode[15+(ofs)*7] * c; \ + x2 += decode[16+(ofs)*7] * c; \ + x3 += decode[17+(ofs)*7] * c; \ + x4 += decode[18+(ofs)*7] * c; \ + x5 += decode[19+(ofs)*7] * c; \ + x6 += decode[20+(ofs)*7] * c; \ + +#define stbir__store_output() \ + output[0] = x0 + y0; \ + output[1] = x1 + y1; \ + output[2] = x2 + y2; \ + output[3] = x3 + y3; \ + output[4] = x4 + y4; \ + output[5] = x5 + y5; \ + output[6] = x6 + y6; \ + horizontal_coefficients += coefficient_width; \ + ++horizontal_contributors; \ + output += 7; + +#endif + +#define STBIR__horizontal_channels 7 +#define STB_IMAGE_RESIZE_DO_HORIZONTALS +#include STBIR__HEADER_FILENAME + + +// include all of the vertical resamplers (both scatter and gather versions) + +#define STBIR__vertical_channels 1 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 1 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 2 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 2 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 3 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 3 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 4 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 4 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 5 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 5 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 6 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 6 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 7 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 7 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 8 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#include STBIR__HEADER_FILENAME + +#define STBIR__vertical_channels 8 +#define STB_IMAGE_RESIZE_DO_VERTICALS +#define STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#include STBIR__HEADER_FILENAME + +typedef void STBIR_VERTICAL_GATHERFUNC( float * output, float const * coeffs, float const ** inputs, float const * input0_end ); + +static STBIR_VERTICAL_GATHERFUNC * stbir__vertical_gathers[ 8 ] = +{ + stbir__vertical_gather_with_1_coeffs,stbir__vertical_gather_with_2_coeffs,stbir__vertical_gather_with_3_coeffs,stbir__vertical_gather_with_4_coeffs,stbir__vertical_gather_with_5_coeffs,stbir__vertical_gather_with_6_coeffs,stbir__vertical_gather_with_7_coeffs,stbir__vertical_gather_with_8_coeffs +}; + +static STBIR_VERTICAL_GATHERFUNC * stbir__vertical_gathers_continues[ 8 ] = +{ + stbir__vertical_gather_with_1_coeffs_cont,stbir__vertical_gather_with_2_coeffs_cont,stbir__vertical_gather_with_3_coeffs_cont,stbir__vertical_gather_with_4_coeffs_cont,stbir__vertical_gather_with_5_coeffs_cont,stbir__vertical_gather_with_6_coeffs_cont,stbir__vertical_gather_with_7_coeffs_cont,stbir__vertical_gather_with_8_coeffs_cont +}; + +typedef void STBIR_VERTICAL_SCATTERFUNC( float ** outputs, float const * coeffs, float const * input, float const * input_end ); + +static STBIR_VERTICAL_SCATTERFUNC * stbir__vertical_scatter_sets[ 8 ] = +{ + stbir__vertical_scatter_with_1_coeffs,stbir__vertical_scatter_with_2_coeffs,stbir__vertical_scatter_with_3_coeffs,stbir__vertical_scatter_with_4_coeffs,stbir__vertical_scatter_with_5_coeffs,stbir__vertical_scatter_with_6_coeffs,stbir__vertical_scatter_with_7_coeffs,stbir__vertical_scatter_with_8_coeffs +}; + +static STBIR_VERTICAL_SCATTERFUNC * stbir__vertical_scatter_blends[ 8 ] = +{ + stbir__vertical_scatter_with_1_coeffs_cont,stbir__vertical_scatter_with_2_coeffs_cont,stbir__vertical_scatter_with_3_coeffs_cont,stbir__vertical_scatter_with_4_coeffs_cont,stbir__vertical_scatter_with_5_coeffs_cont,stbir__vertical_scatter_with_6_coeffs_cont,stbir__vertical_scatter_with_7_coeffs_cont,stbir__vertical_scatter_with_8_coeffs_cont +}; + + +static void stbir__encode_scanline( stbir__info const * stbir_info, void *output_buffer_data, float * encode_buffer, int row STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) +{ + int num_pixels = stbir_info->horizontal.scale_info.output_sub_size; + int channels = stbir_info->channels; + int width_times_channels = num_pixels * channels; + void * output_buffer; + + // un-alpha weight if we need to + if ( stbir_info->alpha_unweight ) + { + STBIR_PROFILE_START( unalpha ); + stbir_info->alpha_unweight( encode_buffer, width_times_channels ); + STBIR_PROFILE_END( unalpha ); + } + + // write directly into output by default + output_buffer = output_buffer_data; + + // if we have an output callback, we first convert the decode buffer in place (and then hand that to the callback) + if ( stbir_info->out_pixels_cb ) + output_buffer = encode_buffer; + + STBIR_PROFILE_START( encode ); + // convert into the output buffer + stbir_info->encode_pixels( output_buffer, width_times_channels, encode_buffer ); + STBIR_PROFILE_END( encode ); + + // if we have an output callback, call it to send the data + if ( stbir_info->out_pixels_cb ) + stbir_info->out_pixels_cb( output_buffer, num_pixels, row, stbir_info->user_data ); +} + + +// Get the ring buffer pointer for an index +static float* stbir__get_ring_buffer_entry(stbir__info const * stbir_info, stbir__per_split_info const * split_info, int index ) +{ + STBIR_ASSERT( index < stbir_info->ring_buffer_num_entries ); + + #ifdef STBIR__SEPARATE_ALLOCATIONS + return split_info->ring_buffers[ index ]; + #else + return (float*) ( ( (char*) split_info->ring_buffer ) + ( index * stbir_info->ring_buffer_length_bytes ) ); + #endif +} + +// Get the specified scan line from the ring buffer +static float* stbir__get_ring_buffer_scanline(stbir__info const * stbir_info, stbir__per_split_info const * split_info, int get_scanline) +{ + int ring_buffer_index = (split_info->ring_buffer_begin_index + (get_scanline - split_info->ring_buffer_first_scanline)) % stbir_info->ring_buffer_num_entries; + return stbir__get_ring_buffer_entry( stbir_info, split_info, ring_buffer_index ); +} + +static void stbir__resample_horizontal_gather(stbir__info const * stbir_info, float* output_buffer, float const * input_buffer STBIR_ONLY_PROFILE_GET_SPLIT_INFO ) +{ + float const * decode_buffer = input_buffer - ( stbir_info->scanline_extents.conservative.n0 * stbir_info->effective_channels ); + + STBIR_PROFILE_START( horizontal ); + if ( ( stbir_info->horizontal.filter_enum == STBIR_FILTER_POINT_SAMPLE ) && ( stbir_info->horizontal.scale_info.scale == 1.0f ) ) + STBIR_MEMCPY( output_buffer, input_buffer, stbir_info->horizontal.scale_info.output_sub_size * sizeof( float ) * stbir_info->effective_channels ); + else + stbir_info->horizontal_gather_channels( output_buffer, stbir_info->horizontal.scale_info.output_sub_size, decode_buffer, stbir_info->horizontal.contributors, stbir_info->horizontal.coefficients, stbir_info->horizontal.coefficient_width ); + STBIR_PROFILE_END( horizontal ); +} + +static void stbir__resample_vertical_gather(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n, int contrib_n0, int contrib_n1, float const * vertical_coefficients ) +{ + float* encode_buffer = split_info->vertical_buffer; + float* decode_buffer = split_info->decode_buffer; + int vertical_first = stbir_info->vertical_first; + int width = (vertical_first) ? ( stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1 ) : stbir_info->horizontal.scale_info.output_sub_size; + int width_times_channels = stbir_info->effective_channels * width; + + STBIR_ASSERT( stbir_info->vertical.is_gather ); + + // loop over the contributing scanlines and scale into the buffer + STBIR_PROFILE_START( vertical ); + { + int k = 0, total = contrib_n1 - contrib_n0 + 1; + STBIR_ASSERT( total > 0 ); + do { + float const * inputs[8]; + int i, cnt = total; if ( cnt > 8 ) cnt = 8; + for( i = 0 ; i < cnt ; i++ ) + inputs[ i ] = stbir__get_ring_buffer_scanline(stbir_info, split_info, k+i+contrib_n0 ); + + // call the N scanlines at a time function (up to 8 scanlines of blending at once) + ((k==0)?stbir__vertical_gathers:stbir__vertical_gathers_continues)[cnt-1]( (vertical_first) ? decode_buffer : encode_buffer, vertical_coefficients + k, inputs, inputs[0] + width_times_channels ); + k += cnt; + total -= cnt; + } while ( total ); + } + STBIR_PROFILE_END( vertical ); + + if ( vertical_first ) + { + // Now resample the gathered vertical data in the horizontal axis into the encode buffer + stbir__resample_horizontal_gather(stbir_info, encode_buffer, decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + } + + stbir__encode_scanline( stbir_info, ( (char *) stbir_info->output_data ) + ((size_t)n * (size_t)stbir_info->output_stride_bytes), + encode_buffer, n STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); +} + +static void stbir__decode_and_resample_for_vertical_gather_loop(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n) +{ + int ring_buffer_index; + float* ring_buffer; + + // Decode the nth scanline from the source image into the decode buffer. + stbir__decode_scanline( stbir_info, n, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + + // update new end scanline + split_info->ring_buffer_last_scanline = n; + + // get ring buffer + ring_buffer_index = (split_info->ring_buffer_begin_index + (split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline)) % stbir_info->ring_buffer_num_entries; + ring_buffer = stbir__get_ring_buffer_entry(stbir_info, split_info, ring_buffer_index); + + // Now resample it into the ring buffer. + stbir__resample_horizontal_gather( stbir_info, ring_buffer, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + + // Now it's sitting in the ring buffer ready to be used as source for the vertical sampling. +} + +static void stbir__vertical_gather_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count ) +{ + int y, start_output_y, end_output_y; + stbir__contributors* vertical_contributors = stbir_info->vertical.contributors; + float const * vertical_coefficients = stbir_info->vertical.coefficients; + + STBIR_ASSERT( stbir_info->vertical.is_gather ); + + start_output_y = split_info->start_output_y; + end_output_y = split_info[split_count-1].end_output_y; + + vertical_contributors += start_output_y; + vertical_coefficients += start_output_y * stbir_info->vertical.coefficient_width; + + // initialize the ring buffer for gathering + split_info->ring_buffer_begin_index = 0; + split_info->ring_buffer_first_scanline = vertical_contributors->n0; + split_info->ring_buffer_last_scanline = split_info->ring_buffer_first_scanline - 1; // means "empty" + + for (y = start_output_y; y < end_output_y; y++) + { + int in_first_scanline, in_last_scanline; + + in_first_scanline = vertical_contributors->n0; + in_last_scanline = vertical_contributors->n1; + + // make sure the indexing hasn't broken + STBIR_ASSERT( in_first_scanline >= split_info->ring_buffer_first_scanline ); + + // Load in new scanlines + while (in_last_scanline > split_info->ring_buffer_last_scanline) + { + STBIR_ASSERT( ( split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline + 1 ) <= stbir_info->ring_buffer_num_entries ); + + // make sure there was room in the ring buffer when we add new scanlines + if ( ( split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline + 1 ) == stbir_info->ring_buffer_num_entries ) + { + split_info->ring_buffer_first_scanline++; + split_info->ring_buffer_begin_index++; + } + + if ( stbir_info->vertical_first ) + { + float * ring_buffer = stbir__get_ring_buffer_scanline( stbir_info, split_info, ++split_info->ring_buffer_last_scanline ); + // Decode the nth scanline from the source image into the decode buffer. + stbir__decode_scanline( stbir_info, split_info->ring_buffer_last_scanline, ring_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + } + else + { + stbir__decode_and_resample_for_vertical_gather_loop(stbir_info, split_info, split_info->ring_buffer_last_scanline + 1); + } + } + + // Now all buffers should be ready to write a row of vertical sampling, so do it. + stbir__resample_vertical_gather(stbir_info, split_info, y, in_first_scanline, in_last_scanline, vertical_coefficients ); + + ++vertical_contributors; + vertical_coefficients += stbir_info->vertical.coefficient_width; + } +} + +#define STBIR__FLOAT_EMPTY_MARKER 3.0e+38F +#define STBIR__FLOAT_BUFFER_IS_EMPTY(ptr) ((ptr)[0]==STBIR__FLOAT_EMPTY_MARKER) + +static void stbir__encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info) +{ + // evict a scanline out into the output buffer + float* ring_buffer_entry = stbir__get_ring_buffer_entry(stbir_info, split_info, split_info->ring_buffer_begin_index ); + + // dump the scanline out + stbir__encode_scanline( stbir_info, ( (char *)stbir_info->output_data ) + ( (size_t)split_info->ring_buffer_first_scanline * (size_t)stbir_info->output_stride_bytes ), ring_buffer_entry, split_info->ring_buffer_first_scanline STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + + // mark it as empty + ring_buffer_entry[ 0 ] = STBIR__FLOAT_EMPTY_MARKER; + + // advance the first scanline + split_info->ring_buffer_first_scanline++; + if ( ++split_info->ring_buffer_begin_index == stbir_info->ring_buffer_num_entries ) + split_info->ring_buffer_begin_index = 0; +} + +static void stbir__horizontal_resample_and_encode_first_scanline_from_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info) +{ + // evict a scanline out into the output buffer + + float* ring_buffer_entry = stbir__get_ring_buffer_entry(stbir_info, split_info, split_info->ring_buffer_begin_index ); + + // Now resample it into the buffer. + stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, ring_buffer_entry STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + + // dump the scanline out + stbir__encode_scanline( stbir_info, ( (char *)stbir_info->output_data ) + ( (size_t)split_info->ring_buffer_first_scanline * (size_t)stbir_info->output_stride_bytes ), split_info->vertical_buffer, split_info->ring_buffer_first_scanline STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + + // mark it as empty + ring_buffer_entry[ 0 ] = STBIR__FLOAT_EMPTY_MARKER; + + // advance the first scanline + split_info->ring_buffer_first_scanline++; + if ( ++split_info->ring_buffer_begin_index == stbir_info->ring_buffer_num_entries ) + split_info->ring_buffer_begin_index = 0; +} + +static void stbir__resample_vertical_scatter(stbir__info const * stbir_info, stbir__per_split_info* split_info, int n0, int n1, float const * vertical_coefficients, float const * vertical_buffer, float const * vertical_buffer_end ) +{ + STBIR_ASSERT( !stbir_info->vertical.is_gather ); + + STBIR_PROFILE_START( vertical ); + { + int k = 0, total = n1 - n0 + 1; + STBIR_ASSERT( total > 0 ); + do { + float * outputs[8]; + int i, n = total; if ( n > 8 ) n = 8; + for( i = 0 ; i < n ; i++ ) + { + outputs[ i ] = stbir__get_ring_buffer_scanline(stbir_info, split_info, k+i+n0 ); + if ( ( i ) && ( STBIR__FLOAT_BUFFER_IS_EMPTY( outputs[i] ) != STBIR__FLOAT_BUFFER_IS_EMPTY( outputs[0] ) ) ) // make sure runs are of the same type + { + n = i; + break; + } + } + // call the scatter to N scanlines at a time function (up to 8 scanlines of scattering at once) + ((STBIR__FLOAT_BUFFER_IS_EMPTY( outputs[0] ))?stbir__vertical_scatter_sets:stbir__vertical_scatter_blends)[n-1]( outputs, vertical_coefficients + k, vertical_buffer, vertical_buffer_end ); + k += n; + total -= n; + } while ( total ); + } + + STBIR_PROFILE_END( vertical ); +} + +typedef void stbir__handle_scanline_for_scatter_func(stbir__info const * stbir_info, stbir__per_split_info* split_info); + +static void stbir__vertical_scatter_loop( stbir__info const * stbir_info, stbir__per_split_info* split_info, int split_count ) +{ + int y, start_output_y, end_output_y, start_input_y, end_input_y; + stbir__contributors* vertical_contributors = stbir_info->vertical.contributors; + float const * vertical_coefficients = stbir_info->vertical.coefficients; + stbir__handle_scanline_for_scatter_func * handle_scanline_for_scatter; + void * scanline_scatter_buffer; + void * scanline_scatter_buffer_end; + int on_first_input_y, last_input_y; + + STBIR_ASSERT( !stbir_info->vertical.is_gather ); + + start_output_y = split_info->start_output_y; + end_output_y = split_info[split_count-1].end_output_y; // may do multiple split counts + + start_input_y = split_info->start_input_y; + end_input_y = split_info[split_count-1].end_input_y; + + // adjust for starting offset start_input_y + y = start_input_y + stbir_info->vertical.filter_pixel_margin; + vertical_contributors += y ; + vertical_coefficients += stbir_info->vertical.coefficient_width * y; + + if ( stbir_info->vertical_first ) + { + handle_scanline_for_scatter = stbir__horizontal_resample_and_encode_first_scanline_from_scatter; + scanline_scatter_buffer = split_info->decode_buffer; + scanline_scatter_buffer_end = ( (char*) scanline_scatter_buffer ) + sizeof( float ) * stbir_info->effective_channels * (stbir_info->scanline_extents.conservative.n1-stbir_info->scanline_extents.conservative.n0+1); + } + else + { + handle_scanline_for_scatter = stbir__encode_first_scanline_from_scatter; + scanline_scatter_buffer = split_info->vertical_buffer; + scanline_scatter_buffer_end = ( (char*) scanline_scatter_buffer ) + sizeof( float ) * stbir_info->effective_channels * stbir_info->horizontal.scale_info.output_sub_size; + } + + // initialize the ring buffer for scattering + split_info->ring_buffer_first_scanline = start_output_y; + split_info->ring_buffer_last_scanline = -1; + split_info->ring_buffer_begin_index = -1; + + // mark all the buffers as empty to start + for( y = 0 ; y < stbir_info->ring_buffer_num_entries ; y++ ) + stbir__get_ring_buffer_entry( stbir_info, split_info, y )[0] = STBIR__FLOAT_EMPTY_MARKER; // only used on scatter + + // do the loop in input space + on_first_input_y = 1; last_input_y = start_input_y; + for (y = start_input_y ; y < end_input_y; y++) + { + int out_first_scanline, out_last_scanline; + + out_first_scanline = vertical_contributors->n0; + out_last_scanline = vertical_contributors->n1; + + STBIR_ASSERT(out_last_scanline - out_first_scanline + 1 <= stbir_info->ring_buffer_num_entries); + + if ( ( out_last_scanline >= out_first_scanline ) && ( ( ( out_first_scanline >= start_output_y ) && ( out_first_scanline < end_output_y ) ) || ( ( out_last_scanline >= start_output_y ) && ( out_last_scanline < end_output_y ) ) ) ) + { + float const * vc = vertical_coefficients; + + // keep track of the range actually seen for the next resize + last_input_y = y; + if ( ( on_first_input_y ) && ( y > start_input_y ) ) + split_info->start_input_y = y; + on_first_input_y = 0; + + // clip the region + if ( out_first_scanline < start_output_y ) + { + vc += start_output_y - out_first_scanline; + out_first_scanline = start_output_y; + } + + if ( out_last_scanline >= end_output_y ) + out_last_scanline = end_output_y - 1; + + // if very first scanline, init the index + if (split_info->ring_buffer_begin_index < 0) + split_info->ring_buffer_begin_index = out_first_scanline - start_output_y; + + STBIR_ASSERT( split_info->ring_buffer_begin_index <= out_first_scanline ); + + // Decode the nth scanline from the source image into the decode buffer. + stbir__decode_scanline( stbir_info, y, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + + // When horizontal first, we resample horizontally into the vertical buffer before we scatter it out + if ( !stbir_info->vertical_first ) + stbir__resample_horizontal_gather( stbir_info, split_info->vertical_buffer, split_info->decode_buffer STBIR_ONLY_PROFILE_SET_SPLIT_INFO ); + + // Now it's sitting in the buffer ready to be distributed into the ring buffers. + + // evict from the ringbuffer, if we need are full + if ( ( ( split_info->ring_buffer_last_scanline - split_info->ring_buffer_first_scanline + 1 ) == stbir_info->ring_buffer_num_entries ) && + ( out_last_scanline > split_info->ring_buffer_last_scanline ) ) + handle_scanline_for_scatter( stbir_info, split_info ); + + // Now the horizontal buffer is ready to write to all ring buffer rows, so do it. + stbir__resample_vertical_scatter(stbir_info, split_info, out_first_scanline, out_last_scanline, vc, (float*)scanline_scatter_buffer, (float*)scanline_scatter_buffer_end ); + + // update the end of the buffer + if ( out_last_scanline > split_info->ring_buffer_last_scanline ) + split_info->ring_buffer_last_scanline = out_last_scanline; + } + ++vertical_contributors; + vertical_coefficients += stbir_info->vertical.coefficient_width; + } + + // now evict the scanlines that are left over in the ring buffer + while ( split_info->ring_buffer_first_scanline < end_output_y ) + handle_scanline_for_scatter(stbir_info, split_info); + + // update the end_input_y if we do multiple resizes with the same data + ++last_input_y; + for( y = 0 ; y < split_count; y++ ) + if ( split_info[y].end_input_y > last_input_y ) + split_info[y].end_input_y = last_input_y; +} + + +static stbir__kernel_callback * stbir__builtin_kernels[] = { 0, stbir__filter_trapezoid, stbir__filter_triangle, stbir__filter_cubic, stbir__filter_catmullrom, stbir__filter_mitchell, stbir__filter_point }; +static stbir__support_callback * stbir__builtin_supports[] = { 0, stbir__support_trapezoid, stbir__support_one, stbir__support_two, stbir__support_two, stbir__support_two, stbir__support_zeropoint5 }; + +static void stbir__set_sampler(stbir__sampler * samp, stbir_filter filter, stbir__kernel_callback * kernel, stbir__support_callback * support, stbir_edge edge, stbir__scale_info * scale_info, int always_gather, void * user_data ) +{ + // set filter + if (filter == 0) + { + filter = STBIR_DEFAULT_FILTER_DOWNSAMPLE; // default to downsample + if (scale_info->scale >= ( 1.0f - stbir__small_float ) ) + { + if ( (scale_info->scale <= ( 1.0f + stbir__small_float ) ) && ( STBIR_CEILF(scale_info->pixel_shift) == scale_info->pixel_shift ) ) + filter = STBIR_FILTER_POINT_SAMPLE; + else + filter = STBIR_DEFAULT_FILTER_UPSAMPLE; + } + } + samp->filter_enum = filter; + + STBIR_ASSERT(samp->filter_enum != 0); + STBIR_ASSERT((unsigned)samp->filter_enum < STBIR_FILTER_OTHER); + samp->filter_kernel = stbir__builtin_kernels[ filter ]; + samp->filter_support = stbir__builtin_supports[ filter ]; + + if ( kernel && support ) + { + samp->filter_kernel = kernel; + samp->filter_support = support; + samp->filter_enum = STBIR_FILTER_OTHER; + } + + samp->edge = edge; + samp->filter_pixel_width = stbir__get_filter_pixel_width (samp->filter_support, scale_info->scale, user_data ); + // Gather is always better, but in extreme downsamples, you have to most or all of the data in memory + // For horizontal, we always have all the pixels, so we always use gather here (always_gather==1). + // For vertical, we use gather if scaling up (which means we will have samp->filter_pixel_width + // scanlines in memory at once). + samp->is_gather = 0; + if ( scale_info->scale >= ( 1.0f - stbir__small_float ) ) + samp->is_gather = 1; + else if ( ( always_gather ) || ( samp->filter_pixel_width <= STBIR_FORCE_GATHER_FILTER_SCANLINES_AMOUNT ) ) + samp->is_gather = 2; + + // pre calculate stuff based on the above + samp->coefficient_width = stbir__get_coefficient_width(samp, samp->is_gather, user_data); + + // filter_pixel_width is the conservative size in pixels of input that affect an output pixel. + // In rare cases (only with 2 pix to 1 pix with the default filters), it's possible that the + // filter will extend before or after the scanline beyond just one extra entire copy of the + // scanline (we would hit the edge twice). We don't let you do that, so we clamp the total + // width to 3x the total of input pixel (once for the scanline, once for the left side + // overhang, and once for the right side). We only do this for edge mode, since the other + // modes can just re-edge clamp back in again. + if ( edge == STBIR_EDGE_WRAP ) + if ( samp->filter_pixel_width > ( scale_info->input_full_size * 3 ) ) + samp->filter_pixel_width = scale_info->input_full_size * 3; + + // This is how much to expand buffers to account for filters seeking outside + // the image boundaries. + samp->filter_pixel_margin = samp->filter_pixel_width / 2; + + // filter_pixel_margin is the amount that this filter can overhang on just one side of either + // end of the scanline (left or the right). Since we only allow you to overhang 1 scanline's + // worth of pixels, we clamp this one side of overhang to the input scanline size. Again, + // this clamping only happens in rare cases with the default filters (2 pix to 1 pix). + if ( edge == STBIR_EDGE_WRAP ) + if ( samp->filter_pixel_margin > scale_info->input_full_size ) + samp->filter_pixel_margin = scale_info->input_full_size; + + samp->num_contributors = stbir__get_contributors(samp, samp->is_gather); + + samp->contributors_size = samp->num_contributors * sizeof(stbir__contributors); + samp->coefficients_size = samp->num_contributors * samp->coefficient_width * sizeof(float) + sizeof(float); // extra sizeof(float) is padding + + samp->gather_prescatter_contributors = 0; + samp->gather_prescatter_coefficients = 0; + if ( samp->is_gather == 0 ) + { + samp->gather_prescatter_coefficient_width = samp->filter_pixel_width; + samp->gather_prescatter_num_contributors = stbir__get_contributors(samp, 2); + samp->gather_prescatter_contributors_size = samp->gather_prescatter_num_contributors * sizeof(stbir__contributors); + samp->gather_prescatter_coefficients_size = samp->gather_prescatter_num_contributors * samp->gather_prescatter_coefficient_width * sizeof(float); + } +} + +static void stbir__get_conservative_extents( stbir__sampler * samp, stbir__contributors * range, void * user_data ) +{ + float scale = samp->scale_info.scale; + float out_shift = samp->scale_info.pixel_shift; + stbir__support_callback * support = samp->filter_support; + int input_full_size = samp->scale_info.input_full_size; + stbir_edge edge = samp->edge; + float inv_scale = samp->scale_info.inv_scale; + + STBIR_ASSERT( samp->is_gather != 0 ); + + if ( samp->is_gather == 1 ) + { + int in_first_pixel, in_last_pixel; + float out_filter_radius = support(inv_scale, user_data) * scale; + + stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, 0.5, out_filter_radius, inv_scale, out_shift, input_full_size, edge ); + range->n0 = in_first_pixel; + stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, ( (float)(samp->scale_info.output_sub_size-1) ) + 0.5f, out_filter_radius, inv_scale, out_shift, input_full_size, edge ); + range->n1 = in_last_pixel; + } + else if ( samp->is_gather == 2 ) // downsample gather, refine + { + float in_pixels_radius = support(scale, user_data) * inv_scale; + int filter_pixel_margin = samp->filter_pixel_margin; + int output_sub_size = samp->scale_info.output_sub_size; + int input_end; + int n; + int in_first_pixel, in_last_pixel; + + // get a conservative area of the input range + stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, 0, 0, inv_scale, out_shift, input_full_size, edge ); + range->n0 = in_first_pixel; + stbir__calculate_in_pixel_range( &in_first_pixel, &in_last_pixel, (float)output_sub_size, 0, inv_scale, out_shift, input_full_size, edge ); + range->n1 = in_last_pixel; + + // now go through the margin to the start of area to find bottom + n = range->n0 + 1; + input_end = -filter_pixel_margin; + while( n >= input_end ) + { + int out_first_pixel, out_last_pixel; + stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, ((float)n)+0.5f, in_pixels_radius, scale, out_shift, output_sub_size ); + if ( out_first_pixel > out_last_pixel ) + break; + + if ( ( out_first_pixel < output_sub_size ) || ( out_last_pixel >= 0 ) ) + range->n0 = n; + --n; + } + + // now go through the end of the area through the margin to find top + n = range->n1 - 1; + input_end = n + 1 + filter_pixel_margin; + while( n <= input_end ) + { + int out_first_pixel, out_last_pixel; + stbir__calculate_out_pixel_range( &out_first_pixel, &out_last_pixel, ((float)n)+0.5f, in_pixels_radius, scale, out_shift, output_sub_size ); + if ( out_first_pixel > out_last_pixel ) + break; + if ( ( out_first_pixel < output_sub_size ) || ( out_last_pixel >= 0 ) ) + range->n1 = n; + ++n; + } + } + + if ( samp->edge == STBIR_EDGE_WRAP ) + { + // if we are wrapping, and we are very close to the image size (so the edges might merge), just use the scanline up to the edge + if ( ( range->n0 > 0 ) && ( range->n1 >= input_full_size ) ) + { + int marg = range->n1 - input_full_size + 1; + if ( ( marg + STBIR__MERGE_RUNS_PIXEL_THRESHOLD ) >= range->n0 ) + range->n0 = 0; + } + if ( ( range->n0 < 0 ) && ( range->n1 < (input_full_size-1) ) ) + { + int marg = -range->n0; + if ( ( input_full_size - marg - STBIR__MERGE_RUNS_PIXEL_THRESHOLD - 1 ) <= range->n1 ) + range->n1 = input_full_size - 1; + } + } + else + { + // for non-edge-wrap modes, we never read over the edge, so clamp + if ( range->n0 < 0 ) + range->n0 = 0; + if ( range->n1 >= input_full_size ) + range->n1 = input_full_size - 1; + } +} + +static void stbir__get_split_info( stbir__per_split_info* split_info, int splits, int output_height, int vertical_pixel_margin, int input_full_height ) +{ + int i, cur; + int left = output_height; + + cur = 0; + for( i = 0 ; i < splits ; i++ ) + { + int each; + split_info[i].start_output_y = cur; + each = left / ( splits - i ); + split_info[i].end_output_y = cur + each; + cur += each; + left -= each; + + // scatter range (updated to minimum as you run it) + split_info[i].start_input_y = -vertical_pixel_margin; + split_info[i].end_input_y = input_full_height + vertical_pixel_margin; + } +} + +static void stbir__free_internal_mem( stbir__info *info ) +{ + #define STBIR__FREE_AND_CLEAR( ptr ) { if ( ptr ) { void * p = (ptr); (ptr) = 0; STBIR_FREE( p, info->user_data); } } + + if ( info ) + { + #ifndef STBIR__SEPARATE_ALLOCATIONS + STBIR__FREE_AND_CLEAR( info->alloced_mem ); + #else + int i,j; + + if ( ( info->vertical.gather_prescatter_contributors ) && ( (void*)info->vertical.gather_prescatter_contributors != (void*)info->split_info[0].decode_buffer ) ) + { + STBIR__FREE_AND_CLEAR( info->vertical.gather_prescatter_coefficients ); + STBIR__FREE_AND_CLEAR( info->vertical.gather_prescatter_contributors ); + } + for( i = 0 ; i < info->splits ; i++ ) + { + for( j = 0 ; j < info->alloc_ring_buffer_num_entries ; j++ ) + { + #ifdef STBIR_SIMD8 + if ( info->effective_channels == 3 ) + --info->split_info[i].ring_buffers[j]; // avx in 3 channel mode needs one float at the start of the buffer + #endif + STBIR__FREE_AND_CLEAR( info->split_info[i].ring_buffers[j] ); + } + + #ifdef STBIR_SIMD8 + if ( info->effective_channels == 3 ) + --info->split_info[i].decode_buffer; // avx in 3 channel mode needs one float at the start of the buffer + #endif + STBIR__FREE_AND_CLEAR( info->split_info[i].decode_buffer ); + STBIR__FREE_AND_CLEAR( info->split_info[i].ring_buffers ); + STBIR__FREE_AND_CLEAR( info->split_info[i].vertical_buffer ); + } + STBIR__FREE_AND_CLEAR( info->split_info ); + if ( info->vertical.coefficients != info->horizontal.coefficients ) + { + STBIR__FREE_AND_CLEAR( info->vertical.coefficients ); + STBIR__FREE_AND_CLEAR( info->vertical.contributors ); + } + STBIR__FREE_AND_CLEAR( info->horizontal.coefficients ); + STBIR__FREE_AND_CLEAR( info->horizontal.contributors ); + STBIR__FREE_AND_CLEAR( info->alloced_mem ); + STBIR_FREE( info, info->user_data ); + #endif + } + + #undef STBIR__FREE_AND_CLEAR +} + +static int stbir__get_max_split( int splits, int height ) +{ + int i; + int max = 0; + + for( i = 0 ; i < splits ; i++ ) + { + int each = height / ( splits - i ); + if ( each > max ) + max = each; + height -= each; + } + return max; +} + +static stbir__horizontal_gather_channels_func ** stbir__horizontal_gather_n_coeffs_funcs[8] = +{ + 0, stbir__horizontal_gather_1_channels_with_n_coeffs_funcs, stbir__horizontal_gather_2_channels_with_n_coeffs_funcs, stbir__horizontal_gather_3_channels_with_n_coeffs_funcs, stbir__horizontal_gather_4_channels_with_n_coeffs_funcs, 0,0, stbir__horizontal_gather_7_channels_with_n_coeffs_funcs +}; + +static stbir__horizontal_gather_channels_func ** stbir__horizontal_gather_channels_funcs[8] = +{ + 0, stbir__horizontal_gather_1_channels_funcs, stbir__horizontal_gather_2_channels_funcs, stbir__horizontal_gather_3_channels_funcs, stbir__horizontal_gather_4_channels_funcs, 0,0, stbir__horizontal_gather_7_channels_funcs +}; + +// there are six resize classifications: 0 == vertical scatter, 1 == vertical gather < 1x scale, 2 == vertical gather 1x-2x scale, 4 == vertical gather < 3x scale, 4 == vertical gather > 3x scale, 5 == <=4 pixel height, 6 == <=4 pixel wide column +#define STBIR_RESIZE_CLASSIFICATIONS 8 + +static float stbir__compute_weights[5][STBIR_RESIZE_CLASSIFICATIONS][4]= // 5 = 0=1chan, 1=2chan, 2=3chan, 3=4chan, 4=7chan +{ + { + { 1.00000f, 1.00000f, 0.31250f, 1.00000f }, + { 0.56250f, 0.59375f, 0.00000f, 0.96875f }, + { 1.00000f, 0.06250f, 0.00000f, 1.00000f }, + { 0.00000f, 0.09375f, 1.00000f, 1.00000f }, + { 1.00000f, 1.00000f, 1.00000f, 1.00000f }, + { 0.03125f, 0.12500f, 1.00000f, 1.00000f }, + { 0.06250f, 0.12500f, 0.00000f, 1.00000f }, + { 0.00000f, 1.00000f, 0.00000f, 0.03125f }, + }, { + { 0.00000f, 0.84375f, 0.00000f, 0.03125f }, + { 0.09375f, 0.93750f, 0.00000f, 0.78125f }, + { 0.87500f, 0.21875f, 0.00000f, 0.96875f }, + { 0.09375f, 0.09375f, 1.00000f, 1.00000f }, + { 1.00000f, 1.00000f, 1.00000f, 1.00000f }, + { 0.03125f, 0.12500f, 1.00000f, 1.00000f }, + { 0.06250f, 0.12500f, 0.00000f, 1.00000f }, + { 0.00000f, 1.00000f, 0.00000f, 0.53125f }, + }, { + { 0.00000f, 0.53125f, 0.00000f, 0.03125f }, + { 0.06250f, 0.96875f, 0.00000f, 0.53125f }, + { 0.87500f, 0.18750f, 0.00000f, 0.93750f }, + { 0.00000f, 0.09375f, 1.00000f, 1.00000f }, + { 1.00000f, 1.00000f, 1.00000f, 1.00000f }, + { 0.03125f, 0.12500f, 1.00000f, 1.00000f }, + { 0.06250f, 0.12500f, 0.00000f, 1.00000f }, + { 0.00000f, 1.00000f, 0.00000f, 0.56250f }, + }, { + { 0.00000f, 0.50000f, 0.00000f, 0.71875f }, + { 0.06250f, 0.84375f, 0.00000f, 0.87500f }, + { 1.00000f, 0.50000f, 0.50000f, 0.96875f }, + { 1.00000f, 0.09375f, 0.31250f, 0.50000f }, + { 1.00000f, 1.00000f, 1.00000f, 1.00000f }, + { 1.00000f, 0.03125f, 0.03125f, 0.53125f }, + { 0.18750f, 0.12500f, 0.00000f, 1.00000f }, + { 0.00000f, 1.00000f, 0.03125f, 0.18750f }, + }, { + { 0.00000f, 0.59375f, 0.00000f, 0.96875f }, + { 0.06250f, 0.81250f, 0.06250f, 0.59375f }, + { 0.75000f, 0.43750f, 0.12500f, 0.96875f }, + { 0.87500f, 0.06250f, 0.18750f, 0.43750f }, + { 1.00000f, 1.00000f, 1.00000f, 1.00000f }, + { 0.15625f, 0.12500f, 1.00000f, 1.00000f }, + { 0.06250f, 0.12500f, 0.00000f, 1.00000f }, + { 0.00000f, 1.00000f, 0.03125f, 0.34375f }, + } +}; + +// structure that allow us to query and override info for training the costs +typedef struct STBIR__V_FIRST_INFO +{ + double v_cost, h_cost; + int control_v_first; // 0 = no control, 1 = force hori, 2 = force vert + int v_first; + int v_resize_classification; + int is_gather; +} STBIR__V_FIRST_INFO; + +#ifdef STBIR__V_FIRST_INFO_BUFFER +static STBIR__V_FIRST_INFO STBIR__V_FIRST_INFO_BUFFER = {0}; +#define STBIR__V_FIRST_INFO_POINTER &STBIR__V_FIRST_INFO_BUFFER +#else +#define STBIR__V_FIRST_INFO_POINTER 0 +#endif + +// Figure out whether to scale along the horizontal or vertical first. +// This only *super* important when you are scaling by a massively +// different amount in the vertical vs the horizontal (for example, if +// you are scaling by 2x in the width, and 0.5x in the height, then you +// want to do the vertical scale first, because it's around 3x faster +// in that order. +// +// In more normal circumstances, this makes a 20-40% differences, so +// it's good to get right, but not critical. The normal way that you +// decide which direction goes first is just figuring out which +// direction does more multiplies. But with modern CPUs with their +// fancy caches and SIMD and high IPC abilities, so there's just a lot +// more that goes into it. +// +// My handwavy sort of solution is to have an app that does a whole +// bunch of timing for both vertical and horizontal first modes, +// and then another app that can read lots of these timing files +// and try to search for the best weights to use. Dotimings.c +// is the app that does a bunch of timings, and vf_train.c is the +// app that solves for the best weights (and shows how well it +// does currently). + +static int stbir__should_do_vertical_first( float weights_table[STBIR_RESIZE_CLASSIFICATIONS][4], int horizontal_filter_pixel_width, float horizontal_scale, int horizontal_output_size, int vertical_filter_pixel_width, float vertical_scale, int vertical_output_size, int is_gather, STBIR__V_FIRST_INFO * info ) +{ + double v_cost, h_cost; + float * weights; + int vertical_first; + int v_classification; + + // categorize the resize into buckets + if ( ( vertical_output_size <= 4 ) || ( horizontal_output_size <= 4 ) ) + v_classification = ( vertical_output_size < horizontal_output_size ) ? 6 : 7; + else if ( vertical_scale <= 1.0f ) + v_classification = ( is_gather ) ? 1 : 0; + else if ( vertical_scale <= 2.0f) + v_classification = 2; + else if ( vertical_scale <= 3.0f) + v_classification = 3; + else if ( vertical_scale <= 4.0f) + v_classification = 5; + else + v_classification = 6; + + // use the right weights + weights = weights_table[ v_classification ]; + + // this is the costs when you don't take into account modern CPUs with high ipc and simd and caches - wish we had a better estimate + h_cost = (float)horizontal_filter_pixel_width * weights[0] + horizontal_scale * (float)vertical_filter_pixel_width * weights[1]; + v_cost = (float)vertical_filter_pixel_width * weights[2] + vertical_scale * (float)horizontal_filter_pixel_width * weights[3]; + + // use computation estimate to decide vertical first or not + vertical_first = ( v_cost <= h_cost ) ? 1 : 0; + + // save these, if requested + if ( info ) + { + info->h_cost = h_cost; + info->v_cost = v_cost; + info->v_resize_classification = v_classification; + info->v_first = vertical_first; + info->is_gather = is_gather; + } + + // and this allows us to override everything for testing (see dotiming.c) + if ( ( info ) && ( info->control_v_first ) ) + vertical_first = ( info->control_v_first == 2 ) ? 1 : 0; + + return vertical_first; +} + +// layout lookups - must match stbir_internal_pixel_layout +static unsigned char stbir__pixel_channels[] = { + 1,2,3,3,4, // 1ch, 2ch, rgb, bgr, 4ch + 4,4,4,4,2,2, // RGBA,BGRA,ARGB,ABGR,RA,AR + 4,4,4,4,2,2, // RGBA_PM,BGRA_PM,ARGB_PM,ABGR_PM,RA_PM,AR_PM +}; + +// the internal pixel layout enums are in a different order, so we can easily do range comparisons of types +// the public pixel layout is ordered in a way that if you cast num_channels (1-4) to the enum, you get something sensible +static stbir_internal_pixel_layout stbir__pixel_layout_convert_public_to_internal[] = { + STBIRI_BGR, STBIRI_1CHANNEL, STBIRI_2CHANNEL, STBIRI_RGB, STBIRI_RGBA, + STBIRI_4CHANNEL, STBIRI_BGRA, STBIRI_ARGB, STBIRI_ABGR, STBIRI_RA, STBIRI_AR, + STBIRI_RGBA_PM, STBIRI_BGRA_PM, STBIRI_ARGB_PM, STBIRI_ABGR_PM, STBIRI_RA_PM, STBIRI_AR_PM, +}; + +static stbir__info * stbir__alloc_internal_mem_and_build_samplers( stbir__sampler * horizontal, stbir__sampler * vertical, stbir__contributors * conservative, stbir_pixel_layout input_pixel_layout_public, stbir_pixel_layout output_pixel_layout_public, int splits, int new_x, int new_y, int fast_alpha, void * user_data STBIR_ONLY_PROFILE_BUILD_GET_INFO ) +{ + static char stbir_channel_count_index[8]={ 9,0,1,2, 3,9,9,4 }; + + stbir__info * info = 0; + void * alloced = 0; + size_t alloced_total = 0; + int vertical_first; + int decode_buffer_size, ring_buffer_length_bytes, ring_buffer_size, vertical_buffer_size, alloc_ring_buffer_num_entries; + + int alpha_weighting_type = 0; // 0=none, 1=simple, 2=fancy + int conservative_split_output_size = stbir__get_max_split( splits, vertical->scale_info.output_sub_size ); + stbir_internal_pixel_layout input_pixel_layout = stbir__pixel_layout_convert_public_to_internal[ input_pixel_layout_public ]; + stbir_internal_pixel_layout output_pixel_layout = stbir__pixel_layout_convert_public_to_internal[ output_pixel_layout_public ]; + int channels = stbir__pixel_channels[ input_pixel_layout ]; + int effective_channels = channels; + + // first figure out what type of alpha weighting to use (if any) + if ( ( horizontal->filter_enum != STBIR_FILTER_POINT_SAMPLE ) || ( vertical->filter_enum != STBIR_FILTER_POINT_SAMPLE ) ) // no alpha weighting on point sampling + { + if ( ( input_pixel_layout >= STBIRI_RGBA ) && ( input_pixel_layout <= STBIRI_AR ) && ( output_pixel_layout >= STBIRI_RGBA ) && ( output_pixel_layout <= STBIRI_AR ) ) + { + if ( fast_alpha ) + { + alpha_weighting_type = 4; + } + else + { + static int fancy_alpha_effective_cnts[6] = { 7, 7, 7, 7, 3, 3 }; + alpha_weighting_type = 2; + effective_channels = fancy_alpha_effective_cnts[ input_pixel_layout - STBIRI_RGBA ]; + } + } + else if ( ( input_pixel_layout >= STBIRI_RGBA_PM ) && ( input_pixel_layout <= STBIRI_AR_PM ) && ( output_pixel_layout >= STBIRI_RGBA ) && ( output_pixel_layout <= STBIRI_AR ) ) + { + // input premult, output non-premult + alpha_weighting_type = 3; + } + else if ( ( input_pixel_layout >= STBIRI_RGBA ) && ( input_pixel_layout <= STBIRI_AR ) && ( output_pixel_layout >= STBIRI_RGBA_PM ) && ( output_pixel_layout <= STBIRI_AR_PM ) ) + { + // input non-premult, output premult + alpha_weighting_type = 1; + } + } + + // channel in and out count must match currently + if ( channels != stbir__pixel_channels[ output_pixel_layout ] ) + return 0; + + // get vertical first + vertical_first = stbir__should_do_vertical_first( stbir__compute_weights[ (int)stbir_channel_count_index[ effective_channels ] ], horizontal->filter_pixel_width, horizontal->scale_info.scale, horizontal->scale_info.output_sub_size, vertical->filter_pixel_width, vertical->scale_info.scale, vertical->scale_info.output_sub_size, vertical->is_gather, STBIR__V_FIRST_INFO_POINTER ); + + // sometimes read one float off in some of the unrolled loops (with a weight of zero coeff, so it doesn't have an effect) + decode_buffer_size = ( conservative->n1 - conservative->n0 + 1 ) * effective_channels * sizeof(float) + sizeof(float); // extra float for padding + +#if defined( STBIR__SEPARATE_ALLOCATIONS ) && defined(STBIR_SIMD8) + if ( effective_channels == 3 ) + decode_buffer_size += sizeof(float); // avx in 3 channel mode needs one float at the start of the buffer (only with separate allocations) +#endif + + ring_buffer_length_bytes = horizontal->scale_info.output_sub_size * effective_channels * sizeof(float) + sizeof(float); // extra float for padding + + // if we do vertical first, the ring buffer holds a whole decoded line + if ( vertical_first ) + ring_buffer_length_bytes = ( decode_buffer_size + 15 ) & ~15; + + if ( ( ring_buffer_length_bytes & 4095 ) == 0 ) ring_buffer_length_bytes += 64*3; // avoid 4k alias + + // One extra entry because floating point precision problems sometimes cause an extra to be necessary. + alloc_ring_buffer_num_entries = vertical->filter_pixel_width + 1; + + // we never need more ring buffer entries than the scanlines we're outputting when in scatter mode + if ( ( !vertical->is_gather ) && ( alloc_ring_buffer_num_entries > conservative_split_output_size ) ) + alloc_ring_buffer_num_entries = conservative_split_output_size; + + ring_buffer_size = alloc_ring_buffer_num_entries * ring_buffer_length_bytes; + + // The vertical buffer is used differently, depending on whether we are scattering + // the vertical scanlines, or gathering them. + // If scattering, it's used at the temp buffer to accumulate each output. + // If gathering, it's just the output buffer. + vertical_buffer_size = horizontal->scale_info.output_sub_size * effective_channels * sizeof(float) + sizeof(float); // extra float for padding + + // we make two passes through this loop, 1st to add everything up, 2nd to allocate and init + for(;;) + { + int i; + void * advance_mem = alloced; + int copy_horizontal = 0; + stbir__sampler * possibly_use_horizontal_for_pivot = 0; + +#ifdef STBIR__SEPARATE_ALLOCATIONS + #define STBIR__NEXT_PTR( ptr, size, ntype ) if ( alloced ) { void * p = STBIR_MALLOC( size, user_data); if ( p == 0 ) { stbir__free_internal_mem( info ); return 0; } (ptr) = (ntype*)p; } +#else + #define STBIR__NEXT_PTR( ptr, size, ntype ) advance_mem = (void*) ( ( ((size_t)advance_mem) + 15 ) & ~15 ); if ( alloced ) ptr = (ntype*)advance_mem; advance_mem = ((char*)advance_mem) + (size); +#endif + + STBIR__NEXT_PTR( info, sizeof( stbir__info ), stbir__info ); + + STBIR__NEXT_PTR( info->split_info, sizeof( stbir__per_split_info ) * splits, stbir__per_split_info ); + + if ( info ) + { + static stbir__alpha_weight_func * fancy_alpha_weights[6] = { stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_4ch, stbir__fancy_alpha_weight_2ch, stbir__fancy_alpha_weight_2ch }; + static stbir__alpha_unweight_func * fancy_alpha_unweights[6] = { stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_4ch, stbir__fancy_alpha_unweight_2ch, stbir__fancy_alpha_unweight_2ch }; + static stbir__alpha_weight_func * simple_alpha_weights[6] = { stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_4ch, stbir__simple_alpha_weight_2ch, stbir__simple_alpha_weight_2ch }; + static stbir__alpha_unweight_func * simple_alpha_unweights[6] = { stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_4ch, stbir__simple_alpha_unweight_2ch, stbir__simple_alpha_unweight_2ch }; + + // initialize info fields + info->alloced_mem = alloced; + info->alloced_total = alloced_total; + + info->channels = channels; + info->effective_channels = effective_channels; + + info->offset_x = new_x; + info->offset_y = new_y; + info->alloc_ring_buffer_num_entries = alloc_ring_buffer_num_entries; + info->ring_buffer_num_entries = 0; + info->ring_buffer_length_bytes = ring_buffer_length_bytes; + info->splits = splits; + info->vertical_first = vertical_first; + + info->input_pixel_layout_internal = input_pixel_layout; + info->output_pixel_layout_internal = output_pixel_layout; + + // setup alpha weight functions + info->alpha_weight = 0; + info->alpha_unweight = 0; + + // handle alpha weighting functions and overrides + if ( alpha_weighting_type == 2 ) + { + // high quality alpha multiplying on the way in, dividing on the way out + info->alpha_weight = fancy_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; + info->alpha_unweight = fancy_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; + } + else if ( alpha_weighting_type == 4 ) + { + // fast alpha multiplying on the way in, dividing on the way out + info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; + info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; + } + else if ( alpha_weighting_type == 1 ) + { + // fast alpha on the way in, leave in premultiplied form on way out + info->alpha_weight = simple_alpha_weights[ input_pixel_layout - STBIRI_RGBA ]; + } + else if ( alpha_weighting_type == 3 ) + { + // incoming is premultiplied, fast alpha dividing on the way out - non-premultiplied output + info->alpha_unweight = simple_alpha_unweights[ output_pixel_layout - STBIRI_RGBA ]; + } + + // handle 3-chan color flipping, using the alpha weight path + if ( ( ( input_pixel_layout == STBIRI_RGB ) && ( output_pixel_layout == STBIRI_BGR ) ) || + ( ( input_pixel_layout == STBIRI_BGR ) && ( output_pixel_layout == STBIRI_RGB ) ) ) + { + // do the flipping on the smaller of the two ends + if ( horizontal->scale_info.scale < 1.0f ) + info->alpha_unweight = stbir__simple_flip_3ch; + else + info->alpha_weight = stbir__simple_flip_3ch; + } + + } + + // get all the per-split buffers + for( i = 0 ; i < splits ; i++ ) + { + STBIR__NEXT_PTR( info->split_info[i].decode_buffer, decode_buffer_size, float ); + +#ifdef STBIR__SEPARATE_ALLOCATIONS + + #ifdef STBIR_SIMD8 + if ( ( info ) && ( effective_channels == 3 ) ) + ++info->split_info[i].decode_buffer; // avx in 3 channel mode needs one float at the start of the buffer + #endif + + STBIR__NEXT_PTR( info->split_info[i].ring_buffers, alloc_ring_buffer_num_entries * sizeof(float*), float* ); + { + int j; + for( j = 0 ; j < alloc_ring_buffer_num_entries ; j++ ) + { + STBIR__NEXT_PTR( info->split_info[i].ring_buffers[j], ring_buffer_length_bytes, float ); + #ifdef STBIR_SIMD8 + if ( ( info ) && ( effective_channels == 3 ) ) + ++info->split_info[i].ring_buffers[j]; // avx in 3 channel mode needs one float at the start of the buffer + #endif + } + } +#else + STBIR__NEXT_PTR( info->split_info[i].ring_buffer, ring_buffer_size, float ); +#endif + STBIR__NEXT_PTR( info->split_info[i].vertical_buffer, vertical_buffer_size, float ); + } + + // alloc memory for to-be-pivoted coeffs (if necessary) + if ( vertical->is_gather == 0 ) + { + int both; + int temp_mem_amt; + + // when in vertical scatter mode, we first build the coefficients in gather mode, and then pivot after, + // that means we need two buffers, so we try to use the decode buffer and ring buffer for this. if that + // is too small, we just allocate extra memory to use as this temp. + + both = vertical->gather_prescatter_contributors_size + vertical->gather_prescatter_coefficients_size; + +#ifdef STBIR__SEPARATE_ALLOCATIONS + temp_mem_amt = decode_buffer_size; + + #ifdef STBIR_SIMD8 + if ( effective_channels == 3 ) + --temp_mem_amt; // avx in 3 channel mode needs one float at the start of the buffer + #endif +#else + temp_mem_amt = ( decode_buffer_size + ring_buffer_size + vertical_buffer_size ) * splits; +#endif + if ( temp_mem_amt >= both ) + { + if ( info ) + { + vertical->gather_prescatter_contributors = (stbir__contributors*)info->split_info[0].decode_buffer; + vertical->gather_prescatter_coefficients = (float*) ( ( (char*)info->split_info[0].decode_buffer ) + vertical->gather_prescatter_contributors_size ); + } + } + else + { + // ring+decode memory is too small, so allocate temp memory + STBIR__NEXT_PTR( vertical->gather_prescatter_contributors, vertical->gather_prescatter_contributors_size, stbir__contributors ); + STBIR__NEXT_PTR( vertical->gather_prescatter_coefficients, vertical->gather_prescatter_coefficients_size, float ); + } + } + + STBIR__NEXT_PTR( horizontal->contributors, horizontal->contributors_size, stbir__contributors ); + STBIR__NEXT_PTR( horizontal->coefficients, horizontal->coefficients_size, float ); + + // are the two filters identical?? (happens a lot with mipmap generation) + if ( ( horizontal->filter_kernel == vertical->filter_kernel ) && ( horizontal->filter_support == vertical->filter_support ) && ( horizontal->edge == vertical->edge ) && ( horizontal->scale_info.output_sub_size == vertical->scale_info.output_sub_size ) ) + { + float diff_scale = horizontal->scale_info.scale - vertical->scale_info.scale; + float diff_shift = horizontal->scale_info.pixel_shift - vertical->scale_info.pixel_shift; + if ( diff_scale < 0.0f ) diff_scale = -diff_scale; + if ( diff_shift < 0.0f ) diff_shift = -diff_shift; + if ( ( diff_scale <= stbir__small_float ) && ( diff_shift <= stbir__small_float ) ) + { + if ( horizontal->is_gather == vertical->is_gather ) + { + copy_horizontal = 1; + goto no_vert_alloc; + } + // everything matches, but vertical is scatter, horizontal is gather, use horizontal coeffs for vertical pivot coeffs + possibly_use_horizontal_for_pivot = horizontal; + } + } + + STBIR__NEXT_PTR( vertical->contributors, vertical->contributors_size, stbir__contributors ); + STBIR__NEXT_PTR( vertical->coefficients, vertical->coefficients_size, float ); + + no_vert_alloc: + + if ( info ) + { + STBIR_PROFILE_BUILD_START( horizontal ); + + stbir__calculate_filters( horizontal, 0, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); + + // setup the horizontal gather functions + // start with defaulting to the n_coeffs functions (specialized on channels and remnant leftover) + info->horizontal_gather_channels = stbir__horizontal_gather_n_coeffs_funcs[ effective_channels ][ horizontal->extent_info.widest & 3 ]; + // but if the number of coeffs <= 12, use another set of special cases. <=12 coeffs is any enlarging resize, or shrinking resize down to about 1/3 size + if ( horizontal->extent_info.widest <= 12 ) + info->horizontal_gather_channels = stbir__horizontal_gather_channels_funcs[ effective_channels ][ horizontal->extent_info.widest - 1 ]; + + info->scanline_extents.conservative.n0 = conservative->n0; + info->scanline_extents.conservative.n1 = conservative->n1; + + // get exact extents + stbir__get_extents( horizontal, &info->scanline_extents ); + + // pack the horizontal coeffs + horizontal->coefficient_width = stbir__pack_coefficients(horizontal->num_contributors, horizontal->contributors, horizontal->coefficients, horizontal->coefficient_width, horizontal->extent_info.widest, info->scanline_extents.conservative.n0, info->scanline_extents.conservative.n1 ); + + STBIR_MEMCPY( &info->horizontal, horizontal, sizeof( stbir__sampler ) ); + + STBIR_PROFILE_BUILD_END( horizontal ); + + if ( copy_horizontal ) + { + STBIR_MEMCPY( &info->vertical, horizontal, sizeof( stbir__sampler ) ); + } + else + { + STBIR_PROFILE_BUILD_START( vertical ); + + stbir__calculate_filters( vertical, possibly_use_horizontal_for_pivot, user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); + STBIR_MEMCPY( &info->vertical, vertical, sizeof( stbir__sampler ) ); + + STBIR_PROFILE_BUILD_END( vertical ); + } + + // setup the vertical split ranges + stbir__get_split_info( info->split_info, info->splits, info->vertical.scale_info.output_sub_size, info->vertical.filter_pixel_margin, info->vertical.scale_info.input_full_size ); + + // now we know precisely how many entries we need + info->ring_buffer_num_entries = info->vertical.extent_info.widest; + + // we never need more ring buffer entries than the scanlines we're outputting + if ( ( !info->vertical.is_gather ) && ( info->ring_buffer_num_entries > conservative_split_output_size ) ) + info->ring_buffer_num_entries = conservative_split_output_size; + STBIR_ASSERT( info->ring_buffer_num_entries <= info->alloc_ring_buffer_num_entries ); + + // a few of the horizontal gather functions read past the end of the decode (but mask it out), + // so put in normal values so no snans or denormals accidentally sneak in (also, in the ring + // buffer for vertical first) + for( i = 0 ; i < splits ; i++ ) + { + int t, ofs, start; + + ofs = decode_buffer_size / 4; + + #if defined( STBIR__SEPARATE_ALLOCATIONS ) && defined(STBIR_SIMD8) + if ( effective_channels == 3 ) + --ofs; // avx in 3 channel mode needs one float at the start of the buffer, so we snap back for clearing + #endif + + start = ofs - 4; + if ( start < 0 ) start = 0; + + for( t = start ; t < ofs; t++ ) + info->split_info[i].decode_buffer[ t ] = 9999.0f; + + if ( vertical_first ) + { + int j; + for( j = 0; j < info->ring_buffer_num_entries ; j++ ) + { + for( t = start ; t < ofs; t++ ) + stbir__get_ring_buffer_entry( info, info->split_info + i, j )[ t ] = 9999.0f; + } + } + } + } + + #undef STBIR__NEXT_PTR + + + // is this the first time through loop? + if ( info == 0 ) + { + alloced_total = ( 15 + (size_t)advance_mem ); + alloced = STBIR_MALLOC( alloced_total, user_data ); + if ( alloced == 0 ) + return 0; + } + else + return info; // success + } +} + +static int stbir__perform_resize( stbir__info const * info, int split_start, int split_count ) +{ + stbir__per_split_info * split_info = info->split_info + split_start; + + STBIR_PROFILE_CLEAR_EXTRAS(); + + STBIR_PROFILE_FIRST_START( looping ); + if (info->vertical.is_gather) + stbir__vertical_gather_loop( info, split_info, split_count ); + else + stbir__vertical_scatter_loop( info, split_info, split_count ); + STBIR_PROFILE_END( looping ); + + return 1; +} + +static void stbir__update_info_from_resize( stbir__info * info, STBIR_RESIZE * resize ) +{ + static stbir__decode_pixels_func * decode_simple[STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]= + { + /* 1ch-4ch */ stbir__decode_uint8_srgb, stbir__decode_uint8_srgb, 0, stbir__decode_float_linear, stbir__decode_half_float_linear, + }; + + static stbir__decode_pixels_func * decode_alphas[STBIRI_AR-STBIRI_RGBA+1][STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]= + { + { /* RGBA */ stbir__decode_uint8_srgb4_linearalpha, stbir__decode_uint8_srgb, 0, stbir__decode_float_linear, stbir__decode_half_float_linear }, + { /* BGRA */ stbir__decode_uint8_srgb4_linearalpha_BGRA, stbir__decode_uint8_srgb_BGRA, 0, stbir__decode_float_linear_BGRA, stbir__decode_half_float_linear_BGRA }, + { /* ARGB */ stbir__decode_uint8_srgb4_linearalpha_ARGB, stbir__decode_uint8_srgb_ARGB, 0, stbir__decode_float_linear_ARGB, stbir__decode_half_float_linear_ARGB }, + { /* ABGR */ stbir__decode_uint8_srgb4_linearalpha_ABGR, stbir__decode_uint8_srgb_ABGR, 0, stbir__decode_float_linear_ABGR, stbir__decode_half_float_linear_ABGR }, + { /* RA */ stbir__decode_uint8_srgb2_linearalpha, stbir__decode_uint8_srgb, 0, stbir__decode_float_linear, stbir__decode_half_float_linear }, + { /* AR */ stbir__decode_uint8_srgb2_linearalpha_AR, stbir__decode_uint8_srgb_AR, 0, stbir__decode_float_linear_AR, stbir__decode_half_float_linear_AR }, + }; + + static stbir__decode_pixels_func * decode_simple_scaled_or_not[2][2]= + { + { stbir__decode_uint8_linear_scaled, stbir__decode_uint8_linear }, { stbir__decode_uint16_linear_scaled, stbir__decode_uint16_linear }, + }; + + static stbir__decode_pixels_func * decode_alphas_scaled_or_not[STBIRI_AR-STBIRI_RGBA+1][2][2]= + { + { /* RGBA */ { stbir__decode_uint8_linear_scaled, stbir__decode_uint8_linear }, { stbir__decode_uint16_linear_scaled, stbir__decode_uint16_linear } }, + { /* BGRA */ { stbir__decode_uint8_linear_scaled_BGRA, stbir__decode_uint8_linear_BGRA }, { stbir__decode_uint16_linear_scaled_BGRA, stbir__decode_uint16_linear_BGRA } }, + { /* ARGB */ { stbir__decode_uint8_linear_scaled_ARGB, stbir__decode_uint8_linear_ARGB }, { stbir__decode_uint16_linear_scaled_ARGB, stbir__decode_uint16_linear_ARGB } }, + { /* ABGR */ { stbir__decode_uint8_linear_scaled_ABGR, stbir__decode_uint8_linear_ABGR }, { stbir__decode_uint16_linear_scaled_ABGR, stbir__decode_uint16_linear_ABGR } }, + { /* RA */ { stbir__decode_uint8_linear_scaled, stbir__decode_uint8_linear }, { stbir__decode_uint16_linear_scaled, stbir__decode_uint16_linear } }, + { /* AR */ { stbir__decode_uint8_linear_scaled_AR, stbir__decode_uint8_linear_AR }, { stbir__decode_uint16_linear_scaled_AR, stbir__decode_uint16_linear_AR } } + }; + + static stbir__encode_pixels_func * encode_simple[STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]= + { + /* 1ch-4ch */ stbir__encode_uint8_srgb, stbir__encode_uint8_srgb, 0, stbir__encode_float_linear, stbir__encode_half_float_linear, + }; + + static stbir__encode_pixels_func * encode_alphas[STBIRI_AR-STBIRI_RGBA+1][STBIR_TYPE_HALF_FLOAT-STBIR_TYPE_UINT8_SRGB+1]= + { + { /* RGBA */ stbir__encode_uint8_srgb4_linearalpha, stbir__encode_uint8_srgb, 0, stbir__encode_float_linear, stbir__encode_half_float_linear }, + { /* BGRA */ stbir__encode_uint8_srgb4_linearalpha_BGRA, stbir__encode_uint8_srgb_BGRA, 0, stbir__encode_float_linear_BGRA, stbir__encode_half_float_linear_BGRA }, + { /* ARGB */ stbir__encode_uint8_srgb4_linearalpha_ARGB, stbir__encode_uint8_srgb_ARGB, 0, stbir__encode_float_linear_ARGB, stbir__encode_half_float_linear_ARGB }, + { /* ABGR */ stbir__encode_uint8_srgb4_linearalpha_ABGR, stbir__encode_uint8_srgb_ABGR, 0, stbir__encode_float_linear_ABGR, stbir__encode_half_float_linear_ABGR }, + { /* RA */ stbir__encode_uint8_srgb2_linearalpha, stbir__encode_uint8_srgb, 0, stbir__encode_float_linear, stbir__encode_half_float_linear }, + { /* AR */ stbir__encode_uint8_srgb2_linearalpha_AR, stbir__encode_uint8_srgb_AR, 0, stbir__encode_float_linear_AR, stbir__encode_half_float_linear_AR } + }; + + static stbir__encode_pixels_func * encode_simple_scaled_or_not[2][2]= + { + { stbir__encode_uint8_linear_scaled, stbir__encode_uint8_linear }, { stbir__encode_uint16_linear_scaled, stbir__encode_uint16_linear }, + }; + + static stbir__encode_pixels_func * encode_alphas_scaled_or_not[STBIRI_AR-STBIRI_RGBA+1][2][2]= + { + { /* RGBA */ { stbir__encode_uint8_linear_scaled, stbir__encode_uint8_linear }, { stbir__encode_uint16_linear_scaled, stbir__encode_uint16_linear } }, + { /* BGRA */ { stbir__encode_uint8_linear_scaled_BGRA, stbir__encode_uint8_linear_BGRA }, { stbir__encode_uint16_linear_scaled_BGRA, stbir__encode_uint16_linear_BGRA } }, + { /* ARGB */ { stbir__encode_uint8_linear_scaled_ARGB, stbir__encode_uint8_linear_ARGB }, { stbir__encode_uint16_linear_scaled_ARGB, stbir__encode_uint16_linear_ARGB } }, + { /* ABGR */ { stbir__encode_uint8_linear_scaled_ABGR, stbir__encode_uint8_linear_ABGR }, { stbir__encode_uint16_linear_scaled_ABGR, stbir__encode_uint16_linear_ABGR } }, + { /* RA */ { stbir__encode_uint8_linear_scaled, stbir__encode_uint8_linear }, { stbir__encode_uint16_linear_scaled, stbir__encode_uint16_linear } }, + { /* AR */ { stbir__encode_uint8_linear_scaled_AR, stbir__encode_uint8_linear_AR }, { stbir__encode_uint16_linear_scaled_AR, stbir__encode_uint16_linear_AR } } + }; + + stbir__decode_pixels_func * decode_pixels = 0; + stbir__encode_pixels_func * encode_pixels = 0; + stbir_datatype input_type, output_type; + + input_type = resize->input_data_type; + output_type = resize->output_data_type; + info->input_data = resize->input_pixels; + info->input_stride_bytes = resize->input_stride_in_bytes; + info->output_stride_bytes = resize->output_stride_in_bytes; + + // if we're completely point sampling, then we can turn off SRGB + if ( ( info->horizontal.filter_enum == STBIR_FILTER_POINT_SAMPLE ) && ( info->vertical.filter_enum == STBIR_FILTER_POINT_SAMPLE ) ) + { + if ( ( ( input_type == STBIR_TYPE_UINT8_SRGB ) || ( input_type == STBIR_TYPE_UINT8_SRGB_ALPHA ) ) && + ( ( output_type == STBIR_TYPE_UINT8_SRGB ) || ( output_type == STBIR_TYPE_UINT8_SRGB_ALPHA ) ) ) + { + input_type = STBIR_TYPE_UINT8; + output_type = STBIR_TYPE_UINT8; + } + } + + // recalc the output and input strides + if ( info->input_stride_bytes == 0 ) + info->input_stride_bytes = info->channels * info->horizontal.scale_info.input_full_size * stbir__type_size[input_type]; + + if ( info->output_stride_bytes == 0 ) + info->output_stride_bytes = info->channels * info->horizontal.scale_info.output_sub_size * stbir__type_size[output_type]; + + // calc offset + info->output_data = ( (char*) resize->output_pixels ) + ( (size_t) info->offset_y * (size_t) resize->output_stride_in_bytes ) + ( info->offset_x * info->channels * stbir__type_size[output_type] ); + + info->in_pixels_cb = resize->input_cb; + info->user_data = resize->user_data; + info->out_pixels_cb = resize->output_cb; + + // setup the input format converters + if ( ( input_type == STBIR_TYPE_UINT8 ) || ( input_type == STBIR_TYPE_UINT16 ) ) + { + int non_scaled = 0; + + // check if we can run unscaled - 0-255.0/0-65535.0 instead of 0-1.0 (which is a tiny bit faster when doing linear 8->8 or 16->16) + if ( ( !info->alpha_weight ) && ( !info->alpha_unweight ) ) // don't short circuit when alpha weighting (get everything to 0-1.0 as usual) + if ( ( ( input_type == STBIR_TYPE_UINT8 ) && ( output_type == STBIR_TYPE_UINT8 ) ) || ( ( input_type == STBIR_TYPE_UINT16 ) && ( output_type == STBIR_TYPE_UINT16 ) ) ) + non_scaled = 1; + + if ( info->input_pixel_layout_internal <= STBIRI_4CHANNEL ) + decode_pixels = decode_simple_scaled_or_not[ input_type == STBIR_TYPE_UINT16 ][ non_scaled ]; + else + decode_pixels = decode_alphas_scaled_or_not[ ( info->input_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ input_type == STBIR_TYPE_UINT16 ][ non_scaled ]; + } + else + { + if ( info->input_pixel_layout_internal <= STBIRI_4CHANNEL ) + decode_pixels = decode_simple[ input_type - STBIR_TYPE_UINT8_SRGB ]; + else + decode_pixels = decode_alphas[ ( info->input_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ input_type - STBIR_TYPE_UINT8_SRGB ]; + } + + // setup the output format converters + if ( ( output_type == STBIR_TYPE_UINT8 ) || ( output_type == STBIR_TYPE_UINT16 ) ) + { + int non_scaled = 0; + + // check if we can run unscaled - 0-255.0/0-65535.0 instead of 0-1.0 (which is a tiny bit faster when doing linear 8->8 or 16->16) + if ( ( !info->alpha_weight ) && ( !info->alpha_unweight ) ) // don't short circuit when alpha weighting (get everything to 0-1.0 as usual) + if ( ( ( input_type == STBIR_TYPE_UINT8 ) && ( output_type == STBIR_TYPE_UINT8 ) ) || ( ( input_type == STBIR_TYPE_UINT16 ) && ( output_type == STBIR_TYPE_UINT16 ) ) ) + non_scaled = 1; + + if ( info->output_pixel_layout_internal <= STBIRI_4CHANNEL ) + encode_pixels = encode_simple_scaled_or_not[ output_type == STBIR_TYPE_UINT16 ][ non_scaled ]; + else + encode_pixels = encode_alphas_scaled_or_not[ ( info->output_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ output_type == STBIR_TYPE_UINT16 ][ non_scaled ]; + } + else + { + if ( info->output_pixel_layout_internal <= STBIRI_4CHANNEL ) + encode_pixels = encode_simple[ output_type - STBIR_TYPE_UINT8_SRGB ]; + else + encode_pixels = encode_alphas[ ( info->output_pixel_layout_internal - STBIRI_RGBA ) % ( STBIRI_AR-STBIRI_RGBA+1 ) ][ output_type - STBIR_TYPE_UINT8_SRGB ]; + } + + info->input_type = input_type; + info->output_type = output_type; + info->decode_pixels = decode_pixels; + info->encode_pixels = encode_pixels; +} + +static void stbir__clip( int * outx, int * outsubw, int outw, double * u0, double * u1 ) +{ + double per, adj; + int over; + + // do left/top edge + if ( *outx < 0 ) + { + per = ( (double)*outx ) / ( (double)*outsubw ); // is negative + adj = per * ( *u1 - *u0 ); + *u0 -= adj; // increases u0 + *outx = 0; + } + + // do right/bot edge + over = outw - ( *outx + *outsubw ); + if ( over < 0 ) + { + per = ( (double)over ) / ( (double)*outsubw ); // is negative + adj = per * ( *u1 - *u0 ); + *u1 += adj; // decrease u1 + *outsubw = outw - *outx; + } +} + +// converts a double to a rational that has less than one float bit of error (returns 0 if unable to do so) +static int stbir__double_to_rational(double f, stbir_uint32 limit, stbir_uint32 *numer, stbir_uint32 *denom, int limit_denom ) // limit_denom (1) or limit numer (0) +{ + double err; + stbir_uint64 top, bot; + stbir_uint64 numer_last = 0; + stbir_uint64 denom_last = 1; + stbir_uint64 numer_estimate = 1; + stbir_uint64 denom_estimate = 0; + + // scale to past float error range + top = (stbir_uint64)( f * (double)(1 << 25) ); + bot = 1 << 25; + + // keep refining, but usually stops in a few loops - usually 5 for bad cases + for(;;) + { + stbir_uint64 est, temp; + + // hit limit, break out and do best full range estimate + if ( ( ( limit_denom ) ? denom_estimate : numer_estimate ) >= limit ) + break; + + // is the current error less than 1 bit of a float? if so, we're done + if ( denom_estimate ) + { + err = ( (double)numer_estimate / (double)denom_estimate ) - f; + if ( err < 0.0 ) err = -err; + if ( err < ( 1.0 / (double)(1<<24) ) ) + { + // yup, found it + *numer = (stbir_uint32) numer_estimate; + *denom = (stbir_uint32) denom_estimate; + return 1; + } + } + + // no more refinement bits left? break out and do full range estimate + if ( bot == 0 ) + break; + + // gcd the estimate bits + est = top / bot; + temp = top % bot; + top = bot; + bot = temp; + + // move remainders + temp = est * denom_estimate + denom_last; + denom_last = denom_estimate; + denom_estimate = temp; + + // move remainders + temp = est * numer_estimate + numer_last; + numer_last = numer_estimate; + numer_estimate = temp; + } + + // we didn't fine anything good enough for float, use a full range estimate + if ( limit_denom ) + { + numer_estimate= (stbir_uint64)( f * (double)limit + 0.5 ); + denom_estimate = limit; + } + else + { + numer_estimate = limit; + denom_estimate = (stbir_uint64)( ( (double)limit / f ) + 0.5 ); + } + + *numer = (stbir_uint32) numer_estimate; + *denom = (stbir_uint32) denom_estimate; + + err = ( denom_estimate ) ? ( ( (double)(stbir_uint32)numer_estimate / (double)(stbir_uint32)denom_estimate ) - f ) : 1.0; + if ( err < 0.0 ) err = -err; + return ( err < ( 1.0 / (double)(1<<24) ) ) ? 1 : 0; +} + +static int stbir__calculate_region_transform( stbir__scale_info * scale_info, int output_full_range, int * output_offset, int output_sub_range, int input_full_range, double input_s0, double input_s1 ) +{ + double output_range, input_range, output_s, input_s, ratio, scale; + + input_s = input_s1 - input_s0; + + // null area + if ( ( output_full_range == 0 ) || ( input_full_range == 0 ) || + ( output_sub_range == 0 ) || ( input_s <= stbir__small_float ) ) + return 0; + + // are either of the ranges completely out of bounds? + if ( ( *output_offset >= output_full_range ) || ( ( *output_offset + output_sub_range ) <= 0 ) || ( input_s0 >= (1.0f-stbir__small_float) ) || ( input_s1 <= stbir__small_float ) ) + return 0; + + output_range = (double)output_full_range; + input_range = (double)input_full_range; + + output_s = ( (double)output_sub_range) / output_range; + + // figure out the scaling to use + ratio = output_s / input_s; + + // save scale before clipping + scale = ( output_range / input_range ) * ratio; + scale_info->scale = (float)scale; + scale_info->inv_scale = (float)( 1.0 / scale ); + + // clip output area to left/right output edges (and adjust input area) + stbir__clip( output_offset, &output_sub_range, output_full_range, &input_s0, &input_s1 ); + + // recalc input area + input_s = input_s1 - input_s0; + + // after clipping do we have zero input area? + if ( input_s <= stbir__small_float ) + return 0; + + // calculate and store the starting source offsets in output pixel space + scale_info->pixel_shift = (float) ( input_s0 * ratio * output_range ); + + scale_info->scale_is_rational = stbir__double_to_rational( scale, ( scale <= 1.0 ) ? output_full_range : input_full_range, &scale_info->scale_numerator, &scale_info->scale_denominator, ( scale >= 1.0 ) ); + + scale_info->input_full_size = input_full_range; + scale_info->output_sub_size = output_sub_range; + + return 1; +} + + +static void stbir__init_and_set_layout( STBIR_RESIZE * resize, stbir_pixel_layout pixel_layout, stbir_datatype data_type ) +{ + resize->input_cb = 0; + resize->output_cb = 0; + resize->user_data = resize; + resize->samplers = 0; + resize->called_alloc = 0; + resize->horizontal_filter = STBIR_FILTER_DEFAULT; + resize->horizontal_filter_kernel = 0; resize->horizontal_filter_support = 0; + resize->vertical_filter = STBIR_FILTER_DEFAULT; + resize->vertical_filter_kernel = 0; resize->vertical_filter_support = 0; + resize->horizontal_edge = STBIR_EDGE_CLAMP; + resize->vertical_edge = STBIR_EDGE_CLAMP; + resize->input_s0 = 0; resize->input_t0 = 0; resize->input_s1 = 1; resize->input_t1 = 1; + resize->output_subx = 0; resize->output_suby = 0; resize->output_subw = resize->output_w; resize->output_subh = resize->output_h; + resize->input_data_type = data_type; + resize->output_data_type = data_type; + resize->input_pixel_layout_public = pixel_layout; + resize->output_pixel_layout_public = pixel_layout; + resize->needs_rebuild = 1; +} + +STBIRDEF void stbir_resize_init( STBIR_RESIZE * resize, + const void *input_pixels, int input_w, int input_h, int input_stride_in_bytes, // stride can be zero + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, // stride can be zero + stbir_pixel_layout pixel_layout, stbir_datatype data_type ) +{ + resize->input_pixels = input_pixels; + resize->input_w = input_w; + resize->input_h = input_h; + resize->input_stride_in_bytes = input_stride_in_bytes; + resize->output_pixels = output_pixels; + resize->output_w = output_w; + resize->output_h = output_h; + resize->output_stride_in_bytes = output_stride_in_bytes; + resize->fast_alpha = 0; + + stbir__init_and_set_layout( resize, pixel_layout, data_type ); +} + +// You can update parameters any time after resize_init +STBIRDEF void stbir_set_datatypes( STBIR_RESIZE * resize, stbir_datatype input_type, stbir_datatype output_type ) // by default, datatype from resize_init +{ + resize->input_data_type = input_type; + resize->output_data_type = output_type; + if ( ( resize->samplers ) && ( !resize->needs_rebuild ) ) + stbir__update_info_from_resize( resize->samplers, resize ); +} + +STBIRDEF void stbir_set_pixel_callbacks( STBIR_RESIZE * resize, stbir_input_callback * input_cb, stbir_output_callback * output_cb ) // no callbacks by default +{ + resize->input_cb = input_cb; + resize->output_cb = output_cb; + + if ( ( resize->samplers ) && ( !resize->needs_rebuild ) ) + { + resize->samplers->in_pixels_cb = input_cb; + resize->samplers->out_pixels_cb = output_cb; + } +} + +STBIRDEF void stbir_set_user_data( STBIR_RESIZE * resize, void * user_data ) // pass back STBIR_RESIZE* by default +{ + resize->user_data = user_data; + if ( ( resize->samplers ) && ( !resize->needs_rebuild ) ) + resize->samplers->user_data = user_data; +} + +STBIRDEF void stbir_set_buffer_ptrs( STBIR_RESIZE * resize, const void * input_pixels, int input_stride_in_bytes, void * output_pixels, int output_stride_in_bytes ) +{ + resize->input_pixels = input_pixels; + resize->input_stride_in_bytes = input_stride_in_bytes; + resize->output_pixels = output_pixels; + resize->output_stride_in_bytes = output_stride_in_bytes; + if ( ( resize->samplers ) && ( !resize->needs_rebuild ) ) + stbir__update_info_from_resize( resize->samplers, resize ); +} + + +STBIRDEF int stbir_set_edgemodes( STBIR_RESIZE * resize, stbir_edge horizontal_edge, stbir_edge vertical_edge ) // CLAMP by default +{ + resize->horizontal_edge = horizontal_edge; + resize->vertical_edge = vertical_edge; + resize->needs_rebuild = 1; + return 1; +} + +STBIRDEF int stbir_set_filters( STBIR_RESIZE * resize, stbir_filter horizontal_filter, stbir_filter vertical_filter ) // STBIR_DEFAULT_FILTER_UPSAMPLE/DOWNSAMPLE by default +{ + resize->horizontal_filter = horizontal_filter; + resize->vertical_filter = vertical_filter; + resize->needs_rebuild = 1; + return 1; +} + +STBIRDEF int stbir_set_filter_callbacks( STBIR_RESIZE * resize, stbir__kernel_callback * horizontal_filter, stbir__support_callback * horizontal_support, stbir__kernel_callback * vertical_filter, stbir__support_callback * vertical_support ) +{ + resize->horizontal_filter_kernel = horizontal_filter; resize->horizontal_filter_support = horizontal_support; + resize->vertical_filter_kernel = vertical_filter; resize->vertical_filter_support = vertical_support; + resize->needs_rebuild = 1; + return 1; +} + +STBIRDEF int stbir_set_pixel_layouts( STBIR_RESIZE * resize, stbir_pixel_layout input_pixel_layout, stbir_pixel_layout output_pixel_layout ) // sets new pixel layouts +{ + resize->input_pixel_layout_public = input_pixel_layout; + resize->output_pixel_layout_public = output_pixel_layout; + resize->needs_rebuild = 1; + return 1; +} + + +STBIRDEF int stbir_set_non_pm_alpha_speed_over_quality( STBIR_RESIZE * resize, int non_pma_alpha_speed_over_quality ) // sets alpha speed +{ + resize->fast_alpha = non_pma_alpha_speed_over_quality; + resize->needs_rebuild = 1; + return 1; +} + +STBIRDEF int stbir_set_input_subrect( STBIR_RESIZE * resize, double s0, double t0, double s1, double t1 ) // sets input region (full region by default) +{ + resize->input_s0 = s0; + resize->input_t0 = t0; + resize->input_s1 = s1; + resize->input_t1 = t1; + resize->needs_rebuild = 1; + + // are we inbounds? + if ( ( s1 < stbir__small_float ) || ( (s1-s0) < stbir__small_float ) || + ( t1 < stbir__small_float ) || ( (t1-t0) < stbir__small_float ) || + ( s0 > (1.0f-stbir__small_float) ) || + ( t0 > (1.0f-stbir__small_float) ) ) + return 0; + + return 1; +} + +STBIRDEF int stbir_set_output_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ) // sets input region (full region by default) +{ + resize->output_subx = subx; + resize->output_suby = suby; + resize->output_subw = subw; + resize->output_subh = subh; + resize->needs_rebuild = 1; + + // are we inbounds? + if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw == 0 ) || ( subh == 0 ) ) + return 0; + + return 1; +} + +STBIRDEF int stbir_set_pixel_subrect( STBIR_RESIZE * resize, int subx, int suby, int subw, int subh ) // sets both regions (full regions by default) +{ + double s0, t0, s1, t1; + + s0 = ( (double)subx ) / ( (double)resize->output_w ); + t0 = ( (double)suby ) / ( (double)resize->output_h ); + s1 = ( (double)(subx+subw) ) / ( (double)resize->output_w ); + t1 = ( (double)(suby+subh) ) / ( (double)resize->output_h ); + + resize->input_s0 = s0; + resize->input_t0 = t0; + resize->input_s1 = s1; + resize->input_t1 = t1; + resize->output_subx = subx; + resize->output_suby = suby; + resize->output_subw = subw; + resize->output_subh = subh; + resize->needs_rebuild = 1; + + // are we inbounds? + if ( ( subx >= resize->output_w ) || ( ( subx + subw ) <= 0 ) || ( suby >= resize->output_h ) || ( ( suby + subh ) <= 0 ) || ( subw == 0 ) || ( subh == 0 ) ) + return 0; + + return 1; +} + +static int stbir__perform_build( STBIR_RESIZE * resize, int splits ) +{ + stbir__contributors conservative = { 0, 0 }; + stbir__sampler horizontal, vertical; + int new_output_subx, new_output_suby; + stbir__info * out_info; + #ifdef STBIR_PROFILE + stbir__info profile_infod; // used to contain building profile info before everything is allocated + stbir__info * profile_info = &profile_infod; + #endif + + // have we already built the samplers? + if ( resize->samplers ) + return 0; + + #define STBIR_RETURN_ERROR_AND_ASSERT( exp ) STBIR_ASSERT( !(exp) ); if (exp) return 0; + STBIR_RETURN_ERROR_AND_ASSERT( (unsigned)resize->horizontal_filter >= STBIR_FILTER_OTHER) + STBIR_RETURN_ERROR_AND_ASSERT( (unsigned)resize->vertical_filter >= STBIR_FILTER_OTHER) + #undef STBIR_RETURN_ERROR_AND_ASSERT + + if ( splits <= 0 ) + return 0; + + STBIR_PROFILE_BUILD_FIRST_START( build ); + + new_output_subx = resize->output_subx; + new_output_suby = resize->output_suby; + + // do horizontal clip and scale calcs + if ( !stbir__calculate_region_transform( &horizontal.scale_info, resize->output_w, &new_output_subx, resize->output_subw, resize->input_w, resize->input_s0, resize->input_s1 ) ) + return 0; + + // do vertical clip and scale calcs + if ( !stbir__calculate_region_transform( &vertical.scale_info, resize->output_h, &new_output_suby, resize->output_subh, resize->input_h, resize->input_t0, resize->input_t1 ) ) + return 0; + + // if nothing to do, just return + if ( ( horizontal.scale_info.output_sub_size == 0 ) || ( vertical.scale_info.output_sub_size == 0 ) ) + return 0; + + stbir__set_sampler(&horizontal, resize->horizontal_filter, resize->horizontal_filter_kernel, resize->horizontal_filter_support, resize->horizontal_edge, &horizontal.scale_info, 1, resize->user_data ); + stbir__get_conservative_extents( &horizontal, &conservative, resize->user_data ); + stbir__set_sampler(&vertical, resize->vertical_filter, resize->horizontal_filter_kernel, resize->vertical_filter_support, resize->vertical_edge, &vertical.scale_info, 0, resize->user_data ); + + if ( ( vertical.scale_info.output_sub_size / splits ) < STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS ) // each split should be a minimum of 4 scanlines (handwavey choice) + { + splits = vertical.scale_info.output_sub_size / STBIR_FORCE_MINIMUM_SCANLINES_FOR_SPLITS; + if ( splits == 0 ) splits = 1; + } + + STBIR_PROFILE_BUILD_START( alloc ); + out_info = stbir__alloc_internal_mem_and_build_samplers( &horizontal, &vertical, &conservative, resize->input_pixel_layout_public, resize->output_pixel_layout_public, splits, new_output_subx, new_output_suby, resize->fast_alpha, resize->user_data STBIR_ONLY_PROFILE_BUILD_SET_INFO ); + STBIR_PROFILE_BUILD_END( alloc ); + STBIR_PROFILE_BUILD_END( build ); + + if ( out_info ) + { + resize->splits = splits; + resize->samplers = out_info; + resize->needs_rebuild = 0; + #ifdef STBIR_PROFILE + STBIR_MEMCPY( &out_info->profile, &profile_infod.profile, sizeof( out_info->profile ) ); + #endif + + // update anything that can be changed without recalcing samplers + stbir__update_info_from_resize( out_info, resize ); + + return splits; + } + + return 0; +} + +void stbir_free_samplers( STBIR_RESIZE * resize ) +{ + if ( resize->samplers ) + { + stbir__free_internal_mem( resize->samplers ); + resize->samplers = 0; + resize->called_alloc = 0; + } +} + +STBIRDEF int stbir_build_samplers_with_splits( STBIR_RESIZE * resize, int splits ) +{ + if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) ) + { + if ( resize->samplers ) + stbir_free_samplers( resize ); + + resize->called_alloc = 1; + return stbir__perform_build( resize, splits ); + } + + STBIR_PROFILE_BUILD_CLEAR( resize->samplers ); + + return 1; +} + +STBIRDEF int stbir_build_samplers( STBIR_RESIZE * resize ) +{ + return stbir_build_samplers_with_splits( resize, 1 ); +} + +STBIRDEF int stbir_resize_extended( STBIR_RESIZE * resize ) +{ + int result; + + if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) ) + { + int alloc_state = resize->called_alloc; // remember allocated state + + if ( resize->samplers ) + { + stbir__free_internal_mem( resize->samplers ); + resize->samplers = 0; + } + + if ( !stbir_build_samplers( resize ) ) + return 0; + + resize->called_alloc = alloc_state; + + // if build_samplers succeeded (above), but there are no samplers set, then + // the area to stretch into was zero pixels, so don't do anything and return + // success + if ( resize->samplers == 0 ) + return 1; + } + else + { + // didn't build anything - clear it + STBIR_PROFILE_BUILD_CLEAR( resize->samplers ); + } + + // do resize + result = stbir__perform_resize( resize->samplers, 0, resize->splits ); + + // if we alloced, then free + if ( !resize->called_alloc ) + { + stbir_free_samplers( resize ); + resize->samplers = 0; + } + + return result; +} + +STBIRDEF int stbir_resize_extended_split( STBIR_RESIZE * resize, int split_start, int split_count ) +{ + STBIR_ASSERT( resize->samplers ); + + // if we're just doing the whole thing, call full + if ( ( split_start == -1 ) || ( ( split_start == 0 ) && ( split_count == resize->splits ) ) ) + return stbir_resize_extended( resize ); + + // you **must** build samplers first when using split resize + if ( ( resize->samplers == 0 ) || ( resize->needs_rebuild ) ) + return 0; + + if ( ( split_start >= resize->splits ) || ( split_start < 0 ) || ( ( split_start + split_count ) > resize->splits ) || ( split_count <= 0 ) ) + return 0; + + // do resize + return stbir__perform_resize( resize->samplers, split_start, split_count ); +} + +static int stbir__check_output_stuff( void ** ret_ptr, int * ret_pitch, void * output_pixels, int type_size, int output_w, int output_h, int output_stride_in_bytes, stbir_internal_pixel_layout pixel_layout ) +{ + size_t size; + int pitch; + void * ptr; + + pitch = output_w * type_size * stbir__pixel_channels[ pixel_layout ]; + if ( pitch == 0 ) + return 0; + + if ( output_stride_in_bytes == 0 ) + output_stride_in_bytes = pitch; + + if ( output_stride_in_bytes < pitch ) + return 0; + + size = (size_t)output_stride_in_bytes * (size_t)output_h; + if ( size == 0 ) + return 0; + + *ret_ptr = 0; + *ret_pitch = output_stride_in_bytes; + + if ( output_pixels == 0 ) + { + ptr = STBIR_MALLOC( size, 0 ); + if ( ptr == 0 ) + return 0; + + *ret_ptr = ptr; + *ret_pitch = pitch; + } + + return 1; +} + + +STBIRDEF unsigned char * stbir_resize_uint8_linear( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_layout ) +{ + STBIR_RESIZE resize; + unsigned char * optr; + int opitch; + + if ( !stbir__check_output_stuff( (void**)&optr, &opitch, output_pixels, sizeof( unsigned char ), output_w, output_h, output_stride_in_bytes, stbir__pixel_layout_convert_public_to_internal[ pixel_layout ] ) ) + return 0; + + stbir_resize_init( &resize, + input_pixels, input_w, input_h, input_stride_in_bytes, + (optr) ? optr : output_pixels, output_w, output_h, opitch, + pixel_layout, STBIR_TYPE_UINT8 ); + + if ( !stbir_resize_extended( &resize ) ) + { + if ( optr ) + STBIR_FREE( optr, 0 ); + return 0; + } + + return (optr) ? optr : output_pixels; +} + +STBIRDEF unsigned char * stbir_resize_uint8_srgb( const unsigned char *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_layout ) +{ + STBIR_RESIZE resize; + unsigned char * optr; + int opitch; + + if ( !stbir__check_output_stuff( (void**)&optr, &opitch, output_pixels, sizeof( unsigned char ), output_w, output_h, output_stride_in_bytes, stbir__pixel_layout_convert_public_to_internal[ pixel_layout ] ) ) + return 0; + + stbir_resize_init( &resize, + input_pixels, input_w, input_h, input_stride_in_bytes, + (optr) ? optr : output_pixels, output_w, output_h, opitch, + pixel_layout, STBIR_TYPE_UINT8_SRGB ); + + if ( !stbir_resize_extended( &resize ) ) + { + if ( optr ) + STBIR_FREE( optr, 0 ); + return 0; + } + + return (optr) ? optr : output_pixels; +} + + +STBIRDEF float * stbir_resize_float_linear( const float *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_layout ) +{ + STBIR_RESIZE resize; + float * optr; + int opitch; + + if ( !stbir__check_output_stuff( (void**)&optr, &opitch, output_pixels, sizeof( float ), output_w, output_h, output_stride_in_bytes, stbir__pixel_layout_convert_public_to_internal[ pixel_layout ] ) ) + return 0; + + stbir_resize_init( &resize, + input_pixels, input_w, input_h, input_stride_in_bytes, + (optr) ? optr : output_pixels, output_w, output_h, opitch, + pixel_layout, STBIR_TYPE_FLOAT ); + + if ( !stbir_resize_extended( &resize ) ) + { + if ( optr ) + STBIR_FREE( optr, 0 ); + return 0; + } + + return (optr) ? optr : output_pixels; +} + + +STBIRDEF void * stbir_resize( const void *input_pixels , int input_w , int input_h, int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_pixel_layout pixel_layout, stbir_datatype data_type, + stbir_edge edge, stbir_filter filter ) +{ + STBIR_RESIZE resize; + float * optr; + int opitch; + + if ( !stbir__check_output_stuff( (void**)&optr, &opitch, output_pixels, stbir__type_size[data_type], output_w, output_h, output_stride_in_bytes, stbir__pixel_layout_convert_public_to_internal[ pixel_layout ] ) ) + return 0; + + stbir_resize_init( &resize, + input_pixels, input_w, input_h, input_stride_in_bytes, + (optr) ? optr : output_pixels, output_w, output_h, output_stride_in_bytes, + pixel_layout, data_type ); + + resize.horizontal_edge = edge; + resize.vertical_edge = edge; + resize.horizontal_filter = filter; + resize.vertical_filter = filter; + + if ( !stbir_resize_extended( &resize ) ) + { + if ( optr ) + STBIR_FREE( optr, 0 ); + return 0; + } + + return (optr) ? optr : output_pixels; +} + +#ifdef STBIR_PROFILE + +STBIRDEF void stbir_resize_build_profile_info( STBIR_PROFILE_INFO * info, STBIR_RESIZE const * resize ) +{ + static char const * bdescriptions[6] = { "Building", "Allocating", "Horizontal sampler", "Vertical sampler", "Coefficient cleanup", "Coefficient piovot" } ; + stbir__info* samp = resize->samplers; + int i; + + typedef int testa[ (STBIR__ARRAY_SIZE( bdescriptions ) == (STBIR__ARRAY_SIZE( samp->profile.array )-1) )?1:-1]; + typedef int testb[ (sizeof( samp->profile.array ) == (sizeof(samp->profile.named)) )?1:-1]; + typedef int testc[ (sizeof( info->clocks ) >= (sizeof(samp->profile.named)) )?1:-1]; + + for( i = 0 ; i < STBIR__ARRAY_SIZE( bdescriptions ) ; i++) + info->clocks[i] = samp->profile.array[i+1]; + + info->total_clocks = samp->profile.named.total; + info->descriptions = bdescriptions; + info->count = STBIR__ARRAY_SIZE( bdescriptions ); +} + +STBIRDEF void stbir_resize_split_profile_info( STBIR_PROFILE_INFO * info, STBIR_RESIZE const * resize, int split_start, int split_count ) +{ + static char const * descriptions[7] = { "Looping", "Vertical sampling", "Horizontal sampling", "Scanline input", "Scanline output", "Alpha weighting", "Alpha unweighting" }; + stbir__per_split_info * split_info; + int s, i; + + typedef int testa[ (STBIR__ARRAY_SIZE( descriptions ) == (STBIR__ARRAY_SIZE( split_info->profile.array )-1) )?1:-1]; + typedef int testb[ (sizeof( split_info->profile.array ) == (sizeof(split_info->profile.named)) )?1:-1]; + typedef int testc[ (sizeof( info->clocks ) >= (sizeof(split_info->profile.named)) )?1:-1]; + + if ( split_start == -1 ) + { + split_start = 0; + split_count = resize->samplers->splits; + } + + if ( ( split_start >= resize->splits ) || ( split_start < 0 ) || ( ( split_start + split_count ) > resize->splits ) || ( split_count <= 0 ) ) + { + info->total_clocks = 0; + info->descriptions = 0; + info->count = 0; + return; + } + + split_info = resize->samplers->split_info + split_start; + + // sum up the profile from all the splits + for( i = 0 ; i < STBIR__ARRAY_SIZE( descriptions ) ; i++ ) + { + stbir_uint64 sum = 0; + for( s = 0 ; s < split_count ; s++ ) + sum += split_info[s].profile.array[i+1]; + info->clocks[i] = sum; + } + + info->total_clocks = split_info->profile.named.total; + info->descriptions = descriptions; + info->count = STBIR__ARRAY_SIZE( descriptions ); +} + +STBIRDEF void stbir_resize_extended_profile_info( STBIR_PROFILE_INFO * info, STBIR_RESIZE const * resize ) +{ + stbir_resize_split_profile_info( info, resize, -1, 0 ); +} + +#endif // STBIR_PROFILE + +#undef STBIR_BGR +#undef STBIR_1CHANNEL +#undef STBIR_2CHANNEL +#undef STBIR_RGB +#undef STBIR_RGBA +#undef STBIR_4CHANNEL +#undef STBIR_BGRA +#undef STBIR_ARGB +#undef STBIR_ABGR +#undef STBIR_RA +#undef STBIR_AR +#undef STBIR_RGBA_PM +#undef STBIR_BGRA_PM +#undef STBIR_ARGB_PM +#undef STBIR_ABGR_PM +#undef STBIR_RA_PM +#undef STBIR_AR_PM + +#endif // STB_IMAGE_RESIZE_IMPLEMENTATION + +#else // STB_IMAGE_RESIZE_HORIZONTALS&STB_IMAGE_RESIZE_DO_VERTICALS + +// we reinclude the header file to define all the horizontal functions +// specializing each function for the number of coeffs is 20-40% faster *OVERALL* + +// by including the header file again this way, we can still debug the functions + +#define STBIR_strs_join2( start, mid, end ) start##mid##end +#define STBIR_strs_join1( start, mid, end ) STBIR_strs_join2( start, mid, end ) + +#define STBIR_strs_join24( start, mid1, mid2, end ) start##mid1##mid2##end +#define STBIR_strs_join14( start, mid1, mid2, end ) STBIR_strs_join24( start, mid1, mid2, end ) + +#ifdef STB_IMAGE_RESIZE_DO_CODERS + +#ifdef stbir__decode_suffix +#define STBIR__CODER_NAME( name ) STBIR_strs_join1( name, _, stbir__decode_suffix ) +#else +#define STBIR__CODER_NAME( name ) name +#endif + +#ifdef stbir__decode_swizzle +#define stbir__decode_simdf8_flip(reg) STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( stbir__simdf8_0123to,stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3),stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3)(reg, reg) +#define stbir__decode_simdf4_flip(reg) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf_0123to,stbir__decode_order0,stbir__decode_order1),stbir__decode_order2,stbir__decode_order3)(reg, reg) +#define stbir__encode_simdf8_unflip(reg) STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( STBIR_strs_join1( stbir__simdf8_0123to,stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3),stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3)(reg, reg) +#define stbir__encode_simdf4_unflip(reg) STBIR_strs_join1( STBIR_strs_join1( stbir__simdf_0123to,stbir__encode_order0,stbir__encode_order1),stbir__encode_order2,stbir__encode_order3)(reg, reg) +#else +#define stbir__decode_order0 0 +#define stbir__decode_order1 1 +#define stbir__decode_order2 2 +#define stbir__decode_order3 3 +#define stbir__encode_order0 0 +#define stbir__encode_order1 1 +#define stbir__encode_order2 2 +#define stbir__encode_order3 3 +#define stbir__decode_simdf8_flip(reg) +#define stbir__decode_simdf4_flip(reg) +#define stbir__encode_simdf8_unflip(reg) +#define stbir__encode_simdf4_unflip(reg) +#endif + +#ifdef STBIR_SIMD8 +#define stbir__encode_simdfX_unflip stbir__encode_simdf8_unflip +#else +#define stbir__encode_simdfX_unflip stbir__encode_simdf4_unflip +#endif + +static void STBIR__CODER_NAME( stbir__decode_uint8_linear_scaled )( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float * decode_end = (float*) decode + width_times_channels; + unsigned char const * input = (unsigned char const*)inputp; + + #ifdef STBIR_SIMD + unsigned char const * end_input_m16 = input + width_times_channels - 16; + if ( width_times_channels >= 16 ) + { + decode_end -= 16; + STBIR_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + #ifdef STBIR_SIMD8 + stbir__simdi i; stbir__simdi8 o0,o1; + stbir__simdf8 of0, of1; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi8_expand_u8_to_u32( o0, o1, i ); + stbir__simdi8_convert_i32_to_float( of0, o0 ); + stbir__simdi8_convert_i32_to_float( of1, o1 ); + stbir__simdf8_mult( of0, of0, STBIR_max_uint8_as_float_inverted8); + stbir__simdf8_mult( of1, of1, STBIR_max_uint8_as_float_inverted8); + stbir__decode_simdf8_flip( of0 ); + stbir__decode_simdf8_flip( of1 ); + stbir__simdf8_store( decode + 0, of0 ); + stbir__simdf8_store( decode + 8, of1 ); + #else + stbir__simdi i, o0, o1, o2, o3; + stbir__simdf of0, of1, of2, of3; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi_expand_u8_to_u32( o0,o1,o2,o3,i); + stbir__simdi_convert_i32_to_float( of0, o0 ); + stbir__simdi_convert_i32_to_float( of1, o1 ); + stbir__simdi_convert_i32_to_float( of2, o2 ); + stbir__simdi_convert_i32_to_float( of3, o3 ); + stbir__simdf_mult( of0, of0, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) ); + stbir__simdf_mult( of1, of1, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) ); + stbir__simdf_mult( of2, of2, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) ); + stbir__simdf_mult( of3, of3, STBIR__CONSTF(STBIR_max_uint8_as_float_inverted) ); + stbir__decode_simdf4_flip( of0 ); + stbir__decode_simdf4_flip( of1 ); + stbir__decode_simdf4_flip( of2 ); + stbir__decode_simdf4_flip( of3 ); + stbir__simdf_store( decode + 0, of0 ); + stbir__simdf_store( decode + 4, of1 ); + stbir__simdf_store( decode + 8, of2 ); + stbir__simdf_store( decode + 12, of3 ); + #endif + decode += 16; + input += 16; + if ( decode <= decode_end ) + continue; + if ( decode == ( decode_end + 16 ) ) + break; + decode = decode_end; // backup and do last couple + input = end_input_m16; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode <= decode_end ) + { + STBIR_SIMD_NO_UNROLL(decode); + decode[0-4] = ((float)(input[stbir__decode_order0])) * stbir__max_uint8_as_float_inverted; + decode[1-4] = ((float)(input[stbir__decode_order1])) * stbir__max_uint8_as_float_inverted; + decode[2-4] = ((float)(input[stbir__decode_order2])) * stbir__max_uint8_as_float_inverted; + decode[3-4] = ((float)(input[stbir__decode_order3])) * stbir__max_uint8_as_float_inverted; + decode += 4; + input += 4; + } + decode -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( decode < decode_end ) + { + STBIR_NO_UNROLL(decode); + decode[0] = ((float)(input[stbir__decode_order0])) * stbir__max_uint8_as_float_inverted; + #if stbir__coder_min_num >= 2 + decode[1] = ((float)(input[stbir__decode_order1])) * stbir__max_uint8_as_float_inverted; + #endif + #if stbir__coder_min_num >= 3 + decode[2] = ((float)(input[stbir__decode_order2])) * stbir__max_uint8_as_float_inverted; + #endif + decode += stbir__coder_min_num; + input += stbir__coder_min_num; + } + #endif +} + +static void STBIR__CODER_NAME( stbir__encode_uint8_linear_scaled )( void * outputp, int width_times_channels, float const * encode ) +{ + unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char *) outputp; + unsigned char * end_output = ( (unsigned char *) output ) + width_times_channels; + + #ifdef STBIR_SIMD + if ( width_times_channels >= stbir__simdfX_float_count*2 ) + { + float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; + end_output -= stbir__simdfX_float_count*2; + STBIR_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdfX e0, e1; + stbir__simdi i; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode ); + stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint8_as_floatX, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + #ifdef STBIR_SIMD8 + stbir__simdf8_pack_to_16bytes( i, e0, e1 ); + stbir__simdi_store( output, i ); + #else + stbir__simdf_pack_to_8bytes( i, e0, e1 ); + stbir__simdi_store2( output, i ); + #endif + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; + if ( output <= end_output ) + continue; + if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m8; + } + return; + } + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + stbir__simdf e0; + stbir__simdi i0; + STBIR_NO_UNROLL(encode); + stbir__simdf_load( e0, encode ); + stbir__simdf_madd( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), e0 ); + stbir__encode_simdf4_unflip( e0 ); + stbir__simdf_pack_to_8bytes( i0, e0, e0 ); // only use first 4 + *(int*)(output-4) = stbir__simdi_to_int( i0 ); + output += 4; + encode += 4; + } + output -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + stbir__simdf e0; + STBIR_NO_UNROLL(encode); + stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order0 ); output[0] = stbir__simdf_convert_float_to_uint8( e0 ); + #if stbir__coder_min_num >= 2 + stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order1 ); output[1] = stbir__simdf_convert_float_to_uint8( e0 ); + #endif + #if stbir__coder_min_num >= 3 + stbir__simdf_madd1_mem( e0, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint8_as_float), encode+stbir__encode_order2 ); output[2] = stbir__simdf_convert_float_to_uint8( e0 ); + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif + + #else + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + while( output <= end_output ) + { + float f; + f = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[0-4] = (unsigned char)f; + f = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[1-4] = (unsigned char)f; + f = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[2-4] = (unsigned char)f; + f = encode[stbir__encode_order3] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[3-4] = (unsigned char)f; + output += 4; + encode += 4; + } + output -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + float f; + STBIR_NO_UNROLL(encode); + f = encode[stbir__encode_order0] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[0] = (unsigned char)f; + #if stbir__coder_min_num >= 2 + f = encode[stbir__encode_order1] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[1] = (unsigned char)f; + #endif + #if stbir__coder_min_num >= 3 + f = encode[stbir__encode_order2] * stbir__max_uint8_as_float + 0.5f; STBIR_CLAMP(f, 0, 255); output[2] = (unsigned char)f; + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif + #endif +} + +static void STBIR__CODER_NAME(stbir__decode_uint8_linear)( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float * decode_end = (float*) decode + width_times_channels; + unsigned char const * input = (unsigned char const*)inputp; + + #ifdef STBIR_SIMD + unsigned char const * end_input_m16 = input + width_times_channels - 16; + if ( width_times_channels >= 16 ) + { + decode_end -= 16; + STBIR_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + #ifdef STBIR_SIMD8 + stbir__simdi i; stbir__simdi8 o0,o1; + stbir__simdf8 of0, of1; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi8_expand_u8_to_u32( o0, o1, i ); + stbir__simdi8_convert_i32_to_float( of0, o0 ); + stbir__simdi8_convert_i32_to_float( of1, o1 ); + stbir__decode_simdf8_flip( of0 ); + stbir__decode_simdf8_flip( of1 ); + stbir__simdf8_store( decode + 0, of0 ); + stbir__simdf8_store( decode + 8, of1 ); + #else + stbir__simdi i, o0, o1, o2, o3; + stbir__simdf of0, of1, of2, of3; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi_expand_u8_to_u32( o0,o1,o2,o3,i); + stbir__simdi_convert_i32_to_float( of0, o0 ); + stbir__simdi_convert_i32_to_float( of1, o1 ); + stbir__simdi_convert_i32_to_float( of2, o2 ); + stbir__simdi_convert_i32_to_float( of3, o3 ); + stbir__decode_simdf4_flip( of0 ); + stbir__decode_simdf4_flip( of1 ); + stbir__decode_simdf4_flip( of2 ); + stbir__decode_simdf4_flip( of3 ); + stbir__simdf_store( decode + 0, of0 ); + stbir__simdf_store( decode + 4, of1 ); + stbir__simdf_store( decode + 8, of2 ); + stbir__simdf_store( decode + 12, of3 ); +#endif + decode += 16; + input += 16; + if ( decode <= decode_end ) + continue; + if ( decode == ( decode_end + 16 ) ) + break; + decode = decode_end; // backup and do last couple + input = end_input_m16; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode <= decode_end ) + { + STBIR_SIMD_NO_UNROLL(decode); + decode[0-4] = ((float)(input[stbir__decode_order0])); + decode[1-4] = ((float)(input[stbir__decode_order1])); + decode[2-4] = ((float)(input[stbir__decode_order2])); + decode[3-4] = ((float)(input[stbir__decode_order3])); + decode += 4; + input += 4; + } + decode -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( decode < decode_end ) + { + STBIR_NO_UNROLL(decode); + decode[0] = ((float)(input[stbir__decode_order0])); + #if stbir__coder_min_num >= 2 + decode[1] = ((float)(input[stbir__decode_order1])); + #endif + #if stbir__coder_min_num >= 3 + decode[2] = ((float)(input[stbir__decode_order2])); + #endif + decode += stbir__coder_min_num; + input += stbir__coder_min_num; + } + #endif +} + +static void STBIR__CODER_NAME( stbir__encode_uint8_linear )( void * outputp, int width_times_channels, float const * encode ) +{ + unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char *) outputp; + unsigned char * end_output = ( (unsigned char *) output ) + width_times_channels; + + #ifdef STBIR_SIMD + if ( width_times_channels >= stbir__simdfX_float_count*2 ) + { + float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; + end_output -= stbir__simdfX_float_count*2; + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdfX e0, e1; + stbir__simdi i; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode ); + stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + #ifdef STBIR_SIMD8 + stbir__simdf8_pack_to_16bytes( i, e0, e1 ); + stbir__simdi_store( output, i ); + #else + stbir__simdf_pack_to_8bytes( i, e0, e1 ); + stbir__simdi_store2( output, i ); + #endif + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; + if ( output <= end_output ) + continue; + if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m8; + } + return; + } + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + stbir__simdf e0; + stbir__simdi i0; + STBIR_NO_UNROLL(encode); + stbir__simdf_load( e0, encode ); + stbir__simdf_add( e0, STBIR__CONSTF(STBIR_simd_point5), e0 ); + stbir__encode_simdf4_unflip( e0 ); + stbir__simdf_pack_to_8bytes( i0, e0, e0 ); // only use first 4 + *(int*)(output-4) = stbir__simdi_to_int( i0 ); + output += 4; + encode += 4; + } + output -= 4; + #endif + + #else + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + while( output <= end_output ) + { + float f; + f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 255); output[0-4] = (unsigned char)f; + f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 255); output[1-4] = (unsigned char)f; + f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 255); output[2-4] = (unsigned char)f; + f = encode[stbir__encode_order3] + 0.5f; STBIR_CLAMP(f, 0, 255); output[3-4] = (unsigned char)f; + output += 4; + encode += 4; + } + output -= 4; + #endif + + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + float f; + STBIR_NO_UNROLL(encode); + f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 255); output[0] = (unsigned char)f; + #if stbir__coder_min_num >= 2 + f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 255); output[1] = (unsigned char)f; + #endif + #if stbir__coder_min_num >= 3 + f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 255); output[2] = (unsigned char)f; + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif +} + +static void STBIR__CODER_NAME(stbir__decode_uint8_srgb)( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float const * decode_end = (float*) decode + width_times_channels; + unsigned char const * input = (unsigned char const *)inputp; + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + decode += 4; + while( decode <= decode_end ) + { + decode[0-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order0 ] ]; + decode[1-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order1 ] ]; + decode[2-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order2 ] ]; + decode[3-4] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order3 ] ]; + decode += 4; + input += 4; + } + decode -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( decode < decode_end ) + { + STBIR_NO_UNROLL(decode); + decode[0] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order0 ] ]; + #if stbir__coder_min_num >= 2 + decode[1] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order1 ] ]; + #endif + #if stbir__coder_min_num >= 3 + decode[2] = stbir__srgb_uchar_to_linear_float[ input[ stbir__decode_order2 ] ]; + #endif + decode += stbir__coder_min_num; + input += stbir__coder_min_num; + } + #endif +} + +#define stbir__min_max_shift20( i, f ) \ + stbir__simdf_max( f, f, stbir_simdf_casti(STBIR__CONSTI( STBIR_almost_zero )) ); \ + stbir__simdf_min( f, f, stbir_simdf_casti(STBIR__CONSTI( STBIR_almost_one )) ); \ + stbir__simdi_32shr( i, stbir_simdi_castf( f ), 20 ); + +#define stbir__scale_and_convert( i, f ) \ + stbir__simdf_madd( f, STBIR__CONSTF( STBIR_simd_point5 ), STBIR__CONSTF( STBIR_max_uint8_as_float ), f ); \ + stbir__simdf_max( f, f, stbir__simdf_zeroP() ); \ + stbir__simdf_min( f, f, STBIR__CONSTF( STBIR_max_uint8_as_float ) ); \ + stbir__simdf_convert_float_to_i32( i, f ); + +#define stbir__linear_to_srgb_finish( i, f ) \ +{ \ + stbir__simdi temp; \ + stbir__simdi_32shr( temp, stbir_simdi_castf( f ), 12 ) ; \ + stbir__simdi_and( temp, temp, STBIR__CONSTI(STBIR_mastissa_mask) ); \ + stbir__simdi_or( temp, temp, STBIR__CONSTI(STBIR_topscale) ); \ + stbir__simdi_16madd( i, i, temp ); \ + stbir__simdi_32shr( i, i, 16 ); \ +} + +#define stbir__simdi_table_lookup2( v0,v1, table ) \ +{ \ + stbir__simdi_u32 temp0,temp1; \ + temp0.m128i_i128 = v0; \ + temp1.m128i_i128 = v1; \ + temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \ + temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \ + v0 = temp0.m128i_i128; \ + v1 = temp1.m128i_i128; \ +} + +#define stbir__simdi_table_lookup3( v0,v1,v2, table ) \ +{ \ + stbir__simdi_u32 temp0,temp1,temp2; \ + temp0.m128i_i128 = v0; \ + temp1.m128i_i128 = v1; \ + temp2.m128i_i128 = v2; \ + temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \ + temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \ + temp2.m128i_u32[0] = table[temp2.m128i_i32[0]]; temp2.m128i_u32[1] = table[temp2.m128i_i32[1]]; temp2.m128i_u32[2] = table[temp2.m128i_i32[2]]; temp2.m128i_u32[3] = table[temp2.m128i_i32[3]]; \ + v0 = temp0.m128i_i128; \ + v1 = temp1.m128i_i128; \ + v2 = temp2.m128i_i128; \ +} + +#define stbir__simdi_table_lookup4( v0,v1,v2,v3, table ) \ +{ \ + stbir__simdi_u32 temp0,temp1,temp2,temp3; \ + temp0.m128i_i128 = v0; \ + temp1.m128i_i128 = v1; \ + temp2.m128i_i128 = v2; \ + temp3.m128i_i128 = v3; \ + temp0.m128i_u32[0] = table[temp0.m128i_i32[0]]; temp0.m128i_u32[1] = table[temp0.m128i_i32[1]]; temp0.m128i_u32[2] = table[temp0.m128i_i32[2]]; temp0.m128i_u32[3] = table[temp0.m128i_i32[3]]; \ + temp1.m128i_u32[0] = table[temp1.m128i_i32[0]]; temp1.m128i_u32[1] = table[temp1.m128i_i32[1]]; temp1.m128i_u32[2] = table[temp1.m128i_i32[2]]; temp1.m128i_u32[3] = table[temp1.m128i_i32[3]]; \ + temp2.m128i_u32[0] = table[temp2.m128i_i32[0]]; temp2.m128i_u32[1] = table[temp2.m128i_i32[1]]; temp2.m128i_u32[2] = table[temp2.m128i_i32[2]]; temp2.m128i_u32[3] = table[temp2.m128i_i32[3]]; \ + temp3.m128i_u32[0] = table[temp3.m128i_i32[0]]; temp3.m128i_u32[1] = table[temp3.m128i_i32[1]]; temp3.m128i_u32[2] = table[temp3.m128i_i32[2]]; temp3.m128i_u32[3] = table[temp3.m128i_i32[3]]; \ + v0 = temp0.m128i_i128; \ + v1 = temp1.m128i_i128; \ + v2 = temp2.m128i_i128; \ + v3 = temp3.m128i_i128; \ +} + +static void STBIR__CODER_NAME( stbir__encode_uint8_srgb )( void * outputp, int width_times_channels, float const * encode ) +{ + unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp; + unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels; + + #ifdef STBIR_SIMD + + if ( width_times_channels >= 16 ) + { + float const * end_encode_m16 = encode + width_times_channels - 16; + end_output -= 16; + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdf f0, f1, f2, f3; + stbir__simdi i0, i1, i2, i3; + STBIR_SIMD_NO_UNROLL(encode); + + stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); + + stbir__min_max_shift20( i0, f0 ); + stbir__min_max_shift20( i1, f1 ); + stbir__min_max_shift20( i2, f2 ); + stbir__min_max_shift20( i3, f3 ); + + stbir__simdi_table_lookup4( i0, i1, i2, i3, ( fp32_to_srgb8_tab4 - (127-13)*8 ) ); + + stbir__linear_to_srgb_finish( i0, f0 ); + stbir__linear_to_srgb_finish( i1, f1 ); + stbir__linear_to_srgb_finish( i2, f2 ); + stbir__linear_to_srgb_finish( i3, f3 ); + + stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); + + encode += 16; + output += 16; + if ( output <= end_output ) + continue; + if ( output == ( end_output + 16 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m16; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( output <= end_output ) + { + STBIR_SIMD_NO_UNROLL(encode); + + output[0-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order0] ); + output[1-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order1] ); + output[2-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order2] ); + output[3-4] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order3] ); + + output += 4; + encode += 4; + } + output -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + STBIR_NO_UNROLL(encode); + output[0] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order0] ); + #if stbir__coder_min_num >= 2 + output[1] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order1] ); + #endif + #if stbir__coder_min_num >= 3 + output[2] = stbir__linear_to_srgb_uchar( encode[stbir__encode_order2] ); + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif +} + +#if ( stbir__coder_min_num == 4 ) || ( ( stbir__coder_min_num == 1 ) && ( !defined(stbir__decode_swizzle) ) ) + +static void STBIR__CODER_NAME(stbir__decode_uint8_srgb4_linearalpha)( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float const * decode_end = (float*) decode + width_times_channels; + unsigned char const * input = (unsigned char const *)inputp; + do { + decode[0] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order0] ]; + decode[1] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order1] ]; + decode[2] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order2] ]; + decode[3] = ( (float) input[stbir__decode_order3] ) * stbir__max_uint8_as_float_inverted; + input += 4; + decode += 4; + } while( decode < decode_end ); +} + + +static void STBIR__CODER_NAME( stbir__encode_uint8_srgb4_linearalpha )( void * outputp, int width_times_channels, float const * encode ) +{ + unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp; + unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels; + + #ifdef STBIR_SIMD + + if ( width_times_channels >= 16 ) + { + float const * end_encode_m16 = encode + width_times_channels - 16; + end_output -= 16; + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdf f0, f1, f2, f3; + stbir__simdi i0, i1, i2, i3; + + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); + + stbir__min_max_shift20( i0, f0 ); + stbir__min_max_shift20( i1, f1 ); + stbir__min_max_shift20( i2, f2 ); + stbir__scale_and_convert( i3, f3 ); + + stbir__simdi_table_lookup3( i0, i1, i2, ( fp32_to_srgb8_tab4 - (127-13)*8 ) ); + + stbir__linear_to_srgb_finish( i0, f0 ); + stbir__linear_to_srgb_finish( i1, f1 ); + stbir__linear_to_srgb_finish( i2, f2 ); + + stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); + + output += 16; + encode += 16; + + if ( output <= end_output ) + continue; + if ( output == ( end_output + 16 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m16; + } + return; + } + #endif + + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float f; + STBIR_SIMD_NO_UNROLL(encode); + + output[stbir__decode_order0] = stbir__linear_to_srgb_uchar( encode[0] ); + output[stbir__decode_order1] = stbir__linear_to_srgb_uchar( encode[1] ); + output[stbir__decode_order2] = stbir__linear_to_srgb_uchar( encode[2] ); + + f = encode[3] * stbir__max_uint8_as_float + 0.5f; + STBIR_CLAMP(f, 0, 255); + output[stbir__decode_order3] = (unsigned char) f; + + output += 4; + encode += 4; + } while( output < end_output ); +} + +#endif + +#if ( stbir__coder_min_num == 2 ) || ( ( stbir__coder_min_num == 1 ) && ( !defined(stbir__decode_swizzle) ) ) + +static void STBIR__CODER_NAME(stbir__decode_uint8_srgb2_linearalpha)( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float const * decode_end = (float*) decode + width_times_channels; + unsigned char const * input = (unsigned char const *)inputp; + decode += 4; + while( decode <= decode_end ) + { + decode[0-4] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order0] ]; + decode[1-4] = ( (float) input[stbir__decode_order1] ) * stbir__max_uint8_as_float_inverted; + decode[2-4] = stbir__srgb_uchar_to_linear_float[ input[stbir__decode_order0+2] ]; + decode[3-4] = ( (float) input[stbir__decode_order1+2] ) * stbir__max_uint8_as_float_inverted; + input += 4; + decode += 4; + } + decode -= 4; + if( decode < decode_end ) + { + decode[0] = stbir__srgb_uchar_to_linear_float[ stbir__decode_order0 ]; + decode[1] = ( (float) input[stbir__decode_order1] ) * stbir__max_uint8_as_float_inverted; + } +} + +static void STBIR__CODER_NAME( stbir__encode_uint8_srgb2_linearalpha )( void * outputp, int width_times_channels, float const * encode ) +{ + unsigned char STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned char*) outputp; + unsigned char * end_output = ( (unsigned char*) output ) + width_times_channels; + + #ifdef STBIR_SIMD + + if ( width_times_channels >= 16 ) + { + float const * end_encode_m16 = encode + width_times_channels - 16; + end_output -= 16; + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdf f0, f1, f2, f3; + stbir__simdi i0, i1, i2, i3; + + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdf_load4_transposed( f0, f1, f2, f3, encode ); + + stbir__min_max_shift20( i0, f0 ); + stbir__scale_and_convert( i1, f1 ); + stbir__min_max_shift20( i2, f2 ); + stbir__scale_and_convert( i3, f3 ); + + stbir__simdi_table_lookup2( i0, i2, ( fp32_to_srgb8_tab4 - (127-13)*8 ) ); + + stbir__linear_to_srgb_finish( i0, f0 ); + stbir__linear_to_srgb_finish( i2, f2 ); + + stbir__interleave_pack_and_store_16_u8( output, STBIR_strs_join1(i, ,stbir__encode_order0), STBIR_strs_join1(i, ,stbir__encode_order1), STBIR_strs_join1(i, ,stbir__encode_order2), STBIR_strs_join1(i, ,stbir__encode_order3) ); + + output += 16; + encode += 16; + if ( output <= end_output ) + continue; + if ( output == ( end_output + 16 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m16; + } + return; + } + #endif + + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float f; + STBIR_SIMD_NO_UNROLL(encode); + + output[stbir__decode_order0] = stbir__linear_to_srgb_uchar( encode[0] ); + + f = encode[1] * stbir__max_uint8_as_float + 0.5f; + STBIR_CLAMP(f, 0, 255); + output[stbir__decode_order1] = (unsigned char) f; + + output += 2; + encode += 2; + } while( output < end_output ); +} + +#endif + +static void STBIR__CODER_NAME(stbir__decode_uint16_linear_scaled)( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float * decode_end = (float*) decode + width_times_channels; + unsigned short const * input = (unsigned short const *)inputp; + + #ifdef STBIR_SIMD + unsigned short const * end_input_m8 = input + width_times_channels - 8; + if ( width_times_channels >= 8 ) + { + decode_end -= 8; + STBIR_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + #ifdef STBIR_SIMD8 + stbir__simdi i; stbir__simdi8 o; + stbir__simdf8 of; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi8_expand_u16_to_u32( o, i ); + stbir__simdi8_convert_i32_to_float( of, o ); + stbir__simdf8_mult( of, of, STBIR_max_uint16_as_float_inverted8); + stbir__decode_simdf8_flip( of ); + stbir__simdf8_store( decode + 0, of ); + #else + stbir__simdi i, o0, o1; + stbir__simdf of0, of1; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi_expand_u16_to_u32( o0,o1,i ); + stbir__simdi_convert_i32_to_float( of0, o0 ); + stbir__simdi_convert_i32_to_float( of1, o1 ); + stbir__simdf_mult( of0, of0, STBIR__CONSTF(STBIR_max_uint16_as_float_inverted) ); + stbir__simdf_mult( of1, of1, STBIR__CONSTF(STBIR_max_uint16_as_float_inverted)); + stbir__decode_simdf4_flip( of0 ); + stbir__decode_simdf4_flip( of1 ); + stbir__simdf_store( decode + 0, of0 ); + stbir__simdf_store( decode + 4, of1 ); + #endif + decode += 8; + input += 8; + if ( decode <= decode_end ) + continue; + if ( decode == ( decode_end + 8 ) ) + break; + decode = decode_end; // backup and do last couple + input = end_input_m8; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode <= decode_end ) + { + STBIR_SIMD_NO_UNROLL(decode); + decode[0-4] = ((float)(input[stbir__decode_order0])) * stbir__max_uint16_as_float_inverted; + decode[1-4] = ((float)(input[stbir__decode_order1])) * stbir__max_uint16_as_float_inverted; + decode[2-4] = ((float)(input[stbir__decode_order2])) * stbir__max_uint16_as_float_inverted; + decode[3-4] = ((float)(input[stbir__decode_order3])) * stbir__max_uint16_as_float_inverted; + decode += 4; + input += 4; + } + decode -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( decode < decode_end ) + { + STBIR_NO_UNROLL(decode); + decode[0] = ((float)(input[stbir__decode_order0])) * stbir__max_uint16_as_float_inverted; + #if stbir__coder_min_num >= 2 + decode[1] = ((float)(input[stbir__decode_order1])) * stbir__max_uint16_as_float_inverted; + #endif + #if stbir__coder_min_num >= 3 + decode[2] = ((float)(input[stbir__decode_order2])) * stbir__max_uint16_as_float_inverted; + #endif + decode += stbir__coder_min_num; + input += stbir__coder_min_num; + } + #endif +} + + +static void STBIR__CODER_NAME(stbir__encode_uint16_linear_scaled)( void * outputp, int width_times_channels, float const * encode ) +{ + unsigned short STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned short*) outputp; + unsigned short * end_output = ( (unsigned short*) output ) + width_times_channels; + + #ifdef STBIR_SIMD + { + if ( width_times_channels >= stbir__simdfX_float_count*2 ) + { + float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; + end_output -= stbir__simdfX_float_count*2; + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdfX e0, e1; + stbir__simdiX i; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_madd_mem( e0, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode ); + stbir__simdfX_madd_mem( e1, STBIR_simd_point5X, STBIR_max_uint16_as_floatX, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + stbir__simdfX_pack_to_words( i, e0, e1 ); + stbir__simdiX_store( output, i ); + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; + if ( output <= end_output ) + continue; + if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m8; + } + return; + } + } + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + stbir__simdf e; + stbir__simdi i; + STBIR_NO_UNROLL(encode); + stbir__simdf_load( e, encode ); + stbir__simdf_madd( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), e ); + stbir__encode_simdf4_unflip( e ); + stbir__simdf_pack_to_8words( i, e, e ); // only use first 4 + stbir__simdi_store2( output-4, i ); + output += 4; + encode += 4; + } + output -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + stbir__simdf e; + STBIR_NO_UNROLL(encode); + stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order0 ); output[0] = stbir__simdf_convert_float_to_short( e ); + #if stbir__coder_min_num >= 2 + stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order1 ); output[1] = stbir__simdf_convert_float_to_short( e ); + #endif + #if stbir__coder_min_num >= 3 + stbir__simdf_madd1_mem( e, STBIR__CONSTF(STBIR_simd_point5), STBIR__CONSTF(STBIR_max_uint16_as_float), encode+stbir__encode_order2 ); output[2] = stbir__simdf_convert_float_to_short( e ); + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif + + #else + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + float f; + STBIR_SIMD_NO_UNROLL(encode); + f = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0-4] = (unsigned short)f; + f = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1-4] = (unsigned short)f; + f = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2-4] = (unsigned short)f; + f = encode[stbir__encode_order3] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[3-4] = (unsigned short)f; + output += 4; + encode += 4; + } + output -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + float f; + STBIR_NO_UNROLL(encode); + f = encode[stbir__encode_order0] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0] = (unsigned short)f; + #if stbir__coder_min_num >= 2 + f = encode[stbir__encode_order1] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1] = (unsigned short)f; + #endif + #if stbir__coder_min_num >= 3 + f = encode[stbir__encode_order2] * stbir__max_uint16_as_float + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2] = (unsigned short)f; + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif + #endif +} + +static void STBIR__CODER_NAME(stbir__decode_uint16_linear)( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float * decode_end = (float*) decode + width_times_channels; + unsigned short const * input = (unsigned short const *)inputp; + + #ifdef STBIR_SIMD + unsigned short const * end_input_m8 = input + width_times_channels - 8; + if ( width_times_channels >= 8 ) + { + decode_end -= 8; + STBIR_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + #ifdef STBIR_SIMD8 + stbir__simdi i; stbir__simdi8 o; + stbir__simdf8 of; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi8_expand_u16_to_u32( o, i ); + stbir__simdi8_convert_i32_to_float( of, o ); + stbir__decode_simdf8_flip( of ); + stbir__simdf8_store( decode + 0, of ); + #else + stbir__simdi i, o0, o1; + stbir__simdf of0, of1; + STBIR_NO_UNROLL(decode); + stbir__simdi_load( i, input ); + stbir__simdi_expand_u16_to_u32( o0, o1, i ); + stbir__simdi_convert_i32_to_float( of0, o0 ); + stbir__simdi_convert_i32_to_float( of1, o1 ); + stbir__decode_simdf4_flip( of0 ); + stbir__decode_simdf4_flip( of1 ); + stbir__simdf_store( decode + 0, of0 ); + stbir__simdf_store( decode + 4, of1 ); + #endif + decode += 8; + input += 8; + if ( decode <= decode_end ) + continue; + if ( decode == ( decode_end + 8 ) ) + break; + decode = decode_end; // backup and do last couple + input = end_input_m8; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode <= decode_end ) + { + STBIR_SIMD_NO_UNROLL(decode); + decode[0-4] = ((float)(input[stbir__decode_order0])); + decode[1-4] = ((float)(input[stbir__decode_order1])); + decode[2-4] = ((float)(input[stbir__decode_order2])); + decode[3-4] = ((float)(input[stbir__decode_order3])); + decode += 4; + input += 4; + } + decode -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( decode < decode_end ) + { + STBIR_NO_UNROLL(decode); + decode[0] = ((float)(input[stbir__decode_order0])); + #if stbir__coder_min_num >= 2 + decode[1] = ((float)(input[stbir__decode_order1])); + #endif + #if stbir__coder_min_num >= 3 + decode[2] = ((float)(input[stbir__decode_order2])); + #endif + decode += stbir__coder_min_num; + input += stbir__coder_min_num; + } + #endif +} + +static void STBIR__CODER_NAME(stbir__encode_uint16_linear)( void * outputp, int width_times_channels, float const * encode ) +{ + unsigned short STBIR_SIMD_STREAMOUT_PTR( * ) output = (unsigned short*) outputp; + unsigned short * end_output = ( (unsigned short*) output ) + width_times_channels; + + #ifdef STBIR_SIMD + { + if ( width_times_channels >= stbir__simdfX_float_count*2 ) + { + float const * end_encode_m8 = encode + width_times_channels - stbir__simdfX_float_count*2; + end_output -= stbir__simdfX_float_count*2; + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdfX e0, e1; + stbir__simdiX i; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_add_mem( e0, STBIR_simd_point5X, encode ); + stbir__simdfX_add_mem( e1, STBIR_simd_point5X, encode+stbir__simdfX_float_count ); + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + stbir__simdfX_pack_to_words( i, e0, e1 ); + stbir__simdiX_store( output, i ); + encode += stbir__simdfX_float_count*2; + output += stbir__simdfX_float_count*2; + if ( output <= end_output ) + continue; + if ( output == ( end_output + stbir__simdfX_float_count*2 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m8; + } + return; + } + } + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + stbir__simdf e; + stbir__simdi i; + STBIR_NO_UNROLL(encode); + stbir__simdf_load( e, encode ); + stbir__simdf_add( e, STBIR__CONSTF(STBIR_simd_point5), e ); + stbir__encode_simdf4_unflip( e ); + stbir__simdf_pack_to_8words( i, e, e ); // only use first 4 + stbir__simdi_store2( output-4, i ); + output += 4; + encode += 4; + } + output -= 4; + #endif + + #else + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + float f; + STBIR_SIMD_NO_UNROLL(encode); + f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0-4] = (unsigned short)f; + f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1-4] = (unsigned short)f; + f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2-4] = (unsigned short)f; + f = encode[stbir__encode_order3] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[3-4] = (unsigned short)f; + output += 4; + encode += 4; + } + output -= 4; + #endif + + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + float f; + STBIR_NO_UNROLL(encode); + f = encode[stbir__encode_order0] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[0] = (unsigned short)f; + #if stbir__coder_min_num >= 2 + f = encode[stbir__encode_order1] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[1] = (unsigned short)f; + #endif + #if stbir__coder_min_num >= 3 + f = encode[stbir__encode_order2] + 0.5f; STBIR_CLAMP(f, 0, 65535); output[2] = (unsigned short)f; + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif +} + +static void STBIR__CODER_NAME(stbir__decode_half_float_linear)( float * decodep, int width_times_channels, void const * inputp ) +{ + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float * decode_end = (float*) decode + width_times_channels; + stbir__FP16 const * input = (stbir__FP16 const *)inputp; + + #ifdef STBIR_SIMD + if ( width_times_channels >= 8 ) + { + stbir__FP16 const * end_input_m8 = input + width_times_channels - 8; + decode_end -= 8; + STBIR_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + STBIR_NO_UNROLL(decode); + + stbir__half_to_float_SIMD( decode, input ); + #ifdef stbir__decode_swizzle + #ifdef STBIR_SIMD8 + { + stbir__simdf8 of; + stbir__simdf8_load( of, decode ); + stbir__decode_simdf8_flip( of ); + stbir__simdf8_store( decode, of ); + } + #else + { + stbir__simdf of0,of1; + stbir__simdf_load( of0, decode ); + stbir__simdf_load( of1, decode+4 ); + stbir__decode_simdf4_flip( of0 ); + stbir__decode_simdf4_flip( of1 ); + stbir__simdf_store( decode, of0 ); + stbir__simdf_store( decode+4, of1 ); + } + #endif + #endif + decode += 8; + input += 8; + if ( decode <= decode_end ) + continue; + if ( decode == ( decode_end + 8 ) ) + break; + decode = decode_end; // backup and do last couple + input = end_input_m8; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode <= decode_end ) + { + STBIR_SIMD_NO_UNROLL(decode); + decode[0-4] = stbir__half_to_float(input[stbir__decode_order0]); + decode[1-4] = stbir__half_to_float(input[stbir__decode_order1]); + decode[2-4] = stbir__half_to_float(input[stbir__decode_order2]); + decode[3-4] = stbir__half_to_float(input[stbir__decode_order3]); + decode += 4; + input += 4; + } + decode -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( decode < decode_end ) + { + STBIR_NO_UNROLL(decode); + decode[0] = stbir__half_to_float(input[stbir__decode_order0]); + #if stbir__coder_min_num >= 2 + decode[1] = stbir__half_to_float(input[stbir__decode_order1]); + #endif + #if stbir__coder_min_num >= 3 + decode[2] = stbir__half_to_float(input[stbir__decode_order2]); + #endif + decode += stbir__coder_min_num; + input += stbir__coder_min_num; + } + #endif +} + +static void STBIR__CODER_NAME( stbir__encode_half_float_linear )( void * outputp, int width_times_channels, float const * encode ) +{ + stbir__FP16 STBIR_SIMD_STREAMOUT_PTR( * ) output = (stbir__FP16*) outputp; + stbir__FP16 * end_output = ( (stbir__FP16*) output ) + width_times_channels; + + #ifdef STBIR_SIMD + if ( width_times_channels >= 8 ) + { + float const * end_encode_m8 = encode + width_times_channels - 8; + end_output -= 8; + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + STBIR_SIMD_NO_UNROLL(encode); + #ifdef stbir__decode_swizzle + #ifdef STBIR_SIMD8 + { + stbir__simdf8 of; + stbir__simdf8_load( of, encode ); + stbir__encode_simdf8_unflip( of ); + stbir__float_to_half_SIMD( output, (float*)&of ); + } + #else + { + stbir__simdf of[2]; + stbir__simdf_load( of[0], encode ); + stbir__simdf_load( of[1], encode+4 ); + stbir__encode_simdf4_unflip( of[0] ); + stbir__encode_simdf4_unflip( of[1] ); + stbir__float_to_half_SIMD( output, (float*)of ); + } + #endif + #else + stbir__float_to_half_SIMD( output, encode ); + #endif + encode += 8; + output += 8; + if ( output <= end_output ) + continue; + if ( output == ( end_output + 8 ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m8; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + STBIR_SIMD_NO_UNROLL(output); + output[0-4] = stbir__float_to_half(encode[stbir__encode_order0]); + output[1-4] = stbir__float_to_half(encode[stbir__encode_order1]); + output[2-4] = stbir__float_to_half(encode[stbir__encode_order2]); + output[3-4] = stbir__float_to_half(encode[stbir__encode_order3]); + output += 4; + encode += 4; + } + output -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + STBIR_NO_UNROLL(output); + output[0] = stbir__float_to_half(encode[stbir__encode_order0]); + #if stbir__coder_min_num >= 2 + output[1] = stbir__float_to_half(encode[stbir__encode_order1]); + #endif + #if stbir__coder_min_num >= 3 + output[2] = stbir__float_to_half(encode[stbir__encode_order2]); + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif +} + +static void STBIR__CODER_NAME(stbir__decode_float_linear)( float * decodep, int width_times_channels, void const * inputp ) +{ + #ifdef stbir__decode_swizzle + float STBIR_STREAMOUT_PTR( * ) decode = decodep; + float * decode_end = (float*) decode + width_times_channels; + float const * input = (float const *)inputp; + + #ifdef STBIR_SIMD + if ( width_times_channels >= 16 ) + { + float const * end_input_m16 = input + width_times_channels - 16; + decode_end -= 16; + STBIR_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + STBIR_NO_UNROLL(decode); + #ifdef stbir__decode_swizzle + #ifdef STBIR_SIMD8 + { + stbir__simdf8 of0,of1; + stbir__simdf8_load( of0, input ); + stbir__simdf8_load( of1, input+8 ); + stbir__decode_simdf8_flip( of0 ); + stbir__decode_simdf8_flip( of1 ); + stbir__simdf8_store( decode, of0 ); + stbir__simdf8_store( decode+8, of1 ); + } + #else + { + stbir__simdf of0,of1,of2,of3; + stbir__simdf_load( of0, input ); + stbir__simdf_load( of1, input+4 ); + stbir__simdf_load( of2, input+8 ); + stbir__simdf_load( of3, input+12 ); + stbir__decode_simdf4_flip( of0 ); + stbir__decode_simdf4_flip( of1 ); + stbir__decode_simdf4_flip( of2 ); + stbir__decode_simdf4_flip( of3 ); + stbir__simdf_store( decode, of0 ); + stbir__simdf_store( decode+4, of1 ); + stbir__simdf_store( decode+8, of2 ); + stbir__simdf_store( decode+12, of3 ); + } + #endif + #endif + decode += 16; + input += 16; + if ( decode <= decode_end ) + continue; + if ( decode == ( decode_end + 16 ) ) + break; + decode = decode_end; // backup and do last couple + input = end_input_m16; + } + return; + } + #endif + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + decode += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( decode <= decode_end ) + { + STBIR_SIMD_NO_UNROLL(decode); + decode[0-4] = input[stbir__decode_order0]; + decode[1-4] = input[stbir__decode_order1]; + decode[2-4] = input[stbir__decode_order2]; + decode[3-4] = input[stbir__decode_order3]; + decode += 4; + input += 4; + } + decode -= 4; + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( decode < decode_end ) + { + STBIR_NO_UNROLL(decode); + decode[0] = input[stbir__decode_order0]; + #if stbir__coder_min_num >= 2 + decode[1] = input[stbir__decode_order1]; + #endif + #if stbir__coder_min_num >= 3 + decode[2] = input[stbir__decode_order2]; + #endif + decode += stbir__coder_min_num; + input += stbir__coder_min_num; + } + #endif + + #else + + if ( (void*)decodep != inputp ) + STBIR_MEMCPY( decodep, inputp, width_times_channels * sizeof( float ) ); + + #endif +} + +static void STBIR__CODER_NAME( stbir__encode_float_linear )( void * outputp, int width_times_channels, float const * encode ) +{ + #if !defined( STBIR_FLOAT_HIGH_CLAMP ) && !defined(STBIR_FLOAT_LO_CLAMP) && !defined(stbir__decode_swizzle) + + if ( (void*)outputp != (void*) encode ) + STBIR_MEMCPY( outputp, encode, width_times_channels * sizeof( float ) ); + + #else + + float STBIR_SIMD_STREAMOUT_PTR( * ) output = (float*) outputp; + float * end_output = ( (float*) output ) + width_times_channels; + + #ifdef STBIR_FLOAT_HIGH_CLAMP + #define stbir_scalar_hi_clamp( v ) if ( v > STBIR_FLOAT_HIGH_CLAMP ) v = STBIR_FLOAT_HIGH_CLAMP; + #else + #define stbir_scalar_hi_clamp( v ) + #endif + #ifdef STBIR_FLOAT_LOW_CLAMP + #define stbir_scalar_lo_clamp( v ) if ( v < STBIR_FLOAT_LOW_CLAMP ) v = STBIR_FLOAT_LOW_CLAMP; + #else + #define stbir_scalar_lo_clamp( v ) + #endif + + #ifdef STBIR_SIMD + + #ifdef STBIR_FLOAT_HIGH_CLAMP + const stbir__simdfX high_clamp = stbir__simdf_frepX(STBIR_FLOAT_HIGH_CLAMP); + #endif + #ifdef STBIR_FLOAT_LOW_CLAMP + const stbir__simdfX low_clamp = stbir__simdf_frepX(STBIR_FLOAT_LOW_CLAMP); + #endif + + if ( width_times_channels >= ( stbir__simdfX_float_count * 2 ) ) + { + float const * end_encode_m8 = encode + width_times_channels - ( stbir__simdfX_float_count * 2 ); + end_output -= ( stbir__simdfX_float_count * 2 ); + STBIR_SIMD_NO_UNROLL_LOOP_START_INF_FOR + for(;;) + { + stbir__simdfX e0, e1; + STBIR_SIMD_NO_UNROLL(encode); + stbir__simdfX_load( e0, encode ); + stbir__simdfX_load( e1, encode+stbir__simdfX_float_count ); +#ifdef STBIR_FLOAT_HIGH_CLAMP + stbir__simdfX_min( e0, e0, high_clamp ); + stbir__simdfX_min( e1, e1, high_clamp ); +#endif +#ifdef STBIR_FLOAT_LOW_CLAMP + stbir__simdfX_max( e0, e0, low_clamp ); + stbir__simdfX_max( e1, e1, low_clamp ); +#endif + stbir__encode_simdfX_unflip( e0 ); + stbir__encode_simdfX_unflip( e1 ); + stbir__simdfX_store( output, e0 ); + stbir__simdfX_store( output+stbir__simdfX_float_count, e1 ); + encode += stbir__simdfX_float_count * 2; + output += stbir__simdfX_float_count * 2; + if ( output < end_output ) + continue; + if ( output == ( end_output + ( stbir__simdfX_float_count * 2 ) ) ) + break; + output = end_output; // backup and do last couple + encode = end_encode_m8; + } + return; + } + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + stbir__simdf e0; + STBIR_NO_UNROLL(encode); + stbir__simdf_load( e0, encode ); +#ifdef STBIR_FLOAT_HIGH_CLAMP + stbir__simdf_min( e0, e0, high_clamp ); +#endif +#ifdef STBIR_FLOAT_LOW_CLAMP + stbir__simdf_max( e0, e0, low_clamp ); +#endif + stbir__encode_simdf4_unflip( e0 ); + stbir__simdf_store( output-4, e0 ); + output += 4; + encode += 4; + } + output -= 4; + #endif + + #else + + // try to do blocks of 4 when you can + #if stbir__coder_min_num != 3 // doesn't divide cleanly by four + output += 4; + STBIR_SIMD_NO_UNROLL_LOOP_START + while( output <= end_output ) + { + float e; + STBIR_SIMD_NO_UNROLL(encode); + e = encode[ stbir__encode_order0 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[0-4] = e; + e = encode[ stbir__encode_order1 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[1-4] = e; + e = encode[ stbir__encode_order2 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[2-4] = e; + e = encode[ stbir__encode_order3 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[3-4] = e; + output += 4; + encode += 4; + } + output -= 4; + + #endif + + #endif + + // do the remnants + #if stbir__coder_min_num < 4 + STBIR_NO_UNROLL_LOOP_START + while( output < end_output ) + { + float e; + STBIR_NO_UNROLL(encode); + e = encode[ stbir__encode_order0 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[0] = e; + #if stbir__coder_min_num >= 2 + e = encode[ stbir__encode_order1 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[1] = e; + #endif + #if stbir__coder_min_num >= 3 + e = encode[ stbir__encode_order2 ]; stbir_scalar_hi_clamp( e ); stbir_scalar_lo_clamp( e ); output[2] = e; + #endif + output += stbir__coder_min_num; + encode += stbir__coder_min_num; + } + #endif + + #endif +} + +#undef stbir__decode_suffix +#undef stbir__decode_simdf8_flip +#undef stbir__decode_simdf4_flip +#undef stbir__decode_order0 +#undef stbir__decode_order1 +#undef stbir__decode_order2 +#undef stbir__decode_order3 +#undef stbir__encode_order0 +#undef stbir__encode_order1 +#undef stbir__encode_order2 +#undef stbir__encode_order3 +#undef stbir__encode_simdf8_unflip +#undef stbir__encode_simdf4_unflip +#undef stbir__encode_simdfX_unflip +#undef STBIR__CODER_NAME +#undef stbir__coder_min_num +#undef stbir__decode_swizzle +#undef stbir_scalar_hi_clamp +#undef stbir_scalar_lo_clamp +#undef STB_IMAGE_RESIZE_DO_CODERS + +#elif defined( STB_IMAGE_RESIZE_DO_VERTICALS) + +#ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#define STBIR_chans( start, end ) STBIR_strs_join14(start,STBIR__vertical_channels,end,_cont) +#else +#define STBIR_chans( start, end ) STBIR_strs_join1(start,STBIR__vertical_channels,end) +#endif + +#if STBIR__vertical_channels >= 1 +#define stbIF0( code ) code +#else +#define stbIF0( code ) +#endif +#if STBIR__vertical_channels >= 2 +#define stbIF1( code ) code +#else +#define stbIF1( code ) +#endif +#if STBIR__vertical_channels >= 3 +#define stbIF2( code ) code +#else +#define stbIF2( code ) +#endif +#if STBIR__vertical_channels >= 4 +#define stbIF3( code ) code +#else +#define stbIF3( code ) +#endif +#if STBIR__vertical_channels >= 5 +#define stbIF4( code ) code +#else +#define stbIF4( code ) +#endif +#if STBIR__vertical_channels >= 6 +#define stbIF5( code ) code +#else +#define stbIF5( code ) +#endif +#if STBIR__vertical_channels >= 7 +#define stbIF6( code ) code +#else +#define stbIF6( code ) +#endif +#if STBIR__vertical_channels >= 8 +#define stbIF7( code ) code +#else +#define stbIF7( code ) +#endif + +static void STBIR_chans( stbir__vertical_scatter_with_,_coeffs)( float ** outputs, float const * vertical_coefficients, float const * input, float const * input_end ) +{ + stbIF0( float STBIR_SIMD_STREAMOUT_PTR( * ) output0 = outputs[0]; float c0s = vertical_coefficients[0]; ) + stbIF1( float STBIR_SIMD_STREAMOUT_PTR( * ) output1 = outputs[1]; float c1s = vertical_coefficients[1]; ) + stbIF2( float STBIR_SIMD_STREAMOUT_PTR( * ) output2 = outputs[2]; float c2s = vertical_coefficients[2]; ) + stbIF3( float STBIR_SIMD_STREAMOUT_PTR( * ) output3 = outputs[3]; float c3s = vertical_coefficients[3]; ) + stbIF4( float STBIR_SIMD_STREAMOUT_PTR( * ) output4 = outputs[4]; float c4s = vertical_coefficients[4]; ) + stbIF5( float STBIR_SIMD_STREAMOUT_PTR( * ) output5 = outputs[5]; float c5s = vertical_coefficients[5]; ) + stbIF6( float STBIR_SIMD_STREAMOUT_PTR( * ) output6 = outputs[6]; float c6s = vertical_coefficients[6]; ) + stbIF7( float STBIR_SIMD_STREAMOUT_PTR( * ) output7 = outputs[7]; float c7s = vertical_coefficients[7]; ) + + #ifdef STBIR_SIMD + { + stbIF0(stbir__simdfX c0 = stbir__simdf_frepX( c0s ); ) + stbIF1(stbir__simdfX c1 = stbir__simdf_frepX( c1s ); ) + stbIF2(stbir__simdfX c2 = stbir__simdf_frepX( c2s ); ) + stbIF3(stbir__simdfX c3 = stbir__simdf_frepX( c3s ); ) + stbIF4(stbir__simdfX c4 = stbir__simdf_frepX( c4s ); ) + stbIF5(stbir__simdfX c5 = stbir__simdf_frepX( c5s ); ) + stbIF6(stbir__simdfX c6 = stbir__simdf_frepX( c6s ); ) + stbIF7(stbir__simdfX c7 = stbir__simdf_frepX( c7s ); ) + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( (char*)input_end - (char*) input ) >= (16*stbir__simdfX_float_count) ) + { + stbir__simdfX o0, o1, o2, o3, r0, r1, r2, r3; + STBIR_SIMD_NO_UNROLL(output0); + + stbir__simdfX_load( r0, input ); stbir__simdfX_load( r1, input+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input+(3*stbir__simdfX_float_count) ); + + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( stbir__simdfX_load( o0, output0 ); stbir__simdfX_load( o1, output0+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output0+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output0+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c0 ); stbir__simdfX_madd( o1, o1, r1, c0 ); stbir__simdfX_madd( o2, o2, r2, c0 ); stbir__simdfX_madd( o3, o3, r3, c0 ); + stbir__simdfX_store( output0, o0 ); stbir__simdfX_store( output0+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output0+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output0+(3*stbir__simdfX_float_count), o3 ); ) + stbIF1( stbir__simdfX_load( o0, output1 ); stbir__simdfX_load( o1, output1+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output1+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output1+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c1 ); stbir__simdfX_madd( o1, o1, r1, c1 ); stbir__simdfX_madd( o2, o2, r2, c1 ); stbir__simdfX_madd( o3, o3, r3, c1 ); + stbir__simdfX_store( output1, o0 ); stbir__simdfX_store( output1+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output1+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output1+(3*stbir__simdfX_float_count), o3 ); ) + stbIF2( stbir__simdfX_load( o0, output2 ); stbir__simdfX_load( o1, output2+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output2+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output2+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c2 ); stbir__simdfX_madd( o1, o1, r1, c2 ); stbir__simdfX_madd( o2, o2, r2, c2 ); stbir__simdfX_madd( o3, o3, r3, c2 ); + stbir__simdfX_store( output2, o0 ); stbir__simdfX_store( output2+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output2+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output2+(3*stbir__simdfX_float_count), o3 ); ) + stbIF3( stbir__simdfX_load( o0, output3 ); stbir__simdfX_load( o1, output3+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output3+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output3+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c3 ); stbir__simdfX_madd( o1, o1, r1, c3 ); stbir__simdfX_madd( o2, o2, r2, c3 ); stbir__simdfX_madd( o3, o3, r3, c3 ); + stbir__simdfX_store( output3, o0 ); stbir__simdfX_store( output3+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output3+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output3+(3*stbir__simdfX_float_count), o3 ); ) + stbIF4( stbir__simdfX_load( o0, output4 ); stbir__simdfX_load( o1, output4+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output4+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output4+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c4 ); stbir__simdfX_madd( o1, o1, r1, c4 ); stbir__simdfX_madd( o2, o2, r2, c4 ); stbir__simdfX_madd( o3, o3, r3, c4 ); + stbir__simdfX_store( output4, o0 ); stbir__simdfX_store( output4+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output4+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output4+(3*stbir__simdfX_float_count), o3 ); ) + stbIF5( stbir__simdfX_load( o0, output5 ); stbir__simdfX_load( o1, output5+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output5+(2*stbir__simdfX_float_count)); stbir__simdfX_load( o3, output5+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c5 ); stbir__simdfX_madd( o1, o1, r1, c5 ); stbir__simdfX_madd( o2, o2, r2, c5 ); stbir__simdfX_madd( o3, o3, r3, c5 ); + stbir__simdfX_store( output5, o0 ); stbir__simdfX_store( output5+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output5+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output5+(3*stbir__simdfX_float_count), o3 ); ) + stbIF6( stbir__simdfX_load( o0, output6 ); stbir__simdfX_load( o1, output6+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output6+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output6+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c6 ); stbir__simdfX_madd( o1, o1, r1, c6 ); stbir__simdfX_madd( o2, o2, r2, c6 ); stbir__simdfX_madd( o3, o3, r3, c6 ); + stbir__simdfX_store( output6, o0 ); stbir__simdfX_store( output6+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output6+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output6+(3*stbir__simdfX_float_count), o3 ); ) + stbIF7( stbir__simdfX_load( o0, output7 ); stbir__simdfX_load( o1, output7+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output7+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output7+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c7 ); stbir__simdfX_madd( o1, o1, r1, c7 ); stbir__simdfX_madd( o2, o2, r2, c7 ); stbir__simdfX_madd( o3, o3, r3, c7 ); + stbir__simdfX_store( output7, o0 ); stbir__simdfX_store( output7+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output7+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output7+(3*stbir__simdfX_float_count), o3 ); ) + #else + stbIF0( stbir__simdfX_mult( o0, r0, c0 ); stbir__simdfX_mult( o1, r1, c0 ); stbir__simdfX_mult( o2, r2, c0 ); stbir__simdfX_mult( o3, r3, c0 ); + stbir__simdfX_store( output0, o0 ); stbir__simdfX_store( output0+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output0+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output0+(3*stbir__simdfX_float_count), o3 ); ) + stbIF1( stbir__simdfX_mult( o0, r0, c1 ); stbir__simdfX_mult( o1, r1, c1 ); stbir__simdfX_mult( o2, r2, c1 ); stbir__simdfX_mult( o3, r3, c1 ); + stbir__simdfX_store( output1, o0 ); stbir__simdfX_store( output1+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output1+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output1+(3*stbir__simdfX_float_count), o3 ); ) + stbIF2( stbir__simdfX_mult( o0, r0, c2 ); stbir__simdfX_mult( o1, r1, c2 ); stbir__simdfX_mult( o2, r2, c2 ); stbir__simdfX_mult( o3, r3, c2 ); + stbir__simdfX_store( output2, o0 ); stbir__simdfX_store( output2+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output2+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output2+(3*stbir__simdfX_float_count), o3 ); ) + stbIF3( stbir__simdfX_mult( o0, r0, c3 ); stbir__simdfX_mult( o1, r1, c3 ); stbir__simdfX_mult( o2, r2, c3 ); stbir__simdfX_mult( o3, r3, c3 ); + stbir__simdfX_store( output3, o0 ); stbir__simdfX_store( output3+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output3+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output3+(3*stbir__simdfX_float_count), o3 ); ) + stbIF4( stbir__simdfX_mult( o0, r0, c4 ); stbir__simdfX_mult( o1, r1, c4 ); stbir__simdfX_mult( o2, r2, c4 ); stbir__simdfX_mult( o3, r3, c4 ); + stbir__simdfX_store( output4, o0 ); stbir__simdfX_store( output4+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output4+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output4+(3*stbir__simdfX_float_count), o3 ); ) + stbIF5( stbir__simdfX_mult( o0, r0, c5 ); stbir__simdfX_mult( o1, r1, c5 ); stbir__simdfX_mult( o2, r2, c5 ); stbir__simdfX_mult( o3, r3, c5 ); + stbir__simdfX_store( output5, o0 ); stbir__simdfX_store( output5+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output5+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output5+(3*stbir__simdfX_float_count), o3 ); ) + stbIF6( stbir__simdfX_mult( o0, r0, c6 ); stbir__simdfX_mult( o1, r1, c6 ); stbir__simdfX_mult( o2, r2, c6 ); stbir__simdfX_mult( o3, r3, c6 ); + stbir__simdfX_store( output6, o0 ); stbir__simdfX_store( output6+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output6+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output6+(3*stbir__simdfX_float_count), o3 ); ) + stbIF7( stbir__simdfX_mult( o0, r0, c7 ); stbir__simdfX_mult( o1, r1, c7 ); stbir__simdfX_mult( o2, r2, c7 ); stbir__simdfX_mult( o3, r3, c7 ); + stbir__simdfX_store( output7, o0 ); stbir__simdfX_store( output7+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output7+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output7+(3*stbir__simdfX_float_count), o3 ); ) + #endif + + input += (4*stbir__simdfX_float_count); + stbIF0( output0 += (4*stbir__simdfX_float_count); ) stbIF1( output1 += (4*stbir__simdfX_float_count); ) stbIF2( output2 += (4*stbir__simdfX_float_count); ) stbIF3( output3 += (4*stbir__simdfX_float_count); ) stbIF4( output4 += (4*stbir__simdfX_float_count); ) stbIF5( output5 += (4*stbir__simdfX_float_count); ) stbIF6( output6 += (4*stbir__simdfX_float_count); ) stbIF7( output7 += (4*stbir__simdfX_float_count); ) + } + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( (char*)input_end - (char*) input ) >= 16 ) + { + stbir__simdf o0, r0; + STBIR_SIMD_NO_UNROLL(output0); + + stbir__simdf_load( r0, input ); + + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( stbir__simdf_load( o0, output0 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); ) + stbIF1( stbir__simdf_load( o0, output1 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); ) + stbIF2( stbir__simdf_load( o0, output2 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); ) + stbIF3( stbir__simdf_load( o0, output3 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); ) + stbIF4( stbir__simdf_load( o0, output4 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); ) + stbIF5( stbir__simdf_load( o0, output5 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); ) + stbIF6( stbir__simdf_load( o0, output6 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); ) + stbIF7( stbir__simdf_load( o0, output7 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); ) + #else + stbIF0( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); stbir__simdf_store( output0, o0 ); ) + stbIF1( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); stbir__simdf_store( output1, o0 ); ) + stbIF2( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); stbir__simdf_store( output2, o0 ); ) + stbIF3( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); stbir__simdf_store( output3, o0 ); ) + stbIF4( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); stbir__simdf_store( output4, o0 ); ) + stbIF5( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); stbir__simdf_store( output5, o0 ); ) + stbIF6( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); stbir__simdf_store( output6, o0 ); ) + stbIF7( stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); stbir__simdf_store( output7, o0 ); ) + #endif + + input += 4; + stbIF0( output0 += 4; ) stbIF1( output1 += 4; ) stbIF2( output2 += 4; ) stbIF3( output3 += 4; ) stbIF4( output4 += 4; ) stbIF5( output5 += 4; ) stbIF6( output6 += 4; ) stbIF7( output7 += 4; ) + } + } + #else + STBIR_NO_UNROLL_LOOP_START + while ( ( (char*)input_end - (char*) input ) >= 16 ) + { + float r0, r1, r2, r3; + STBIR_NO_UNROLL(input); + + r0 = input[0], r1 = input[1], r2 = input[2], r3 = input[3]; + + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( output0[0] += ( r0 * c0s ); output0[1] += ( r1 * c0s ); output0[2] += ( r2 * c0s ); output0[3] += ( r3 * c0s ); ) + stbIF1( output1[0] += ( r0 * c1s ); output1[1] += ( r1 * c1s ); output1[2] += ( r2 * c1s ); output1[3] += ( r3 * c1s ); ) + stbIF2( output2[0] += ( r0 * c2s ); output2[1] += ( r1 * c2s ); output2[2] += ( r2 * c2s ); output2[3] += ( r3 * c2s ); ) + stbIF3( output3[0] += ( r0 * c3s ); output3[1] += ( r1 * c3s ); output3[2] += ( r2 * c3s ); output3[3] += ( r3 * c3s ); ) + stbIF4( output4[0] += ( r0 * c4s ); output4[1] += ( r1 * c4s ); output4[2] += ( r2 * c4s ); output4[3] += ( r3 * c4s ); ) + stbIF5( output5[0] += ( r0 * c5s ); output5[1] += ( r1 * c5s ); output5[2] += ( r2 * c5s ); output5[3] += ( r3 * c5s ); ) + stbIF6( output6[0] += ( r0 * c6s ); output6[1] += ( r1 * c6s ); output6[2] += ( r2 * c6s ); output6[3] += ( r3 * c6s ); ) + stbIF7( output7[0] += ( r0 * c7s ); output7[1] += ( r1 * c7s ); output7[2] += ( r2 * c7s ); output7[3] += ( r3 * c7s ); ) + #else + stbIF0( output0[0] = ( r0 * c0s ); output0[1] = ( r1 * c0s ); output0[2] = ( r2 * c0s ); output0[3] = ( r3 * c0s ); ) + stbIF1( output1[0] = ( r0 * c1s ); output1[1] = ( r1 * c1s ); output1[2] = ( r2 * c1s ); output1[3] = ( r3 * c1s ); ) + stbIF2( output2[0] = ( r0 * c2s ); output2[1] = ( r1 * c2s ); output2[2] = ( r2 * c2s ); output2[3] = ( r3 * c2s ); ) + stbIF3( output3[0] = ( r0 * c3s ); output3[1] = ( r1 * c3s ); output3[2] = ( r2 * c3s ); output3[3] = ( r3 * c3s ); ) + stbIF4( output4[0] = ( r0 * c4s ); output4[1] = ( r1 * c4s ); output4[2] = ( r2 * c4s ); output4[3] = ( r3 * c4s ); ) + stbIF5( output5[0] = ( r0 * c5s ); output5[1] = ( r1 * c5s ); output5[2] = ( r2 * c5s ); output5[3] = ( r3 * c5s ); ) + stbIF6( output6[0] = ( r0 * c6s ); output6[1] = ( r1 * c6s ); output6[2] = ( r2 * c6s ); output6[3] = ( r3 * c6s ); ) + stbIF7( output7[0] = ( r0 * c7s ); output7[1] = ( r1 * c7s ); output7[2] = ( r2 * c7s ); output7[3] = ( r3 * c7s ); ) + #endif + + input += 4; + stbIF0( output0 += 4; ) stbIF1( output1 += 4; ) stbIF2( output2 += 4; ) stbIF3( output3 += 4; ) stbIF4( output4 += 4; ) stbIF5( output5 += 4; ) stbIF6( output6 += 4; ) stbIF7( output7 += 4; ) + } + #endif + STBIR_NO_UNROLL_LOOP_START + while ( input < input_end ) + { + float r = input[0]; + STBIR_NO_UNROLL(output0); + + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( output0[0] += ( r * c0s ); ) + stbIF1( output1[0] += ( r * c1s ); ) + stbIF2( output2[0] += ( r * c2s ); ) + stbIF3( output3[0] += ( r * c3s ); ) + stbIF4( output4[0] += ( r * c4s ); ) + stbIF5( output5[0] += ( r * c5s ); ) + stbIF6( output6[0] += ( r * c6s ); ) + stbIF7( output7[0] += ( r * c7s ); ) + #else + stbIF0( output0[0] = ( r * c0s ); ) + stbIF1( output1[0] = ( r * c1s ); ) + stbIF2( output2[0] = ( r * c2s ); ) + stbIF3( output3[0] = ( r * c3s ); ) + stbIF4( output4[0] = ( r * c4s ); ) + stbIF5( output5[0] = ( r * c5s ); ) + stbIF6( output6[0] = ( r * c6s ); ) + stbIF7( output7[0] = ( r * c7s ); ) + #endif + + ++input; + stbIF0( ++output0; ) stbIF1( ++output1; ) stbIF2( ++output2; ) stbIF3( ++output3; ) stbIF4( ++output4; ) stbIF5( ++output5; ) stbIF6( ++output6; ) stbIF7( ++output7; ) + } +} + +static void STBIR_chans( stbir__vertical_gather_with_,_coeffs)( float * outputp, float const * vertical_coefficients, float const ** inputs, float const * input0_end ) +{ + float STBIR_SIMD_STREAMOUT_PTR( * ) output = outputp; + + stbIF0( float const * input0 = inputs[0]; float c0s = vertical_coefficients[0]; ) + stbIF1( float const * input1 = inputs[1]; float c1s = vertical_coefficients[1]; ) + stbIF2( float const * input2 = inputs[2]; float c2s = vertical_coefficients[2]; ) + stbIF3( float const * input3 = inputs[3]; float c3s = vertical_coefficients[3]; ) + stbIF4( float const * input4 = inputs[4]; float c4s = vertical_coefficients[4]; ) + stbIF5( float const * input5 = inputs[5]; float c5s = vertical_coefficients[5]; ) + stbIF6( float const * input6 = inputs[6]; float c6s = vertical_coefficients[6]; ) + stbIF7( float const * input7 = inputs[7]; float c7s = vertical_coefficients[7]; ) + +#if ( STBIR__vertical_channels == 1 ) && !defined(STB_IMAGE_RESIZE_VERTICAL_CONTINUE) + // check single channel one weight + if ( ( c0s >= (1.0f-0.000001f) ) && ( c0s <= (1.0f+0.000001f) ) ) + { + STBIR_MEMCPY( output, input0, (char*)input0_end - (char*)input0 ); + return; + } +#endif + + #ifdef STBIR_SIMD + { + stbIF0(stbir__simdfX c0 = stbir__simdf_frepX( c0s ); ) + stbIF1(stbir__simdfX c1 = stbir__simdf_frepX( c1s ); ) + stbIF2(stbir__simdfX c2 = stbir__simdf_frepX( c2s ); ) + stbIF3(stbir__simdfX c3 = stbir__simdf_frepX( c3s ); ) + stbIF4(stbir__simdfX c4 = stbir__simdf_frepX( c4s ); ) + stbIF5(stbir__simdfX c5 = stbir__simdf_frepX( c5s ); ) + stbIF6(stbir__simdfX c6 = stbir__simdf_frepX( c6s ); ) + stbIF7(stbir__simdfX c7 = stbir__simdf_frepX( c7s ); ) + + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( (char*)input0_end - (char*) input0 ) >= (16*stbir__simdfX_float_count) ) + { + stbir__simdfX o0, o1, o2, o3, r0, r1, r2, r3; + STBIR_SIMD_NO_UNROLL(output); + + // prefetch four loop iterations ahead (doesn't affect much for small resizes, but helps with big ones) + stbIF0( stbir__prefetch( input0 + (16*stbir__simdfX_float_count) ); ) + stbIF1( stbir__prefetch( input1 + (16*stbir__simdfX_float_count) ); ) + stbIF2( stbir__prefetch( input2 + (16*stbir__simdfX_float_count) ); ) + stbIF3( stbir__prefetch( input3 + (16*stbir__simdfX_float_count) ); ) + stbIF4( stbir__prefetch( input4 + (16*stbir__simdfX_float_count) ); ) + stbIF5( stbir__prefetch( input5 + (16*stbir__simdfX_float_count) ); ) + stbIF6( stbir__prefetch( input6 + (16*stbir__simdfX_float_count) ); ) + stbIF7( stbir__prefetch( input7 + (16*stbir__simdfX_float_count) ); ) + + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( stbir__simdfX_load( o0, output ); stbir__simdfX_load( o1, output+stbir__simdfX_float_count ); stbir__simdfX_load( o2, output+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( o3, output+(3*stbir__simdfX_float_count) ); + stbir__simdfX_load( r0, input0 ); stbir__simdfX_load( r1, input0+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input0+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input0+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c0 ); stbir__simdfX_madd( o1, o1, r1, c0 ); stbir__simdfX_madd( o2, o2, r2, c0 ); stbir__simdfX_madd( o3, o3, r3, c0 ); ) + #else + stbIF0( stbir__simdfX_load( r0, input0 ); stbir__simdfX_load( r1, input0+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input0+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input0+(3*stbir__simdfX_float_count) ); + stbir__simdfX_mult( o0, r0, c0 ); stbir__simdfX_mult( o1, r1, c0 ); stbir__simdfX_mult( o2, r2, c0 ); stbir__simdfX_mult( o3, r3, c0 ); ) + #endif + + stbIF1( stbir__simdfX_load( r0, input1 ); stbir__simdfX_load( r1, input1+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input1+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input1+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c1 ); stbir__simdfX_madd( o1, o1, r1, c1 ); stbir__simdfX_madd( o2, o2, r2, c1 ); stbir__simdfX_madd( o3, o3, r3, c1 ); ) + stbIF2( stbir__simdfX_load( r0, input2 ); stbir__simdfX_load( r1, input2+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input2+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input2+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c2 ); stbir__simdfX_madd( o1, o1, r1, c2 ); stbir__simdfX_madd( o2, o2, r2, c2 ); stbir__simdfX_madd( o3, o3, r3, c2 ); ) + stbIF3( stbir__simdfX_load( r0, input3 ); stbir__simdfX_load( r1, input3+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input3+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input3+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c3 ); stbir__simdfX_madd( o1, o1, r1, c3 ); stbir__simdfX_madd( o2, o2, r2, c3 ); stbir__simdfX_madd( o3, o3, r3, c3 ); ) + stbIF4( stbir__simdfX_load( r0, input4 ); stbir__simdfX_load( r1, input4+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input4+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input4+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c4 ); stbir__simdfX_madd( o1, o1, r1, c4 ); stbir__simdfX_madd( o2, o2, r2, c4 ); stbir__simdfX_madd( o3, o3, r3, c4 ); ) + stbIF5( stbir__simdfX_load( r0, input5 ); stbir__simdfX_load( r1, input5+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input5+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input5+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c5 ); stbir__simdfX_madd( o1, o1, r1, c5 ); stbir__simdfX_madd( o2, o2, r2, c5 ); stbir__simdfX_madd( o3, o3, r3, c5 ); ) + stbIF6( stbir__simdfX_load( r0, input6 ); stbir__simdfX_load( r1, input6+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input6+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input6+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c6 ); stbir__simdfX_madd( o1, o1, r1, c6 ); stbir__simdfX_madd( o2, o2, r2, c6 ); stbir__simdfX_madd( o3, o3, r3, c6 ); ) + stbIF7( stbir__simdfX_load( r0, input7 ); stbir__simdfX_load( r1, input7+stbir__simdfX_float_count ); stbir__simdfX_load( r2, input7+(2*stbir__simdfX_float_count) ); stbir__simdfX_load( r3, input7+(3*stbir__simdfX_float_count) ); + stbir__simdfX_madd( o0, o0, r0, c7 ); stbir__simdfX_madd( o1, o1, r1, c7 ); stbir__simdfX_madd( o2, o2, r2, c7 ); stbir__simdfX_madd( o3, o3, r3, c7 ); ) + + stbir__simdfX_store( output, o0 ); stbir__simdfX_store( output+stbir__simdfX_float_count, o1 ); stbir__simdfX_store( output+(2*stbir__simdfX_float_count), o2 ); stbir__simdfX_store( output+(3*stbir__simdfX_float_count), o3 ); + output += (4*stbir__simdfX_float_count); + stbIF0( input0 += (4*stbir__simdfX_float_count); ) stbIF1( input1 += (4*stbir__simdfX_float_count); ) stbIF2( input2 += (4*stbir__simdfX_float_count); ) stbIF3( input3 += (4*stbir__simdfX_float_count); ) stbIF4( input4 += (4*stbir__simdfX_float_count); ) stbIF5( input5 += (4*stbir__simdfX_float_count); ) stbIF6( input6 += (4*stbir__simdfX_float_count); ) stbIF7( input7 += (4*stbir__simdfX_float_count); ) + } + + STBIR_SIMD_NO_UNROLL_LOOP_START + while ( ( (char*)input0_end - (char*) input0 ) >= 16 ) + { + stbir__simdf o0, r0; + STBIR_SIMD_NO_UNROLL(output); + + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( stbir__simdf_load( o0, output ); stbir__simdf_load( r0, input0 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); ) + #else + stbIF0( stbir__simdf_load( r0, input0 ); stbir__simdf_mult( o0, r0, stbir__if_simdf8_cast_to_simdf4( c0 ) ); ) + #endif + stbIF1( stbir__simdf_load( r0, input1 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c1 ) ); ) + stbIF2( stbir__simdf_load( r0, input2 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c2 ) ); ) + stbIF3( stbir__simdf_load( r0, input3 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c3 ) ); ) + stbIF4( stbir__simdf_load( r0, input4 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c4 ) ); ) + stbIF5( stbir__simdf_load( r0, input5 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c5 ) ); ) + stbIF6( stbir__simdf_load( r0, input6 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c6 ) ); ) + stbIF7( stbir__simdf_load( r0, input7 ); stbir__simdf_madd( o0, o0, r0, stbir__if_simdf8_cast_to_simdf4( c7 ) ); ) + + stbir__simdf_store( output, o0 ); + output += 4; + stbIF0( input0 += 4; ) stbIF1( input1 += 4; ) stbIF2( input2 += 4; ) stbIF3( input3 += 4; ) stbIF4( input4 += 4; ) stbIF5( input5 += 4; ) stbIF6( input6 += 4; ) stbIF7( input7 += 4; ) + } + } + #else + STBIR_NO_UNROLL_LOOP_START + while ( ( (char*)input0_end - (char*) input0 ) >= 16 ) + { + float o0, o1, o2, o3; + STBIR_NO_UNROLL(output); + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( o0 = output[0] + input0[0] * c0s; o1 = output[1] + input0[1] * c0s; o2 = output[2] + input0[2] * c0s; o3 = output[3] + input0[3] * c0s; ) + #else + stbIF0( o0 = input0[0] * c0s; o1 = input0[1] * c0s; o2 = input0[2] * c0s; o3 = input0[3] * c0s; ) + #endif + stbIF1( o0 += input1[0] * c1s; o1 += input1[1] * c1s; o2 += input1[2] * c1s; o3 += input1[3] * c1s; ) + stbIF2( o0 += input2[0] * c2s; o1 += input2[1] * c2s; o2 += input2[2] * c2s; o3 += input2[3] * c2s; ) + stbIF3( o0 += input3[0] * c3s; o1 += input3[1] * c3s; o2 += input3[2] * c3s; o3 += input3[3] * c3s; ) + stbIF4( o0 += input4[0] * c4s; o1 += input4[1] * c4s; o2 += input4[2] * c4s; o3 += input4[3] * c4s; ) + stbIF5( o0 += input5[0] * c5s; o1 += input5[1] * c5s; o2 += input5[2] * c5s; o3 += input5[3] * c5s; ) + stbIF6( o0 += input6[0] * c6s; o1 += input6[1] * c6s; o2 += input6[2] * c6s; o3 += input6[3] * c6s; ) + stbIF7( o0 += input7[0] * c7s; o1 += input7[1] * c7s; o2 += input7[2] * c7s; o3 += input7[3] * c7s; ) + output[0] = o0; output[1] = o1; output[2] = o2; output[3] = o3; + output += 4; + stbIF0( input0 += 4; ) stbIF1( input1 += 4; ) stbIF2( input2 += 4; ) stbIF3( input3 += 4; ) stbIF4( input4 += 4; ) stbIF5( input5 += 4; ) stbIF6( input6 += 4; ) stbIF7( input7 += 4; ) + } + #endif + STBIR_NO_UNROLL_LOOP_START + while ( input0 < input0_end ) + { + float o0; + STBIR_NO_UNROLL(output); + #ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE + stbIF0( o0 = output[0] + input0[0] * c0s; ) + #else + stbIF0( o0 = input0[0] * c0s; ) + #endif + stbIF1( o0 += input1[0] * c1s; ) + stbIF2( o0 += input2[0] * c2s; ) + stbIF3( o0 += input3[0] * c3s; ) + stbIF4( o0 += input4[0] * c4s; ) + stbIF5( o0 += input5[0] * c5s; ) + stbIF6( o0 += input6[0] * c6s; ) + stbIF7( o0 += input7[0] * c7s; ) + output[0] = o0; + ++output; + stbIF0( ++input0; ) stbIF1( ++input1; ) stbIF2( ++input2; ) stbIF3( ++input3; ) stbIF4( ++input4; ) stbIF5( ++input5; ) stbIF6( ++input6; ) stbIF7( ++input7; ) + } +} + +#undef stbIF0 +#undef stbIF1 +#undef stbIF2 +#undef stbIF3 +#undef stbIF4 +#undef stbIF5 +#undef stbIF6 +#undef stbIF7 +#undef STB_IMAGE_RESIZE_DO_VERTICALS +#undef STBIR__vertical_channels +#undef STB_IMAGE_RESIZE_DO_HORIZONTALS +#undef STBIR_strs_join24 +#undef STBIR_strs_join14 +#undef STBIR_chans +#ifdef STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#undef STB_IMAGE_RESIZE_VERTICAL_CONTINUE +#endif + +#else // !STB_IMAGE_RESIZE_DO_VERTICALS + +#define STBIR_chans( start, end ) STBIR_strs_join1(start,STBIR__horizontal_channels,end) + +#ifndef stbir__2_coeff_only +#define stbir__2_coeff_only() \ + stbir__1_coeff_only(); \ + stbir__1_coeff_remnant(1); +#endif + +#ifndef stbir__2_coeff_remnant +#define stbir__2_coeff_remnant( ofs ) \ + stbir__1_coeff_remnant(ofs); \ + stbir__1_coeff_remnant((ofs)+1); +#endif + +#ifndef stbir__3_coeff_only +#define stbir__3_coeff_only() \ + stbir__2_coeff_only(); \ + stbir__1_coeff_remnant(2); +#endif + +#ifndef stbir__3_coeff_remnant +#define stbir__3_coeff_remnant( ofs ) \ + stbir__2_coeff_remnant(ofs); \ + stbir__1_coeff_remnant((ofs)+2); +#endif + +#ifndef stbir__3_coeff_setup +#define stbir__3_coeff_setup() +#endif + +#ifndef stbir__4_coeff_start +#define stbir__4_coeff_start() \ + stbir__2_coeff_only(); \ + stbir__2_coeff_remnant(2); +#endif + +#ifndef stbir__4_coeff_continue_from_4 +#define stbir__4_coeff_continue_from_4( ofs ) \ + stbir__2_coeff_remnant(ofs); \ + stbir__2_coeff_remnant((ofs)+2); +#endif + +#ifndef stbir__store_output_tiny +#define stbir__store_output_tiny stbir__store_output +#endif + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_1_coeff)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__1_coeff_only(); + stbir__store_output_tiny(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_2_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__2_coeff_only(); + stbir__store_output_tiny(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_3_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__3_coeff_only(); + stbir__store_output_tiny(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_4_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_5_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__1_coeff_remnant(4); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_6_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__2_coeff_remnant(4); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_7_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + stbir__3_coeff_setup(); + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + + stbir__4_coeff_start(); + stbir__3_coeff_remnant(4); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_8_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_9_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__1_coeff_remnant(8); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_10_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__2_coeff_remnant(8); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_11_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + stbir__3_coeff_setup(); + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__3_coeff_remnant(8); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_12_coeffs)( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + float const * hc = horizontal_coefficients; + stbir__4_coeff_start(); + stbir__4_coeff_continue_from_4(4); + stbir__4_coeff_continue_from_4(8); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod0 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 4 + 3 ) >> 2; + float const * hc = horizontal_coefficients; + + stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + hc += 4; + decode += STBIR__horizontal_channels * 4; + stbir__4_coeff_continue_from_4( 0 ); + --n; + } while ( n > 0 ); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod1 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 5 + 3 ) >> 2; + float const * hc = horizontal_coefficients; + + stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + hc += 4; + decode += STBIR__horizontal_channels * 4; + stbir__4_coeff_continue_from_4( 0 ); + --n; + } while ( n > 0 ); + stbir__1_coeff_remnant( 4 ); + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod2 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 6 + 3 ) >> 2; + float const * hc = horizontal_coefficients; + + stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + hc += 4; + decode += STBIR__horizontal_channels * 4; + stbir__4_coeff_continue_from_4( 0 ); + --n; + } while ( n > 0 ); + stbir__2_coeff_remnant( 4 ); + + stbir__store_output(); + } while ( output < output_end ); +} + +static void STBIR_chans( stbir__horizontal_gather_,_channels_with_n_coeffs_mod3 )( float * output_buffer, unsigned int output_sub_size, float const * decode_buffer, stbir__contributors const * horizontal_contributors, float const * horizontal_coefficients, int coefficient_width ) +{ + float const * output_end = output_buffer + output_sub_size * STBIR__horizontal_channels; + float STBIR_SIMD_STREAMOUT_PTR( * ) output = output_buffer; + stbir__3_coeff_setup(); + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + float const * decode = decode_buffer + horizontal_contributors->n0 * STBIR__horizontal_channels; + int n = ( ( horizontal_contributors->n1 - horizontal_contributors->n0 + 1 ) - 7 + 3 ) >> 2; + float const * hc = horizontal_coefficients; + + stbir__4_coeff_start(); + STBIR_SIMD_NO_UNROLL_LOOP_START + do { + hc += 4; + decode += STBIR__horizontal_channels * 4; + stbir__4_coeff_continue_from_4( 0 ); + --n; + } while ( n > 0 ); + stbir__3_coeff_remnant( 4 ); + + stbir__store_output(); + } while ( output < output_end ); +} + +static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_funcs)[4]= +{ + STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod0), + STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod1), + STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod2), + STBIR_chans(stbir__horizontal_gather_,_channels_with_n_coeffs_mod3), +}; + +static stbir__horizontal_gather_channels_func * STBIR_chans(stbir__horizontal_gather_,_channels_funcs)[12]= +{ + STBIR_chans(stbir__horizontal_gather_,_channels_with_1_coeff), + STBIR_chans(stbir__horizontal_gather_,_channels_with_2_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_3_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_4_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_5_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_6_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_7_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_8_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_9_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_10_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_11_coeffs), + STBIR_chans(stbir__horizontal_gather_,_channels_with_12_coeffs), +}; + +#undef STBIR__horizontal_channels +#undef STB_IMAGE_RESIZE_DO_HORIZONTALS +#undef stbir__1_coeff_only +#undef stbir__1_coeff_remnant +#undef stbir__2_coeff_only +#undef stbir__2_coeff_remnant +#undef stbir__3_coeff_only +#undef stbir__3_coeff_remnant +#undef stbir__3_coeff_setup +#undef stbir__4_coeff_start +#undef stbir__4_coeff_continue_from_4 +#undef stbir__store_output +#undef stbir__store_output_tiny +#undef STBIR_chans + +#endif // HORIZONALS + +#undef STBIR_strs_join2 +#undef STBIR_strs_join1 + +#endif // STB_IMAGE_RESIZE_DO_HORIZONTALS/VERTICALS/CODERS + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/common/stb_image_write.h b/common/stb_image_write.h new file mode 100644 index 0000000000000..e4b32ed1bc32e --- /dev/null +++ b/common/stb_image_write.h @@ -0,0 +1,1724 @@ +/* stb_image_write - v1.16 - public domain - http://nothings.org/stb + writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015 + no warranty implied; use at your own risk + + Before #including, + + #define STB_IMAGE_WRITE_IMPLEMENTATION + + in the file that you want to have the implementation. + + Will probably not work correctly with strict-aliasing optimizations. + +ABOUT: + + This header file is a library for writing images to C stdio or a callback. + + The PNG output is not optimal; it is 20-50% larger than the file + written by a decent optimizing implementation; though providing a custom + zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that. + This library is designed for source code compactness and simplicity, + not optimal image file size or run-time performance. + +BUILDING: + + You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h. + You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace + malloc,realloc,free. + You can #define STBIW_MEMMOVE() to replace memmove() + You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress function + for PNG compression (instead of the builtin one), it must have the following signature: + unsigned char * my_compress(unsigned char *data, int data_len, int *out_len, int quality); + The returned data will be freed with STBIW_FREE() (free() by default), + so it must be heap allocated with STBIW_MALLOC() (malloc() by default), + +UNICODE: + + If compiling for Windows and you wish to use Unicode filenames, compile + with + #define STBIW_WINDOWS_UTF8 + and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert + Windows wchar_t filenames to utf8. + +USAGE: + + There are five functions, one for each image file format: + + int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_jpg(char const *filename, int w, int h, int comp, const void *data, int quality); + int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); + + void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip data vertically + + There are also five equivalent functions that use an arbitrary write function. You are + expected to open/close your file-equivalent before and after calling these: + + int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); + int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + + where the callback is: + void stbi_write_func(void *context, void *data, int size); + + You can configure it with these global variables: + int stbi_write_tga_with_rle; // defaults to true; set to 0 to disable RLE + int stbi_write_png_compression_level; // defaults to 8; set to higher for more compression + int stbi_write_force_png_filter; // defaults to -1; set to 0..5 to force a filter mode + + + You can define STBI_WRITE_NO_STDIO to disable the file variant of these + functions, so the library will not use stdio.h at all. However, this will + also disable HDR writing, because it requires stdio for formatted output. + + Each function returns 0 on failure and non-0 on success. + + The functions create an image file defined by the parameters. The image + is a rectangle of pixels stored from left-to-right, top-to-bottom. + Each pixel contains 'comp' channels of data stored interleaved with 8-bits + per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is + monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall. + The *data pointer points to the first byte of the top-left-most pixel. + For PNG, "stride_in_bytes" is the distance in bytes from the first byte of + a row of pixels to the first byte of the next row of pixels. + + PNG creates output files with the same number of components as the input. + The BMP format expands Y to RGB in the file format and does not + output alpha. + + PNG supports writing rectangles of data even when the bytes storing rows of + data are not consecutive in memory (e.g. sub-rectangles of a larger image), + by supplying the stride between the beginning of adjacent rows. The other + formats do not. (Thus you cannot write a native-format BMP through the BMP + writer, both because it is in BGR order and because it may have padding + at the end of the line.) + + PNG allows you to set the deflate compression level by setting the global + variable 'stbi_write_png_compression_level' (it defaults to 8). + + HDR expects linear float data. Since the format is always 32-bit rgb(e) + data, alpha (if provided) is discarded, and for monochrome data it is + replicated across all three channels. + + TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed + data, set the global variable 'stbi_write_tga_with_rle' to 0. + + JPEG does ignore alpha channels in input data; quality is between 1 and 100. + Higher quality looks better but results in a bigger image. + JPEG baseline (no JPEG progressive). + +CREDITS: + + + Sean Barrett - PNG/BMP/TGA + Baldur Karlsson - HDR + Jean-Sebastien Guay - TGA monochrome + Tim Kelsey - misc enhancements + Alan Hickman - TGA RLE + Emmanuel Julien - initial file IO callback implementation + Jon Olick - original jo_jpeg.cpp code + Daniel Gibson - integrate JPEG, allow external zlib + Aarni Koskela - allow choosing PNG filter + + bugfixes: + github:Chribba + Guillaume Chereau + github:jry2 + github:romigrou + Sergio Gonzalez + Jonas Karlsson + Filip Wasil + Thatcher Ulrich + github:poppolopoppo + Patrick Boettcher + github:xeekworx + Cap Petschulat + Simon Rodriguez + Ivan Tikhonov + github:ignotion + Adam Schackart + Andrew Kensler + +LICENSE + + See end of file for license information. + +*/ + +#ifndef INCLUDE_STB_IMAGE_WRITE_H +#define INCLUDE_STB_IMAGE_WRITE_H + +#include + +// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' or 'static inline' +#ifndef STBIWDEF +#ifdef STB_IMAGE_WRITE_STATIC +#define STBIWDEF static +#else +#ifdef __cplusplus +#define STBIWDEF extern "C" +#else +#define STBIWDEF extern +#endif +#endif +#endif + +#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations +STBIWDEF int stbi_write_tga_with_rle; +STBIWDEF int stbi_write_png_compression_level; +STBIWDEF int stbi_write_force_png_filter; +#endif + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality); + +#ifdef STBIW_WINDOWS_UTF8 +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif +#endif + +typedef void stbi_write_func(void *context, void *data, int size); + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + +STBIWDEF void stbi_flip_vertically_on_write(int flip_boolean); + +#endif//INCLUDE_STB_IMAGE_WRITE_H + +#ifdef STB_IMAGE_WRITE_IMPLEMENTATION + +#ifdef _WIN32 + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif + #ifndef _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_NONSTDC_NO_DEPRECATE + #endif +#endif + +#ifndef STBI_WRITE_NO_STDIO +#include +#endif // STBI_WRITE_NO_STDIO + +#include +#include +#include +#include + +#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED)) +// ok +#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED)." +#endif + +#ifndef STBIW_MALLOC +#define STBIW_MALLOC(sz) malloc(sz) +#define STBIW_REALLOC(p,newsz) realloc(p,newsz) +#define STBIW_FREE(p) free(p) +#endif + +#ifndef STBIW_REALLOC_SIZED +#define STBIW_REALLOC_SIZED(p,oldsz,newsz) STBIW_REALLOC(p,newsz) +#endif + + +#ifndef STBIW_MEMMOVE +#define STBIW_MEMMOVE(a,b,sz) memmove(a,b,sz) +#endif + + +#ifndef STBIW_ASSERT +#include +#define STBIW_ASSERT(x) assert(x) +#endif + +#define STBIW_UCHAR(x) (unsigned char) ((x) & 0xff) + +#ifdef STB_IMAGE_WRITE_STATIC +static int stbi_write_png_compression_level = 8; +static int stbi_write_tga_with_rle = 1; +static int stbi_write_force_png_filter = -1; +#else +int stbi_write_png_compression_level = 8; +int stbi_write_tga_with_rle = 1; +int stbi_write_force_png_filter = -1; +#endif + +static int stbi__flip_vertically_on_write = 0; + +STBIWDEF void stbi_flip_vertically_on_write(int flag) +{ + stbi__flip_vertically_on_write = flag; +} + +typedef struct +{ + stbi_write_func *func; + void *context; + unsigned char buffer[64]; + int buf_used; +} stbi__write_context; + +// initialize a callback-based context +static void stbi__start_write_callbacks(stbi__write_context *s, stbi_write_func *c, void *context) +{ + s->func = c; + s->context = context; +} + +#ifndef STBI_WRITE_NO_STDIO + +static void stbi__stdio_write(void *context, void *data, int size) +{ + fwrite(data,1,size,(FILE*) context); +} + +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) +#ifdef __cplusplus +#define STBIW_EXTERN extern "C" +#else +#define STBIW_EXTERN extern +#endif +STBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); + +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbiw__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + +static int stbi__start_write_file(stbi__write_context *s, const char *filename) +{ + FILE *f = stbiw__fopen(filename, "wb"); + stbi__start_write_callbacks(s, stbi__stdio_write, (void *) f); + return f != NULL; +} + +static void stbi__end_write_file(stbi__write_context *s) +{ + fclose((FILE *)s->context); +} + +#endif // !STBI_WRITE_NO_STDIO + +typedef unsigned int stbiw_uint32; +typedef int stb_image_write_test[sizeof(stbiw_uint32)==4 ? 1 : -1]; + +static void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v) +{ + while (*fmt) { + switch (*fmt++) { + case ' ': break; + case '1': { unsigned char x = STBIW_UCHAR(va_arg(v, int)); + s->func(s->context,&x,1); + break; } + case '2': { int x = va_arg(v,int); + unsigned char b[2]; + b[0] = STBIW_UCHAR(x); + b[1] = STBIW_UCHAR(x>>8); + s->func(s->context,b,2); + break; } + case '4': { stbiw_uint32 x = va_arg(v,int); + unsigned char b[4]; + b[0]=STBIW_UCHAR(x); + b[1]=STBIW_UCHAR(x>>8); + b[2]=STBIW_UCHAR(x>>16); + b[3]=STBIW_UCHAR(x>>24); + s->func(s->context,b,4); + break; } + default: + STBIW_ASSERT(0); + return; + } + } +} + +static void stbiw__writef(stbi__write_context *s, const char *fmt, ...) +{ + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); +} + +static void stbiw__write_flush(stbi__write_context *s) +{ + if (s->buf_used) { + s->func(s->context, &s->buffer, s->buf_used); + s->buf_used = 0; + } +} + +static void stbiw__putc(stbi__write_context *s, unsigned char c) +{ + s->func(s->context, &c, 1); +} + +static void stbiw__write1(stbi__write_context *s, unsigned char a) +{ + if ((size_t)s->buf_used + 1 > sizeof(s->buffer)) + stbiw__write_flush(s); + s->buffer[s->buf_used++] = a; +} + +static void stbiw__write3(stbi__write_context *s, unsigned char a, unsigned char b, unsigned char c) +{ + int n; + if ((size_t)s->buf_used + 3 > sizeof(s->buffer)) + stbiw__write_flush(s); + n = s->buf_used; + s->buf_used = n+3; + s->buffer[n+0] = a; + s->buffer[n+1] = b; + s->buffer[n+2] = c; +} + +static void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, int write_alpha, int expand_mono, unsigned char *d) +{ + unsigned char bg[3] = { 255, 0, 255}, px[3]; + int k; + + if (write_alpha < 0) + stbiw__write1(s, d[comp - 1]); + + switch (comp) { + case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as 1-channel case + case 1: + if (expand_mono) + stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp + else + stbiw__write1(s, d[0]); // monochrome TGA + break; + case 4: + if (!write_alpha) { + // composite against pink background + for (k = 0; k < 3; ++k) + px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255; + stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]); + break; + } + /* FALLTHROUGH */ + case 3: + stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]); + break; + } + if (write_alpha > 0) + stbiw__write1(s, d[comp - 1]); +} + +static void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, void *data, int write_alpha, int scanline_pad, int expand_mono) +{ + stbiw_uint32 zero = 0; + int i,j, j_end; + + if (y <= 0) + return; + + if (stbi__flip_vertically_on_write) + vdir *= -1; + + if (vdir < 0) { + j_end = -1; j = y-1; + } else { + j_end = y; j = 0; + } + + for (; j != j_end; j += vdir) { + for (i=0; i < x; ++i) { + unsigned char *d = (unsigned char *) data + (j*x+i)*comp; + stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d); + } + stbiw__write_flush(s); + s->func(s->context, &zero, scanline_pad); + } +} + +static int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, int expand_mono, void *data, int alpha, int pad, const char *fmt, ...) +{ + if (y < 0 || x < 0) { + return 0; + } else { + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); + stbiw__write_pixels(s,rgb_dir,vdir,x,y,comp,data,alpha,pad, expand_mono); + return 1; + } +} + +static int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, const void *data) +{ + if (comp != 4) { + // write RGB bitmap + int pad = (-x*3) & 3; + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *) data,0,pad, + "11 4 22 4" "4 44 22 444444", + 'B', 'M', 14+40+(x*3+pad)*y, 0,0, 14+40, // file header + 40, x,y, 1,24, 0,0,0,0,0,0); // bitmap header + } else { + // RGBA bitmaps need a v4 header + // use BI_BITFIELDS mode with 32bpp and alpha mask + // (straight BI_RGB with alpha mask doesn't work in most readers) + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *)data,1,0, + "11 4 22 4" "4 44 22 444444 4444 4 444 444 444 444", + 'B', 'M', 14+108+x*y*4, 0, 0, 14+108, // file header + 108, x,y, 1,32, 3,0,0,0,0,0, 0xff0000,0xff00,0xff,0xff000000u, 0, 0,0,0, 0,0,0, 0,0,0, 0,0,0); // bitmap V4 header + } +} + +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_bmp_core(&s, x, y, comp, data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_bmp_core(&s, x, y, comp, data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif //!STBI_WRITE_NO_STDIO + +static int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, void *data) +{ + int has_alpha = (comp == 2 || comp == 4); + int colorbytes = has_alpha ? comp-1 : comp; + int format = colorbytes < 2 ? 3 : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3 + + if (y < 0 || x < 0) + return 0; + + if (!stbi_write_tga_with_rle) { + return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *) data, has_alpha, 0, + "111 221 2222 11", 0, 0, format, 0, 0, 0, 0, 0, x, y, (colorbytes + has_alpha) * 8, has_alpha * 8); + } else { + int i,j,k; + int jend, jdir; + + stbiw__writef(s, "111 221 2222 11", 0,0,format+8, 0,0,0, 0,0,x,y, (colorbytes + has_alpha) * 8, has_alpha * 8); + + if (stbi__flip_vertically_on_write) { + j = 0; + jend = y; + jdir = 1; + } else { + j = y-1; + jend = -1; + jdir = -1; + } + for (; j != jend; j += jdir) { + unsigned char *row = (unsigned char *) data + j * x * comp; + int len; + + for (i = 0; i < x; i += len) { + unsigned char *begin = row + i * comp; + int diff = 1; + len = 1; + + if (i < x - 1) { + ++len; + diff = memcmp(begin, row + (i + 1) * comp, comp); + if (diff) { + const unsigned char *prev = begin; + for (k = i + 2; k < x && len < 128; ++k) { + if (memcmp(prev, row + k * comp, comp)) { + prev += comp; + ++len; + } else { + --len; + break; + } + } + } else { + for (k = i + 2; k < x && len < 128; ++k) { + if (!memcmp(begin, row + k * comp, comp)) { + ++len; + } else { + break; + } + } + } + } + + if (diff) { + unsigned char header = STBIW_UCHAR(len - 1); + stbiw__write1(s, header); + for (k = 0; k < len; ++k) { + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp); + } + } else { + unsigned char header = STBIW_UCHAR(len - 129); + stbiw__write1(s, header); + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin); + } + } + } + stbiw__write_flush(s); + } + return 1; +} + +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_tga_core(&s, x, y, comp, (void *) data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_tga_core(&s, x, y, comp, (void *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR writer +// by Baldur Karlsson + +#define stbiw__max(a, b) ((a) > (b) ? (a) : (b)) + +#ifndef STBI_WRITE_NO_STDIO + +static void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear) +{ + int exponent; + float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2])); + + if (maxcomp < 1e-32f) { + rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0; + } else { + float normalize = (float) frexp(maxcomp, &exponent) * 256.0f/maxcomp; + + rgbe[0] = (unsigned char)(linear[0] * normalize); + rgbe[1] = (unsigned char)(linear[1] * normalize); + rgbe[2] = (unsigned char)(linear[2] * normalize); + rgbe[3] = (unsigned char)(exponent + 128); + } +} + +static void stbiw__write_run_data(stbi__write_context *s, int length, unsigned char databyte) +{ + unsigned char lengthbyte = STBIW_UCHAR(length+128); + STBIW_ASSERT(length+128 <= 255); + s->func(s->context, &lengthbyte, 1); + s->func(s->context, &databyte, 1); +} + +static void stbiw__write_dump_data(stbi__write_context *s, int length, unsigned char *data) +{ + unsigned char lengthbyte = STBIW_UCHAR(length); + STBIW_ASSERT(length <= 128); // inconsistent with spec but consistent with official code + s->func(s->context, &lengthbyte, 1); + s->func(s->context, data, length); +} + +static void stbiw__write_hdr_scanline(stbi__write_context *s, int width, int ncomp, unsigned char *scratch, float *scanline) +{ + unsigned char scanlineheader[4] = { 2, 2, 0, 0 }; + unsigned char rgbe[4]; + float linear[3]; + int x; + + scanlineheader[2] = (width&0xff00)>>8; + scanlineheader[3] = (width&0x00ff); + + /* skip RLE for images too small or large */ + if (width < 8 || width >= 32768) { + for (x=0; x < width; x++) { + switch (ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + s->func(s->context, rgbe, 4); + } + } else { + int c,r; + /* encode into scratch buffer */ + for (x=0; x < width; x++) { + switch(ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + scratch[x + width*0] = rgbe[0]; + scratch[x + width*1] = rgbe[1]; + scratch[x + width*2] = rgbe[2]; + scratch[x + width*3] = rgbe[3]; + } + + s->func(s->context, scanlineheader, 4); + + /* RLE each component separately */ + for (c=0; c < 4; c++) { + unsigned char *comp = &scratch[width*c]; + + x = 0; + while (x < width) { + // find first run + r = x; + while (r+2 < width) { + if (comp[r] == comp[r+1] && comp[r] == comp[r+2]) + break; + ++r; + } + if (r+2 >= width) + r = width; + // dump up to first run + while (x < r) { + int len = r-x; + if (len > 128) len = 128; + stbiw__write_dump_data(s, len, &comp[x]); + x += len; + } + // if there's a run, output it + if (r+2 < width) { // same test as what we break out of in search loop, so only true if we break'd + // find next byte after run + while (r < width && comp[r] == comp[x]) + ++r; + // output run up to r + while (x < r) { + int len = r-x; + if (len > 127) len = 127; + stbiw__write_run_data(s, len, comp[x]); + x += len; + } + } + } + } + } +} + +static int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, float *data) +{ + if (y <= 0 || x <= 0 || data == NULL) + return 0; + else { + // Each component is stored separately. Allocate scratch space for full output scanline. + unsigned char *scratch = (unsigned char *) STBIW_MALLOC(x*4); + int i, len; + char buffer[128]; + char header[] = "#?RADIANCE\n# Written by stb_image_write.h\nFORMAT=32-bit_rle_rgbe\n"; + s->func(s->context, header, sizeof(header)-1); + +#ifdef __STDC_LIB_EXT1__ + len = sprintf_s(buffer, sizeof(buffer), "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#else + len = sprintf(buffer, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#endif + s->func(s->context, buffer, len); + + for(i=0; i < y; i++) + stbiw__write_hdr_scanline(s, x, comp, scratch, data + comp*x*(stbi__flip_vertically_on_write ? y-1-i : i)); + STBIW_FREE(scratch); + return 1; + } +} + +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const float *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_hdr_core(&s, x, y, comp, (float *) data); +} + +STBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, const float *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_hdr_core(&s, x, y, comp, (float *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif // STBI_WRITE_NO_STDIO + + +////////////////////////////////////////////////////////////////////////////// +// +// PNG writer +// + +#ifndef STBIW_ZLIB_COMPRESS +// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() == vector<>::size() +#define stbiw__sbraw(a) ((int *) (void *) (a) - 2) +#define stbiw__sbm(a) stbiw__sbraw(a)[0] +#define stbiw__sbn(a) stbiw__sbraw(a)[1] + +#define stbiw__sbneedgrow(a,n) ((a)==0 || stbiw__sbn(a)+n >= stbiw__sbm(a)) +#define stbiw__sbmaybegrow(a,n) (stbiw__sbneedgrow(a,(n)) ? stbiw__sbgrow(a,n) : 0) +#define stbiw__sbgrow(a,n) stbiw__sbgrowf((void **) &(a), (n), sizeof(*(a))) + +#define stbiw__sbpush(a, v) (stbiw__sbmaybegrow(a,1), (a)[stbiw__sbn(a)++] = (v)) +#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0) +#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)),0 : 0) + +static void *stbiw__sbgrowf(void **arr, int increment, int itemsize) +{ + int m = *arr ? 2*stbiw__sbm(*arr)+increment : increment+1; + void *p = STBIW_REALLOC_SIZED(*arr ? stbiw__sbraw(*arr) : 0, *arr ? (stbiw__sbm(*arr)*itemsize + sizeof(int)*2) : 0, itemsize * m + sizeof(int)*2); + STBIW_ASSERT(p); + if (p) { + if (!*arr) ((int *) p)[1] = 0; + *arr = (void *) ((int *) p + 2); + stbiw__sbm(*arr) = m; + } + return *arr; +} + +static unsigned char *stbiw__zlib_flushf(unsigned char *data, unsigned int *bitbuffer, int *bitcount) +{ + while (*bitcount >= 8) { + stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer)); + *bitbuffer >>= 8; + *bitcount -= 8; + } + return data; +} + +static int stbiw__zlib_bitrev(int code, int codebits) +{ + int res=0; + while (codebits--) { + res = (res << 1) | (code & 1); + code >>= 1; + } + return res; +} + +static unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, int limit) +{ + int i; + for (i=0; i < limit && i < 258; ++i) + if (a[i] != b[i]) break; + return i; +} + +static unsigned int stbiw__zhash(unsigned char *data) +{ + stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16); + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + return hash; +} + +#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount)) +#define stbiw__zlib_add(code,codebits) \ + (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush()) +#define stbiw__zlib_huffa(b,c) stbiw__zlib_add(stbiw__zlib_bitrev(b,c),c) +// default huffman tables +#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8) +#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n)-144, 9) +#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n)-256,7) +#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n)-280,8) +#define stbiw__zlib_huff(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : (n) <= 255 ? stbiw__zlib_huff2(n) : (n) <= 279 ? stbiw__zlib_huff3(n) : stbiw__zlib_huff4(n)) +#define stbiw__zlib_huffb(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n)) + +#define stbiw__ZHASH 16384 + +#endif // STBIW_ZLIB_COMPRESS + +STBIWDEF unsigned char * stbi_zlib_compress(unsigned char *data, int data_len, int *out_len, int quality) +{ +#ifdef STBIW_ZLIB_COMPRESS + // user provided a zlib compress implementation, use that + return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality); +#else // use builtin + static unsigned short lengthc[] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 259 }; + static unsigned char lengtheb[]= { 0,0,0,0,0,0,0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 }; + static unsigned short distc[] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 32768 }; + static unsigned char disteb[] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13 }; + unsigned int bitbuf=0; + int i,j, bitcount=0; + unsigned char *out = NULL; + unsigned char ***hash_table = (unsigned char***) STBIW_MALLOC(stbiw__ZHASH * sizeof(unsigned char**)); + if (hash_table == NULL) + return NULL; + if (quality < 5) quality = 5; + + stbiw__sbpush(out, 0x78); // DEFLATE 32K window + stbiw__sbpush(out, 0x5e); // FLEVEL = 1 + stbiw__zlib_add(1,1); // BFINAL = 1 + stbiw__zlib_add(1,2); // BTYPE = 1 -- fixed huffman + + for (i=0; i < stbiw__ZHASH; ++i) + hash_table[i] = NULL; + + i=0; + while (i < data_len-3) { + // hash next 3 bytes of data to be compressed + int h = stbiw__zhash(data+i)&(stbiw__ZHASH-1), best=3; + unsigned char *bestloc = 0; + unsigned char **hlist = hash_table[h]; + int n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32768) { // if entry lies within window + int d = stbiw__zlib_countm(hlist[j], data+i, data_len-i); + if (d >= best) { best=d; bestloc=hlist[j]; } + } + } + // when hash table entry is too long, delete half the entries + if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2*quality) { + STBIW_MEMMOVE(hash_table[h], hash_table[h]+quality, sizeof(hash_table[h][0])*quality); + stbiw__sbn(hash_table[h]) = quality; + } + stbiw__sbpush(hash_table[h],data+i); + + if (bestloc) { + // "lazy matching" - check match at *next* byte, and if it's better, do cur byte as literal + h = stbiw__zhash(data+i+1)&(stbiw__ZHASH-1); + hlist = hash_table[h]; + n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32767) { + int e = stbiw__zlib_countm(hlist[j], data+i+1, data_len-i-1); + if (e > best) { // if next match is better, bail on current match + bestloc = NULL; + break; + } + } + } + } + + if (bestloc) { + int d = (int) (data+i - bestloc); // distance back + STBIW_ASSERT(d <= 32767 && best <= 258); + for (j=0; best > lengthc[j+1]-1; ++j); + stbiw__zlib_huff(j+257); + if (lengtheb[j]) stbiw__zlib_add(best - lengthc[j], lengtheb[j]); + for (j=0; d > distc[j+1]-1; ++j); + stbiw__zlib_add(stbiw__zlib_bitrev(j,5),5); + if (disteb[j]) stbiw__zlib_add(d - distc[j], disteb[j]); + i += best; + } else { + stbiw__zlib_huffb(data[i]); + ++i; + } + } + // write out final bytes + for (;i < data_len; ++i) + stbiw__zlib_huffb(data[i]); + stbiw__zlib_huff(256); // end of block + // pad with 0 bits to byte boundary + while (bitcount) + stbiw__zlib_add(0,1); + + for (i=0; i < stbiw__ZHASH; ++i) + (void) stbiw__sbfree(hash_table[i]); + STBIW_FREE(hash_table); + + // store uncompressed instead if compression was worse + if (stbiw__sbn(out) > data_len + 2 + ((data_len+32766)/32767)*5) { + stbiw__sbn(out) = 2; // truncate to DEFLATE 32K window and FLEVEL = 1 + for (j = 0; j < data_len;) { + int blocklen = data_len - j; + if (blocklen > 32767) blocklen = 32767; + stbiw__sbpush(out, data_len - j == blocklen); // BFINAL = ?, BTYPE = 0 -- no compression + stbiw__sbpush(out, STBIW_UCHAR(blocklen)); // LEN + stbiw__sbpush(out, STBIW_UCHAR(blocklen >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(~blocklen)); // NLEN + stbiw__sbpush(out, STBIW_UCHAR(~blocklen >> 8)); + memcpy(out+stbiw__sbn(out), data+j, blocklen); + stbiw__sbn(out) += blocklen; + j += blocklen; + } + } + + { + // compute adler32 on input + unsigned int s1=1, s2=0; + int blocklen = (int) (data_len % 5552); + j=0; + while (j < data_len) { + for (i=0; i < blocklen; ++i) { s1 += data[j+i]; s2 += s1; } + s1 %= 65521; s2 %= 65521; + j += blocklen; + blocklen = 5552; + } + stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s2)); + stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s1)); + } + *out_len = stbiw__sbn(out); + // make returned pointer freeable + STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len); + return (unsigned char *) stbiw__sbraw(out); +#endif // STBIW_ZLIB_COMPRESS +} + +static unsigned int stbiw__crc32(unsigned char *buffer, int len) +{ +#ifdef STBIW_CRC32 + return STBIW_CRC32(buffer, len); +#else + static unsigned int crc_table[256] = + { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }; + + unsigned int crc = ~0u; + int i; + for (i=0; i < len; ++i) + crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)]; + return ~crc; +#endif +} + +#define stbiw__wpng4(o,a,b,c,d) ((o)[0]=STBIW_UCHAR(a),(o)[1]=STBIW_UCHAR(b),(o)[2]=STBIW_UCHAR(c),(o)[3]=STBIW_UCHAR(d),(o)+=4) +#define stbiw__wp32(data,v) stbiw__wpng4(data, (v)>>24,(v)>>16,(v)>>8,(v)); +#define stbiw__wptag(data,s) stbiw__wpng4(data, s[0],s[1],s[2],s[3]) + +static void stbiw__wpcrc(unsigned char **data, int len) +{ + unsigned int crc = stbiw__crc32(*data - len - 4, len+4); + stbiw__wp32(*data, crc); +} + +static unsigned char stbiw__paeth(int a, int b, int c) +{ + int p = a + b - c, pa = abs(p-a), pb = abs(p-b), pc = abs(p-c); + if (pa <= pb && pa <= pc) return STBIW_UCHAR(a); + if (pb <= pc) return STBIW_UCHAR(b); + return STBIW_UCHAR(c); +} + +// @OPTIMIZE: provide an option that always forces left-predict or paeth predict +static void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, int width, int height, int y, int n, int filter_type, signed char *line_buffer) +{ + static int mapping[] = { 0,1,2,3,4 }; + static int firstmap[] = { 0,1,0,5,6 }; + int *mymap = (y != 0) ? mapping : firstmap; + int i; + int type = mymap[filter_type]; + unsigned char *z = pixels + stride_bytes * (stbi__flip_vertically_on_write ? height-1-y : y); + int signed_stride = stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes; + + if (type==0) { + memcpy(line_buffer, z, width*n); + return; + } + + // first loop isn't optimized since it's just one pixel + for (i = 0; i < n; ++i) { + switch (type) { + case 1: line_buffer[i] = z[i]; break; + case 2: line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: line_buffer[i] = z[i] - (z[i-signed_stride]>>1); break; + case 4: line_buffer[i] = (signed char) (z[i] - stbiw__paeth(0,z[i-signed_stride],0)); break; + case 5: line_buffer[i] = z[i]; break; + case 6: line_buffer[i] = z[i]; break; + } + } + switch (type) { + case 1: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-n]; break; + case 2: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - ((z[i-n] + z[i-signed_stride])>>1); break; + case 4: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], z[i-signed_stride], z[i-signed_stride-n]); break; + case 5: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - (z[i-n]>>1); break; + case 6: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], 0,0); break; + } +} + +STBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, int stride_bytes, int x, int y, int n, int *out_len) +{ + int force_filter = stbi_write_force_png_filter; + int ctype[5] = { -1, 0, 4, 2, 6 }; + unsigned char sig[8] = { 137,80,78,71,13,10,26,10 }; + unsigned char *out,*o, *filt, *zlib; + signed char *line_buffer; + int j,zlen; + + if (stride_bytes == 0) + stride_bytes = x * n; + + if (force_filter >= 5) { + force_filter = -1; + } + + filt = (unsigned char *) STBIW_MALLOC((x*n+1) * y); if (!filt) return 0; + line_buffer = (signed char *) STBIW_MALLOC(x * n); if (!line_buffer) { STBIW_FREE(filt); return 0; } + for (j=0; j < y; ++j) { + int filter_type; + if (force_filter > -1) { + filter_type = force_filter; + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, force_filter, line_buffer); + } else { // Estimate the best filter by running through all of them: + int best_filter = 0, best_filter_val = 0x7fffffff, est, i; + for (filter_type = 0; filter_type < 5; filter_type++) { + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, filter_type, line_buffer); + + // Estimate the entropy of the line using this filter; the less, the better. + est = 0; + for (i = 0; i < x*n; ++i) { + est += abs((signed char) line_buffer[i]); + } + if (est < best_filter_val) { + best_filter_val = est; + best_filter = filter_type; + } + } + if (filter_type != best_filter) { // If the last iteration already got us the best filter, don't redo it + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, best_filter, line_buffer); + filter_type = best_filter; + } + } + // when we get here, filter_type contains the filter type, and line_buffer contains the data + filt[j*(x*n+1)] = (unsigned char) filter_type; + STBIW_MEMMOVE(filt+j*(x*n+1)+1, line_buffer, x*n); + } + STBIW_FREE(line_buffer); + zlib = stbi_zlib_compress(filt, y*( x*n+1), &zlen, stbi_write_png_compression_level); + STBIW_FREE(filt); + if (!zlib) return 0; + + // each tag requires 12 bytes of overhead + out = (unsigned char *) STBIW_MALLOC(8 + 12+13 + 12+zlen + 12); + if (!out) return 0; + *out_len = 8 + 12+13 + 12+zlen + 12; + + o=out; + STBIW_MEMMOVE(o,sig,8); o+= 8; + stbiw__wp32(o, 13); // header length + stbiw__wptag(o, "IHDR"); + stbiw__wp32(o, x); + stbiw__wp32(o, y); + *o++ = 8; + *o++ = STBIW_UCHAR(ctype[n]); + *o++ = 0; + *o++ = 0; + *o++ = 0; + stbiw__wpcrc(&o,13); + + stbiw__wp32(o, zlen); + stbiw__wptag(o, "IDAT"); + STBIW_MEMMOVE(o, zlib, zlen); + o += zlen; + STBIW_FREE(zlib); + stbiw__wpcrc(&o, zlen); + + stbiw__wp32(o,0); + stbiw__wptag(o, "IEND"); + stbiw__wpcrc(&o,0); + + STBIW_ASSERT(o == out + *out_len); + + return out; +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, const void *data, int stride_bytes) +{ + FILE *f; + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + + f = stbiw__fopen(filename, "wb"); + if (!f) { STBIW_FREE(png); return 0; } + fwrite(png, 1, len, f); + fclose(f); + STBIW_FREE(png); + return 1; +} +#endif + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int stride_bytes) +{ + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + func(context, png, len); + STBIW_FREE(png); + return 1; +} + + +/* *************************************************************************** + * + * JPEG writer + * + * This is based on Jon Olick's jo_jpeg.cpp: + * public domain Simple, Minimalistic JPEG writer - http://www.jonolick.com/code.html + */ + +static const unsigned char stbiw__jpg_ZigZag[] = { 0,1,5,6,14,15,27,28,2,4,7,13,16,26,29,42,3,8,12,17,25,30,41,43,9,11,18, + 24,31,40,44,53,10,19,23,32,39,45,52,54,20,22,33,38,46,51,55,60,21,34,37,47,50,56,59,61,35,36,48,49,57,58,62,63 }; + +static void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, int *bitCntP, const unsigned short *bs) { + int bitBuf = *bitBufP, bitCnt = *bitCntP; + bitCnt += bs[1]; + bitBuf |= bs[0] << (24 - bitCnt); + while(bitCnt >= 8) { + unsigned char c = (bitBuf >> 16) & 255; + stbiw__putc(s, c); + if(c == 255) { + stbiw__putc(s, 0); + } + bitBuf <<= 8; + bitCnt -= 8; + } + *bitBufP = bitBuf; + *bitCntP = bitCnt; +} + +static void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, float *d4p, float *d5p, float *d6p, float *d7p) { + float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, d6 = *d6p, d7 = *d7p; + float z1, z2, z3, z4, z5, z11, z13; + + float tmp0 = d0 + d7; + float tmp7 = d0 - d7; + float tmp1 = d1 + d6; + float tmp6 = d1 - d6; + float tmp2 = d2 + d5; + float tmp5 = d2 - d5; + float tmp3 = d3 + d4; + float tmp4 = d3 - d4; + + // Even part + float tmp10 = tmp0 + tmp3; // phase 2 + float tmp13 = tmp0 - tmp3; + float tmp11 = tmp1 + tmp2; + float tmp12 = tmp1 - tmp2; + + d0 = tmp10 + tmp11; // phase 3 + d4 = tmp10 - tmp11; + + z1 = (tmp12 + tmp13) * 0.707106781f; // c4 + d2 = tmp13 + z1; // phase 5 + d6 = tmp13 - z1; + + // Odd part + tmp10 = tmp4 + tmp5; // phase 2 + tmp11 = tmp5 + tmp6; + tmp12 = tmp6 + tmp7; + + // The rotator is modified from fig 4-8 to avoid extra negations. + z5 = (tmp10 - tmp12) * 0.382683433f; // c6 + z2 = tmp10 * 0.541196100f + z5; // c2-c6 + z4 = tmp12 * 1.306562965f + z5; // c2+c6 + z3 = tmp11 * 0.707106781f; // c4 + + z11 = tmp7 + z3; // phase 5 + z13 = tmp7 - z3; + + *d5p = z13 + z2; // phase 6 + *d3p = z13 - z2; + *d1p = z11 + z4; + *d7p = z11 - z4; + + *d0p = d0; *d2p = d2; *d4p = d4; *d6p = d6; +} + +static void stbiw__jpg_calcBits(int val, unsigned short bits[2]) { + int tmp1 = val < 0 ? -val : val; + val = val < 0 ? val-1 : val; + bits[1] = 1; + while(tmp1 >>= 1) { + ++bits[1]; + } + bits[0] = val & ((1<0)&&(DU[end0pos]==0); --end0pos) { + } + // end0pos = first element in reverse order !=0 + if(end0pos == 0) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + return DU[0]; + } + for(i = 1; i <= end0pos; ++i) { + int startpos = i; + int nrzeroes; + unsigned short bits[2]; + for (; DU[i]==0 && i<=end0pos; ++i) { + } + nrzeroes = i-startpos; + if ( nrzeroes >= 16 ) { + int lng = nrzeroes>>4; + int nrmarker; + for (nrmarker=1; nrmarker <= lng; ++nrmarker) + stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes); + nrzeroes &= 15; + } + stbiw__jpg_calcBits(DU[i], bits); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes<<4)+bits[1]]); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); + } + if(end0pos != 63) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + } + return DU[0]; +} + +static int stbi_write_jpg_core(stbi__write_context *s, int width, int height, int comp, const void* data, int quality) { + // Constants that don't pollute global namespace + static const unsigned char std_dc_luminance_nrcodes[] = {0,0,1,5,1,1,1,1,1,1,0,0,0,0,0,0,0}; + static const unsigned char std_dc_luminance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_luminance_nrcodes[] = {0,0,2,1,3,3,2,4,3,5,5,4,4,0,0,1,0x7d}; + static const unsigned char std_ac_luminance_values[] = { + 0x01,0x02,0x03,0x00,0x04,0x11,0x05,0x12,0x21,0x31,0x41,0x06,0x13,0x51,0x61,0x07,0x22,0x71,0x14,0x32,0x81,0x91,0xa1,0x08, + 0x23,0x42,0xb1,0xc1,0x15,0x52,0xd1,0xf0,0x24,0x33,0x62,0x72,0x82,0x09,0x0a,0x16,0x17,0x18,0x19,0x1a,0x25,0x26,0x27,0x28, + 0x29,0x2a,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,0x59, + 0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x83,0x84,0x85,0x86,0x87,0x88,0x89, + 0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,0xb5,0xb6, + 0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xe1,0xe2, + 0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + static const unsigned char std_dc_chrominance_nrcodes[] = {0,0,3,1,1,1,1,1,1,1,1,1,0,0,0,0,0}; + static const unsigned char std_dc_chrominance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_chrominance_nrcodes[] = {0,0,2,1,2,4,4,3,4,7,5,4,4,0,1,2,0x77}; + static const unsigned char std_ac_chrominance_values[] = { + 0x00,0x01,0x02,0x03,0x11,0x04,0x05,0x21,0x31,0x06,0x12,0x41,0x51,0x07,0x61,0x71,0x13,0x22,0x32,0x81,0x08,0x14,0x42,0x91, + 0xa1,0xb1,0xc1,0x09,0x23,0x33,0x52,0xf0,0x15,0x62,0x72,0xd1,0x0a,0x16,0x24,0x34,0xe1,0x25,0xf1,0x17,0x18,0x19,0x1a,0x26, + 0x27,0x28,0x29,0x2a,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58, + 0x59,0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x82,0x83,0x84,0x85,0x86,0x87, + 0x88,0x89,0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4, + 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda, + 0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + // Huffman tables + static const unsigned short YDC_HT[256][2] = { {0,2},{2,3},{3,3},{4,3},{5,3},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9}}; + static const unsigned short UVDC_HT[256][2] = { {0,2},{1,2},{2,2},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9},{1022,10},{2046,11}}; + static const unsigned short YAC_HT[256][2] = { + {10,4},{0,2},{1,2},{4,3},{11,4},{26,5},{120,7},{248,8},{1014,10},{65410,16},{65411,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {12,4},{27,5},{121,7},{502,9},{2038,11},{65412,16},{65413,16},{65414,16},{65415,16},{65416,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {28,5},{249,8},{1015,10},{4084,12},{65417,16},{65418,16},{65419,16},{65420,16},{65421,16},{65422,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{503,9},{4085,12},{65423,16},{65424,16},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1016,10},{65430,16},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2039,11},{65438,16},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {123,7},{4086,12},{65446,16},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {250,8},{4087,12},{65454,16},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{32704,15},{65462,16},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65470,16},{65471,16},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65479,16},{65480,16},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1017,10},{65488,16},{65489,16},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{65497,16},{65498,16},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2040,11},{65506,16},{65507,16},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {65515,16},{65516,16},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65525,16},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const unsigned short UVAC_HT[256][2] = { + {0,2},{1,2},{4,3},{10,4},{24,5},{25,5},{56,6},{120,7},{500,9},{1014,10},{4084,12},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {11,4},{57,6},{246,8},{501,9},{2038,11},{4085,12},{65416,16},{65417,16},{65418,16},{65419,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {26,5},{247,8},{1015,10},{4086,12},{32706,15},{65420,16},{65421,16},{65422,16},{65423,16},{65424,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {27,5},{248,8},{1016,10},{4087,12},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{65430,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{502,9},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{65438,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1017,10},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{65446,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {121,7},{2039,11},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{65454,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2040,11},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{65462,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {249,8},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{65470,16},{65471,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {503,9},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{65479,16},{65480,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{65488,16},{65489,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{65497,16},{65498,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{65506,16},{65507,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{65515,16},{65516,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {16352,14},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{65525,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{32707,15},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const int YQT[] = {16,11,10,16,24,40,51,61,12,12,14,19,26,58,60,55,14,13,16,24,40,57,69,56,14,17,22,29,51,87,80,62,18,22, + 37,56,68,109,103,77,24,35,55,64,81,104,113,92,49,64,78,87,103,121,120,101,72,92,95,98,112,100,103,99}; + static const int UVQT[] = {17,18,24,47,99,99,99,99,18,21,26,66,99,99,99,99,24,26,56,99,99,99,99,99,47,66,99,99,99,99,99,99, + 99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99}; + static const float aasf[] = { 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f, + 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f }; + + int row, col, i, k, subsample; + float fdtbl_Y[64], fdtbl_UV[64]; + unsigned char YTable[64], UVTable[64]; + + if(!data || !width || !height || comp > 4 || comp < 1) { + return 0; + } + + quality = quality ? quality : 90; + subsample = quality <= 90 ? 1 : 0; + quality = quality < 1 ? 1 : quality > 100 ? 100 : quality; + quality = quality < 50 ? 5000 / quality : 200 - quality * 2; + + for(i = 0; i < 64; ++i) { + int uvti, yti = (YQT[i]*quality+50)/100; + YTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (yti < 1 ? 1 : yti > 255 ? 255 : yti); + uvti = (UVQT[i]*quality+50)/100; + UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (uvti < 1 ? 1 : uvti > 255 ? 255 : uvti); + } + + for(row = 0, k = 0; row < 8; ++row) { + for(col = 0; col < 8; ++col, ++k) { + fdtbl_Y[k] = 1 / (YTable [stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + } + } + + // Write Headers + { + static const unsigned char head0[] = { 0xFF,0xD8,0xFF,0xE0,0,0x10,'J','F','I','F',0,1,1,0,0,1,0,1,0,0,0xFF,0xDB,0,0x84,0 }; + static const unsigned char head2[] = { 0xFF,0xDA,0,0xC,3,1,0,2,0x11,3,0x11,0,0x3F,0 }; + const unsigned char head1[] = { 0xFF,0xC0,0,0x11,8,(unsigned char)(height>>8),STBIW_UCHAR(height),(unsigned char)(width>>8),STBIW_UCHAR(width), + 3,1,(unsigned char)(subsample?0x22:0x11),0,2,0x11,1,3,0x11,1,0xFF,0xC4,0x01,0xA2,0 }; + s->func(s->context, (void*)head0, sizeof(head0)); + s->func(s->context, (void*)YTable, sizeof(YTable)); + stbiw__putc(s, 1); + s->func(s->context, UVTable, sizeof(UVTable)); + s->func(s->context, (void*)head1, sizeof(head1)); + s->func(s->context, (void*)(std_dc_luminance_nrcodes+1), sizeof(std_dc_luminance_nrcodes)-1); + s->func(s->context, (void*)std_dc_luminance_values, sizeof(std_dc_luminance_values)); + stbiw__putc(s, 0x10); // HTYACinfo + s->func(s->context, (void*)(std_ac_luminance_nrcodes+1), sizeof(std_ac_luminance_nrcodes)-1); + s->func(s->context, (void*)std_ac_luminance_values, sizeof(std_ac_luminance_values)); + stbiw__putc(s, 1); // HTUDCinfo + s->func(s->context, (void*)(std_dc_chrominance_nrcodes+1), sizeof(std_dc_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_dc_chrominance_values, sizeof(std_dc_chrominance_values)); + stbiw__putc(s, 0x11); // HTUACinfo + s->func(s->context, (void*)(std_ac_chrominance_nrcodes+1), sizeof(std_ac_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_ac_chrominance_values, sizeof(std_ac_chrominance_values)); + s->func(s->context, (void*)head2, sizeof(head2)); + } + + // Encode 8x8 macroblocks + { + static const unsigned short fillBits[] = {0x7F, 7}; + int DCY=0, DCU=0, DCV=0; + int bitBuf=0, bitCnt=0; + // comp == 2 is grey+alpha (alpha is ignored) + int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0; + const unsigned char *dataR = (const unsigned char *)data; + const unsigned char *dataG = dataR + ofsG; + const unsigned char *dataB = dataR + ofsB; + int x, y, pos; + if(subsample) { + for(y = 0; y < height; y += 16) { + for(x = 0; x < width; x += 16) { + float Y[256], U[256], V[256]; + for(row = y, pos = 0; row < y+16; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+16; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+0, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+8, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+128, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+136, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + + // subsample U,V + { + float subU[64], subV[64]; + int yy, xx; + for(yy = 0, pos = 0; yy < 8; ++yy) { + for(xx = 0; xx < 8; ++xx, ++pos) { + int j = yy*32+xx*2; + subU[pos] = (U[j+0] + U[j+1] + U[j+16] + U[j+17]) * 0.25f; + subV[pos] = (V[j+0] + V[j+1] + V[j+16] + V[j+17]) * 0.25f; + } + } + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subU, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subV, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + } else { + for(y = 0; y < height; y += 8) { + for(x = 0; x < width; x += 8) { + float Y[64], U[64], V[64]; + for(row = y, pos = 0; row < y+8; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+8; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y, 8, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, U, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, V, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + + // Do the bit alignment of the EOI marker + stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits); + } + + // EOI + stbiw__putc(s, 0xFF); + stbiw__putc(s, 0xD9); + + return 1; +} + +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_jpg_core(&s, x, y, comp, (void *) data, quality); +} + + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_jpg_core(&s, x, y, comp, data, quality); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +#endif // STB_IMAGE_WRITE_IMPLEMENTATION + +/* Revision history + 1.16 (2021-07-11) + make Deflate code emit uncompressed blocks when it would otherwise expand + support writing BMPs with alpha channel + 1.15 (2020-07-13) unknown + 1.14 (2020-02-02) updated JPEG writer to downsample chroma channels + 1.13 + 1.12 + 1.11 (2019-08-11) + + 1.10 (2019-02-07) + support utf8 filenames in Windows; fix warnings and platform ifdefs + 1.09 (2018-02-11) + fix typo in zlib quality API, improve STB_I_W_STATIC in C++ + 1.08 (2018-01-29) + add stbi__flip_vertically_on_write, external zlib, zlib quality, choose PNG filter + 1.07 (2017-07-24) + doc fix + 1.06 (2017-07-23) + writing JPEG (using Jon Olick's code) + 1.05 ??? + 1.04 (2017-03-03) + monochrome BMP expansion + 1.03 ??? + 1.02 (2016-04-02) + avoid allocating large structures on the stack + 1.01 (2016-01-16) + STBIW_REALLOC_SIZED: support allocators with no realloc support + avoid race-condition in crc initialization + minor compile issues + 1.00 (2015-09-14) + installable file IO function + 0.99 (2015-09-13) + warning fixes; TGA rle support + 0.98 (2015-04-08) + added STBIW_MALLOC, STBIW_ASSERT etc + 0.97 (2015-01-18) + fixed HDR asserts, rewrote HDR rle logic + 0.96 (2015-01-17) + add HDR output + fix monochrome BMP + 0.95 (2014-08-17) + add monochrome TGA output + 0.94 (2014-05-31) + rename private functions to avoid conflicts with stb_image.h + 0.93 (2014-05-27) + warning fixes + 0.92 (2010-08-01) + casts to unsigned char to fix warnings + 0.91 (2010-07-17) + first public release + 0.90 first internal release +*/ + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/common/vision.cpp b/common/vision.cpp index 2b37ded16fa3c..ddd8eccfeb5ea 100644 --- a/common/vision.cpp +++ b/common/vision.cpp @@ -3,6 +3,9 @@ #define STB_IMAGE_IMPLEMENTATION #include "stb_image.h" +#define STB_IMAGE_RESIZE_IMPLEMENTATION +#include "stb_image_resize2.h" + #include #include @@ -21,6 +24,10 @@ llama_img * load_image_from_file(const char * fname) { if (!img) { throw std::runtime_error("failed to decode image bytes"); } + printf("Raw image data first 10 bytes:\n"); +for (int i = 0; i < 10; i++) { + printf("img[%d] = %d\n", i, img[i]); +} // printf("nx=%d ny=%d nc=%d\n", nx, ny, nc); // GGML_ASSERT(nc == 3); // for (int y = 0; y < ny; y++) { @@ -36,3 +43,312 @@ llama_img * load_image_from_file(const char * fname) { stbi_image_free(img); return result; } + +std::vector normalize_image_data(const llama_img* img) { + std::vector normalized(img->nx * img->ny * 3); + for (size_t i = 0; i < img->nx * img->ny * 3; i++) { + normalized[i] = img->data[i] / 255.0f * 1.46f; + } + return normalized; +} + +int clip(int a, int a_min, int a_max) { + return std::min(std::max(a, a_min), a_max); +} + +// MLlama specific types and functions + +struct Point { + int x; + int y; + Point(int x_, int y_) : x(x_), y(y_) {} +}; + +// TODO: extract these from the model. +struct ImagePreprocessParams { + const float mean[3] = {0.48145466f, 0.4578275f, 0.40821073f}; + const float std[3] = {0.26862954f, 0.26130258f, 0.27577711f}; + const int output_size = 560; + const int max_tiles = 4; +}; + +struct ProcessedImage { + std::vector data; + Point size; + Point aspect_ratio; +}; + +std::vector get_supported_aspect_ratios(int max_tiles) { + std::vector ratios; + for (int w = 0; w < max_tiles; w++) { + for (int h = 0; h < max_tiles; h++) { + if ((w + 1) * (h + 1) <= max_tiles) { + ratios.emplace_back(w + 1, h + 1); + } + } + } + return ratios; +} + +// Get optimal canvas size based on possible tile arrangements +Point get_optimal_tiled_canvas(Point image_size, int max_tiles, int tile_size) { + auto possible_tile_arrangements = get_supported_aspect_ratios(max_tiles); + std::vector possible_canvas_sizes; + std::vector scales; + + // Calculate possible canvas sizes and their scales + for (const auto& pta : possible_tile_arrangements) { + possible_canvas_sizes.emplace_back( + pta.x * tile_size, + pta.y * tile_size + ); + } + + // Calculate scales for each possible canvas size + for (const auto& pcs : possible_canvas_sizes) { + double scale_height = static_cast(pcs.y) / image_size.y; + double scale_width = static_cast(pcs.x) / image_size.x; + + scales.push_back(scale_width < scale_height ? scale_width : scale_height); + } + + // Find minimum upscale and maximum downscale + double min_upscale = std::numeric_limits::max(); + double max_downscale = 0.0; + bool has_upscale = false; + + for (double s : scales) { + if (s > 1.0) { + has_upscale = true; + min_upscale = std::min(min_upscale, s); + } else { + max_downscale = std::max(max_downscale, s); + } + } + + // Select the appropriate scale + double selected_scale = has_upscale ? min_upscale : max_downscale; + + // Find the smallest possible canvas with the selected scale + Point selected_canvas(0, 0); + for (size_t i = 0; i < possible_canvas_sizes.size(); i++) { + if (std::abs(scales[i] - selected_scale) < 1e-6) { + if (selected_canvas.x == 0 && selected_canvas.y == 0) { + selected_canvas = possible_canvas_sizes[i]; + } else if (possible_canvas_sizes[i].x * possible_canvas_sizes[i].y < + selected_canvas.x * selected_canvas.y) { + selected_canvas = possible_canvas_sizes[i]; + } + } + } + + return selected_canvas; +} + +// Split image into tiles based on the aspect ratio +std::vector> split_to_tiles(const std::vector & img, Point size, Point num_tiles_size +) { + std::vector> images; + int tile_height = size.y / num_tiles_size.y; + int tile_width = size.x / num_tiles_size.x; + + for (int h = 0; h < num_tiles_size.y; h++) { + for (int w = 0; w < num_tiles_size.x; w++) { + std::vector tile(tile_width * tile_height * 3); + + // Copy tile data + for (int y = 0; y < tile_height; y++) { + for (int x = 0; x < tile_width; x++) { + int src_x = w * tile_width + x; + int src_y = h * tile_height + y; + int src_idx = (src_y * size.x + src_x) * 3; + int dst_idx = (y * tile_width + x) * 3; + + for (int c = 0; c < 3; c++) { + tile[dst_idx + c] = img[src_idx + c]; + } + } + } + images.push_back(tile); + } + } + + return images; +} + +// Pad image to match the desired aspect ratio +std::vector pad_image( const std::vector& img, Point current_size, Point output_size, Point aspect_ratio) { + Point padded_size( + output_size.x * aspect_ratio.x, + output_size.y * aspect_ratio.y + ); + + std::vector padded(padded_size.x * padded_size.y * 3, 0); + + // Copy original image data + for (int y = 0; y < current_size.y; y++) { + for (int x = 0; x < current_size.x; x++) { + for (int c = 0; c < 3; c++) { + padded[(y * padded_size.x + x) * 3 + c] = + img[(y * current_size.x + x) * 3 + c]; + } + } + } + + return padded; +} + +Point get_image_size_fit_to_canvas(Point image_size, Point canvas_size, int tile_size) { + // Get canvas dimensions that are multiples of tile_size + int target_width = clip(image_size.x, tile_size, canvas_size.x); + int target_height = clip(image_size.y, tile_size, canvas_size.y); + + float scale_width = static_cast(target_width) / image_size.x; + float scale_height = static_cast(target_height) / image_size.y; + + int w, h; + if (scale_width < scale_height) { + w = target_width; + h = std::min(static_cast(std::floor(image_size.y * scale_width)), target_height); + } else { + w = std::min(static_cast(std::floor(image_size.x * scale_height)), target_width); + h = target_height; + } + + // Ensure dimensions are multiples of tile_size + w = (w / tile_size) * tile_size; + h = (h / tile_size) * tile_size; + + return Point(w, h); +} + +std::pair, std::pair> resize_image( + const unsigned char* input, + int input_w, + int input_h, + int output_w, + int output_h, + int n_tiles +) { + int tile_size = output_h; + + // Calculate optimal canvas size based on tile size + auto get_optimal_tiled_canvas = [](int width, int height, int n_tiles, int tile_size) { + int max_width = std::min(width, n_tiles * tile_size); + int max_height = std::min(height, n_tiles * tile_size); + return std::make_pair(max_width, max_height); + }; + + // Calculate new size to fit the canvas while maintaining aspect ratio + auto get_image_size_fit_to_canvas = [](int width, int height, int canvas_w, int canvas_h, int tile_size) { + double scale = std::min( + static_cast(canvas_w) / width, + static_cast(canvas_h) / height + ); + + int new_width = static_cast(width * scale); + int new_height = static_cast(height * scale); + + // Ensure dimensions are multiples of tile_size + new_width = (new_width / tile_size) * tile_size; + new_height = (new_height / tile_size) * tile_size; + + return std::make_pair(new_width, new_height); + }; + + // Calculate canvas and new image dimensions + auto [canvas_w, canvas_h] = get_optimal_tiled_canvas(input_w, input_h, n_tiles, tile_size); + auto [final_w, final_h] = get_image_size_fit_to_canvas(input_w, input_h, canvas_w, canvas_h, tile_size); + + // Calculate aspect ratio in tiles + int aspect_ratio_x = final_w / tile_size; + int aspect_ratio_y = final_h / tile_size; + + // Resize the image + std::vector output(final_w * final_h * 3); + + stbir_resize_uint8_linear( + input, input_w, input_h, input_w * 3, + output.data(), final_w, final_h, final_w * 3, + STBIR_RGB + ); + + return {output, {aspect_ratio_x, aspect_ratio_y}}; +} + +ProcessedImage preprocess_image(const char* fname) { + ImagePreprocessParams params; + + int nx, ny, nc; + auto * img = stbi_load(fname, &nx, &ny, &nc, 3); + if (!img) { + throw std::runtime_error("failed to load image"); + } + + Point image_size(nx, ny); + Point output_size(params.output_size, params.output_size); + int tile_size = output_size.y; + + // Get optimal canvas size using the new function + Point canvas_size = get_optimal_tiled_canvas(image_size, params.max_tiles, tile_size); + Point new_size = get_image_size_fit_to_canvas(image_size, canvas_size, tile_size); + + auto [resized_image, aspect_ratio] = resize_image( + img, nx, ny, + new_size.x, new_size.y, + params.max_tiles + ); + + stbi_image_free(img); + + // Calculate the full padded size (output_size * max_tiles) + Point padded_size( + output_size.x * params.max_tiles, + output_size.y * params.max_tiles + ); + + // Reserve the correct size for normalized data + // output_size * output_size * 3 channels * max_tiles * max_tiles + size_t expected_size = output_size.x * output_size.y * 3 * params.max_tiles * params.max_tiles; + std::vector normalized_data; + normalized_data.reserve(expected_size); + + // Process all possible tiles up to max_tiles * max_tiles + for (int ty = 0; ty < params.max_tiles; ty++) { + for (int tx = 0; tx < params.max_tiles; tx++) { + // Process each channel (R, G, B) separately + for (int c = 0; c < 3; c++) { + // Process each pixel in the tile + for (int y = 0; y < output_size.y; y++) { + for (int x = 0; x < output_size.x; x++) { + // Calculate source position + int src_x = tx * output_size.x + x; + int src_y = ty * output_size.y + y; + + float pixel = 0.0f; // Default to 0 for padding + + // Only read from source image if within bounds + if (src_x < new_size.x && src_y < new_size.y) { + pixel = resized_image[3 * (src_y * new_size.x + src_x) + c] / 255.0f; + } + + float normalized = (pixel - params.mean[c]) / params.std[c]; + normalized_data.push_back(normalized); + } + } + } + } + } + + return {normalized_data, new_size, Point(aspect_ratio.first, aspect_ratio.second)}; +} + +llama_img * mllama_load_image_from_file(const char * fname) { + ProcessedImage processed = preprocess_image(fname); + llama_img * result = llama_img_init(processed.size.x, processed.size.y); + + result->data = reinterpret_cast(malloc(processed.data.size() * sizeof(float))); + memcpy(result->data, processed.data.data(), processed.data.size() * sizeof(float)); + + return result; +} diff --git a/common/vision.h b/common/vision.h index 16c6325fd5ac2..2a70a7c35a21d 100644 --- a/common/vision.h +++ b/common/vision.h @@ -6,3 +6,5 @@ #include llama_img * load_image_from_file(const char * fname); +llama_img * mllama_load_image_from_file(const char * fname); +std::vector normalize_image_data(const llama_img* img); diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 1c856fe978b09..fc6b11c2ebcad 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -102,7 +102,6 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, self.metadata_override = metadata_override self.model_name = model_name self.dir_model_card = dir_model # overridden in convert_lora_to_gguf.py - self.is_lora = is_lora # true if model is used inside convert_lora_to_gguf.py self.preprocessor_config = self.load_preprocessor_config(self.dir_model) # Apply heuristics to figure out typical tensor encoding based on first layer tensor encoding type @@ -219,7 +218,8 @@ def match_model_tensor_name(self, name: str, key: gguf.MODEL_TENSOR, bid: int | def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str: new_name = self.tensor_map.get_name(key=name, try_suffixes=try_suffixes) - new_name_vision = self.v_tensor_map.get_name(key=name, try_suffixes=try_suffixes) + if hasattr(self, 'v_tensor_map') and self.v_tensor_map is not None: + new_name_vision = self.v_tensor_map.get_name(key=name, try_suffixes=try_suffixes) if new_name is not None: return new_name elif new_name_vision is not None: @@ -341,6 +341,20 @@ def prepare_tensors(self): gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2, gguf.MODEL_TENSOR.POSNET_NORM1, gguf.MODEL_TENSOR.POSNET_NORM2, + gguf.MODEL_TENSOR.MM_V_PATCH_EMBD, + gguf.MODEL_TENSOR.MM_V_TILE_POS_EMBD, + gguf.MODEL_TENSOR.MM_PROJECTOR, + gguf.MODEL_TENSOR.MM_V_PRE_TILE_POS_EMBD, + gguf.MODEL_TENSOR.MM_V_PRE_TILE_POS_GATE, + gguf.MODEL_TENSOR.MM_V_POST_TILE_POS_EMBD, + gguf.MODEL_TENSOR.MM_V_POST_TILE_POS_GATE, + #gguf.MODEL_TENSOR.TOKEN_EMBD, + #gguf.MODEL_TENSOR.OUTPUT, + #gguf.MODEL_TENSOR.FFN_GATE, + #gguf.MODEL_TENSOR.FFN_UP, + #gguf.MODEL_TENSOR.FFN_DOWN, + #gguf.MODEL_TENSOR.FFN_NORM, + #gguf.MODEL_TENSOR.ATTN_NORM, ) ) or not new_name.endswith(".weight") @@ -481,7 +495,7 @@ def load_hparams(dir_model: Path): hparams = json.load(f) if "text_config" in hparams: text_config = hparams["text_config"] - if "_name_or_path" in text_config: + if "_name_or_path" in text_config and text_config["_name_or_path"].strip(): text_config = AutoConfig.from_pretrained(text_config["_name_or_path"]).to_dict() hparams = {**text_config, **hparams} return hparams @@ -542,8 +556,10 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) - vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) - assert max(tokenizer.vocab.values()) < vocab_size + #vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) + vocab_size = len(tokenizer.vocab) + print(f"vocab_size: {vocab_size}") + assert max(tokenizer.vocab.values()) <= vocab_size tokpre = self.get_vocab_base_pre(tokenizer) @@ -699,6 +715,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "ad851be1dba641f2e3711822f816db2c265f788b37c63b4e1aeacb9ee92de8eb": # ref: https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct res = "gigachat" + if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": + # ref: https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct + res = "mllama" if res is None: logger.warning("\n") @@ -1751,6 +1770,163 @@ def prepare_tensors(self): raise ValueError(f"Unprocessed experts: {experts}") +@Model.register("MllamaForConditionalGeneration") +class MLlamaModel(Model): + model_arch = gguf.MODEL_ARCH.MLLAMA + + def __init__(self, *args, **kwargs): + from transformers import AutoTokenizer + self.dir_model = args[0] if args else kwargs.get('dir_model') + hparams = Model.load_hparams(self.dir_model) + #tokenizer = AutoTokenizer.from_pretrained(self.dir_model) + #vocab_size = len(tokenizer.vocab) + self.text_config = hparams.get("text_config", {}) + #self.text_config['vocab_size'] = vocab_size + #hparams['vocab_size'] = vocab_size + self.vision_config = hparams.get("vision_config", {}) + + kwargs['hparams'] = hparams + super().__init__(*args, **kwargs) + + def find_hparam(self, keys: Iterable[str], optional: bool = False) -> Any: + key = next((k for k in keys if k in self.hparams), None) + if key is not None: + return self.hparams[key] + if optional: + return None + raise KeyError(f"could not find any of: {keys} in self.text_config") + + def set_gguf_parameters(self): + n_embd = self.text_config["hidden_size"] + n_head = self.text_config["num_attention_heads"] + + image_token_index = self.find_hparam(["image_token_index"], optional=False) + self.gguf_writer.add_image_token_index(image_token_index) + + ### Text model metadata + self.gguf_writer.add_context_length(self.text_config["max_position_embeddings"]) + self.gguf_writer.add_block_count(self.text_config["num_hidden_layers"]) + self.gguf_writer.add_embedding_length(n_embd) + self.gguf_writer.add_feed_forward_length(self.text_config["intermediate_size"]) + self.gguf_writer.add_head_count(n_head) + self.gguf_writer.add_head_count_kv(self.text_config["num_key_value_heads"]) + self.gguf_writer.add_rope_freq_base(self.text_config["rope_theta"]) + self.gguf_writer.add_layer_norm_rms_eps(self.text_config["rms_norm_eps"]) + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_cross_attention_layers(self.text_config["cross_attention_layers"]) + self.gguf_writer.add_vocab_size(self.text_config["vocab_size"]) + self.gguf_writer.add_rope_dimension_count(n_embd // n_head) + #self.gguf_writer.add_activation_function(self.text_config["hidden_act"]) + + ### Vision metadata + # TODO(danbev): cross-attn is to indicate that this vision model uses + # cross-attention with the text model as opposed to the "prompt-based" + # type of model where the image patch embeddings are returned to + # caller and then passed to decode as normal tokens in the prompt. + self.gguf_writer.add_vision_type("cross-attn") + self.gguf_writer.add_vision_architecture(self.vision_config["model_type"]) + self.gguf_writer.add_vision_image_size(self.vision_config["image_size"]) + self.gguf_writer.add_vision_block_count(self.vision_config["num_hidden_layers"]) + self.gguf_writer.add_vision_layer_norm_rms_eps(self.vision_config["norm_eps"]) + self.gguf_writer.add_vision_embedding_length(self.vision_config["hidden_size"]) + self.gguf_writer.add_vision_activation_function(self.vision_config["hidden_act"]) + self.gguf_writer.add_vision_feed_forward_length(self.vision_config["intermediate_size"]) + self.gguf_writer.add_vision_global_block_count(self.vision_config["num_global_layers"]) + self.gguf_writer.add_vision_max_num_tiles(self.vision_config["max_num_tiles"]) + self.gguf_writer.add_vision_channels_count(self.vision_config["num_channels"]) + self.gguf_writer.add_vision_patch_size(self.vision_config["patch_size"]) + self.gguf_writer.add_vision_intermediate_layer_indices(self.vision_config["intermediate_layers_indices"]) + self.gguf_writer.add_vision_attention_head_count(self.vision_config["attention_heads"]) + self.gguf_writer.add_vision_output_dimension(self.vision_config["vision_output_dim"]) + self.gguf_writer.add_vision_model_type(self.vision_config["model_type"]) + + max_pos_embd = (self.vision_config["image_size"] // self.vision_config["patch_size"])**2 + 1 + self.gguf_writer.add_vision_clip_max_position_embeddings(max_pos_embd) + self.gguf_writer.add_vision_supported_aspect_ratios(self.vision_config["supported_aspect_ratios"]) + self.gguf_writer.add_vision_clip_image_mean(self.preprocessor_config["image_mean"]) + self.gguf_writer.add_vision_clip_image_std(self.preprocessor_config["image_std"]) + self.gguf_writer.add_vision_clip_projection_dimension(self.vision_config["vision_output_dim"]) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + n_head = self.hparams["num_attention_heads"] + n_kv_head = self.hparams.get("num_key_value_heads") + + # Permute the cross-attention Q and K tensors + if name.startswith('language_model') and name.endswith(("k_proj.weight", "k_proj.bias")): + print(f'[danbev] Permuting Language {name}') + data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head) + + if name.startswith('language_model') and name.endswith(("q_proj.weight", "q_proj.bias")): + print(f'[danbev] Permuting Language {name}') + data_torch = LlamaModel.permute(data_torch, n_head, n_head) + + # Just making the values exactly what ollama uses for testing and + # debugging purposes. + if name == "vision_model.pre_tile_positional_embedding.gate": + print("vision_model.pre_tile_positional_embedding.gate") + data_torch = data_torch.fill_(0.6351490020751953) + + if name == "vision_model.post_tile_positional_embedding.gate": + print("vision_model.post_tile_positional_embedding.gate") + data_torch = data_torch.fill_(-0.19474606215953827) + + if name == "vision_model.gated_positional_embedding.gate": + print("vision_model.gated_positional_embedding.gate") + data_torch = data_torch.fill_(1.8687903881072998) + + return [(self.map_tensor_name(name), data_torch)] + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + if rope_scaling := self.text_config.get("rope_scaling"): + if rope_scaling.get("rope_type", '').lower() == "llama3": + base = self.text_config.get("rope_theta", 500000.0) + dim = self.text_config.get("head_dim", self.text_config["hidden_size"] // self.text_config["num_attention_heads"]) + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + + factor = rope_scaling.get("factor", 8.0) + low_freq_factor = rope_scaling.get("low_freq_factor", 1.0) + high_freq_factor = rope_scaling.get("high_freq_factor", 4.0) + old_context_len = rope_scaling.get("original_max_position_embeddings", 8192) + + low_freq_wavelen = old_context_len / low_freq_factor + high_freq_wavelen = old_context_len / high_freq_factor + assert low_freq_wavelen != high_freq_wavelen + + rope_factors = [] + for freq in freqs: + wavelen = 2 * math.pi / freq + if wavelen < high_freq_wavelen: + rope_factors.append(1) + elif wavelen > low_freq_wavelen: + rope_factors.append(factor) + else: + smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor) + rope_factors.append(1 / ((1 - smooth) / factor + smooth)) + + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32)) + + tile_position_embd_gate = torch.tensor([-0.868790328502655], dtype=torch.float32) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.MM_V_TILE_POS_GATE, suffix=''), tile_position_embd_gate) + + def set_vocab(self): + tokens, toktypes, tokpre = self.get_vocab_base() + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab._set_special_token("start_header", 128006) # <|start_header_id|> + special_vocab._set_special_token("end_header", 128007) # <|end_header_id|> + #special_vocab._set_special_token("finetune_right_pad_id", 128004) # <|finetune_right_pad_id|> + #special_vocab._set_special_token("step", 128005) # <|step_id|> + special_vocab._set_special_token("eom", 128008) # <|eom_id|> + special_vocab._set_special_token("eot", 128009) # <|eot_id|> + special_vocab._set_special_token("python_tag", 128010) # <|python_tag|> + special_vocab._set_special_token("image", 128256) # <|image|> + special_vocab.add_to_gguf(self.gguf_writer) + + @Model.register("BitnetForCausalLM") class BitnetModel(Model): model_arch = gguf.MODEL_ARCH.BITNET diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 88058442f6dc4..f43d67bcca3eb 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -105,6 +105,7 @@ class TOKENIZER_TYPE(IntEnum): {"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", }, {"name": "roberta-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sentence-transformers/stsb-roberta-base"}, {"name": "gigachat", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct"}, + {"name": "mllama", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct", }, ] diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 66cfab2c3b796..72390821cface 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -48,6 +48,8 @@ else() add_subdirectory(run) add_subdirectory(simple) add_subdirectory(simple-chat) + add_subdirectory(simple-vision) + add_subdirectory(simple-vision-mllama) add_subdirectory(speculative) add_subdirectory(speculative-simple) add_subdirectory(tokenize) diff --git a/examples/batched-bench/batched-bench.cpp b/examples/batched-bench/batched-bench.cpp index a3b21ad6bce44..eac9d423f0053 100644 --- a/examples/batched-bench/batched-bench.cpp +++ b/examples/batched-bench/batched-bench.cpp @@ -70,6 +70,7 @@ int main(int argc, char ** argv) { n_tokens, batch.token + i, nullptr, + 0, batch.pos + i, batch.n_seq_id + i, batch.seq_id + i, diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 16f30c56ceac9..ab850299c63dd 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -441,6 +441,7 @@ struct llava_embd_batch { /*n_tokens =*/ n_tokens, /*tokens =*/ nullptr, /*embd =*/ embd, + /*n_embd =*/ 0, /*pos =*/ pos.data(), /*n_seq_id =*/ n_seq_id.data(), /*seq_id =*/ seq_ids.data(), diff --git a/examples/llava/qwen2vl-cli.cpp b/examples/llava/qwen2vl-cli.cpp index e86a60280aed6..115c5ce0fb758 100644 --- a/examples/llava/qwen2vl-cli.cpp +++ b/examples/llava/qwen2vl-cli.cpp @@ -70,6 +70,7 @@ static bool qwen2vl_eval_image_embed(llama_context * ctx_llama, const struct lla int32_t(n_eval), // n_tokens nullptr, // token (image_embed->embed+i*n_embd), // embed + 0, batch_mrope_pos.data(), // pos nullptr, // n_seq_id nullptr, // seq_id diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp index fd2b1c0112838..6203b9aea8c85 100644 --- a/examples/parallel/parallel.cpp +++ b/examples/parallel/parallel.cpp @@ -304,6 +304,7 @@ int main(int argc, char ** argv) { n_tokens, batch.token + i, nullptr, + 0, batch.pos + i, batch.n_seq_id + i, batch.seq_id + i, diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 64a84607c22d8..876be229879b1 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -660,6 +660,7 @@ static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector< n_tokens, batch.token + i, nullptr, + 0, batch.pos + i, batch.n_seq_id + i, batch.seq_id + i, diff --git a/examples/server/server.cpp b/examples/server/server.cpp index fa3682a920649..4a6f3309d12e6 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2812,6 +2812,7 @@ struct server_context { n_tokens, batch.token + i, nullptr, + 0, batch.pos + i, batch.n_seq_id + i, batch.seq_id + i, diff --git a/examples/simple-vision-mllama/CMakeLists.txt b/examples/simple-vision-mllama/CMakeLists.txt new file mode 100644 index 0000000000000..2a2eaaa120b24 --- /dev/null +++ b/examples/simple-vision-mllama/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-simple-vision-mllama) +add_executable(${TARGET} simple-vision.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) diff --git a/examples/simple-vision-mllama/README.md b/examples/simple-vision-mllama/README.md new file mode 100644 index 0000000000000..9cb9e254d404e --- /dev/null +++ b/examples/simple-vision-mllama/README.md @@ -0,0 +1,486 @@ +## Simple Vision MLlama Example +This examples tries to mimic the simple-vision example which uses the new +Vision API. This example uses Llama 3.2 Vision Instruct model which is somewhat +different from the simpl-vision example which is a "prompt-based" vision model +meaning that we first generate the patch embeddings for an image. These are +then passed to into llama.cpp using the llama_decode function, so the image +is passed like a prompt, hence the term prompt based. The Llama 3.2 Vision +model uses cross attention instead. + +> **This is a work in progress** +> The instructions below should work but there is an issue in the code +> around the backend schedular (I think). Increasing the number of layers to +> offload to the CPU will cause the output to be incorrect. I'm currently +> looking into this. + +### Model conversion +First we need to install the required python packages: +```console +$ python3 -m venv venv +$ source venv/bin/activate +(venv) pip install -r requirements.txt +``` +Next we can convert a LLaMA 3.2 Vision Instruct model to GGUF: +```console +(venv) python3 ./convert_hf_to_gguf.py --verbose /path/to/Llama-3.2-11B-Vision-Instruct --outfile models/llama-3-2-11b-f32.gguf --outtype f32 +``` + +And then quantize the model to a lower precision: +```console +(venv) ./build/bin/llama-quantize models/llama-3-2-11b-f32.gguf models/llama-3-2-11b-Q4_1.gguf Q4_K +``` + +### Building the example +This can be build with cmake using the following commands: +```console +$ cmake -S . -B build -DGGML_CUDA=On +$ cmake --build build --target llama-simple-vision-mllama -- -j 8 +``` +### Running the example: +```console +$ ./build/bin/llama-simple-vision-mllama -m models/llama-3-2-11b-Q4_K.gguf -ngl 30 --image eiffel-tower-3349075_1280.jpg +``` + +
Output + +```console +ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no +ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no +ggml_cuda_init: found 1 CUDA devices: + Device 0: NVIDIA GeForce RTX 4070, compute capability 8.9, VMM: yes +build: 4313 (f5499ea0) with cc (Ubuntu 13.2.0-23ubuntu4) 13.2.0 for x86_64-linux-gnu +llama_load_model_from_file: using device CUDA0 (NVIDIA GeForce RTX 4070) - 11743 MiB free +llama_model_loader: loaded meta data with 59 key-value pairs and 908 tensors from models/llama-3-2-11b-Q4_K.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = mllama +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.name str = Llama 3.2 11B Vision Instruct +llama_model_loader: - kv 3: general.finetune str = vision-instruct-new +llama_model_loader: - kv 4: general.basename str = llama-3.2 +llama_model_loader: - kv 5: general.size_label str = 11B +llama_model_loader: - kv 6: general.license str = llama3.2 +llama_model_loader: - kv 7: general.tags arr[str,6] = ["facebook", "meta", "pytorch", "llam... +llama_model_loader: - kv 8: general.languages arr[str,8] = ["en", "de", "fr", "it", "pt", "hi", ... +llama_model_loader: - kv 9: mllama.image_token_index u32 = 128256 +llama_model_loader: - kv 10: mllama.context_length u32 = 131072 +llama_model_loader: - kv 11: mllama.block_count u32 = 40 +llama_model_loader: - kv 12: mllama.embedding_length u32 = 4096 +llama_model_loader: - kv 13: mllama.feed_forward_length u32 = 14336 +llama_model_loader: - kv 14: mllama.attention.head_count u32 = 32 +llama_model_loader: - kv 15: mllama.attention.head_count_kv u32 = 8 +llama_model_loader: - kv 16: mllama.rope.freq_base f32 = 500000.000000 +llama_model_loader: - kv 17: mllama.attention.layer_norm_rms_epsilon f32 = 0.000010 +llama_model_loader: - kv 18: general.file_type u32 = 15 +llama_model_loader: - kv 19: mllama.cross_attention_layers arr[i32,8] = [3, 8, 13, 18, 23, 28, 33, 38] +llama_model_loader: - kv 20: mllama.vocab_size u32 = 128256 +llama_model_loader: - kv 21: mllama.rope.dimension_count u32 = 128 +llama_model_loader: - kv 22: vision.type str = cross-attn +llama_model_loader: - kv 23: vision.architecture str = mllama_vision_model +llama_model_loader: - kv 24: vision.image_size u32 = 560 +llama_model_loader: - kv 25: vision.block_count u32 = 32 +llama_model_loader: - kv 26: vision.attention.layer_norm_rms_epsilon f32 = 0.000010 +llama_model_loader: - kv 27: vision.embedding_length u32 = 1280 +llama_model_loader: - kv 28: vision.cross.mllama.activation_function str = gelu +llama_model_loader: - kv 29: vision.feed_forward_length u32 = 5120 +llama_model_loader: - kv 30: vision.cross.mllama.global_block_count u32 = 8 +llama_model_loader: - kv 31: vision.cross.mllama.max_num_tiles u32 = 4 +llama_model_loader: - kv 32: vision.cross.mllama.channels_count u32 = 3 +llama_model_loader: - kv 33: vision.patch_size u32 = 14 +llama_model_loader: - kv 34: vision.cross.mllama.intermediate_layers_indices arr[i32,5] = [3, 7, 15, 23, 30] +llama_model_loader: - kv 35: vision.attention.head_count u32 = 16 +llama_model_loader: - kv 36: vision.cross.mllama.output_dim u32 = 7680 +llama_model_loader: - kv 37: vision.cross.mllama.model_type str = mllama_vision_model +llama_model_loader: - kv 38: vision.clip.max_position_embeddings u32 = 1601 +llama_model_loader: - kv 39: vision.cross.mllama.supported_aspect_ratios arr[i32,16] = [1, 1, 1, 2, 1, 3, 1, 4, 2, 1, 2, 2, ... +llama_model_loader: - kv 40: vision.image_mean arr[f32,3] = [0.481455, 0.457828, 0.408211] +llama_model_loader: - kv 41: vision.image_std arr[f32,3] = [0.268630, 0.261303, 0.275777] +llama_model_loader: - kv 42: vision.clip.projection_dim u32 = 7680 +llama_model_loader: - kv 43: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 44: tokenizer.ggml.pre str = mllama +llama_model_loader: - kv 45: tokenizer.ggml.tokens arr[str,128257] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 46: tokenizer.ggml.token_type arr[i32,128257] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 47: tokenizer.ggml.merges arr[str,280147] = ["Ġ Ġ", "Ġ ĠĠĠ", "ĠĠ ĠĠ", "... +llama_model_loader: - kv 48: tokenizer.ggml.bos_token_id u32 = 128000 +llama_model_loader: - kv 49: tokenizer.ggml.eos_token_id u32 = 128009 +llama_model_loader: - kv 50: tokenizer.ggml.padding_token_id u32 = 128004 +llama_model_loader: - kv 51: tokenizer.ggml.start_header_token_id u32 = 128006 +llama_model_loader: - kv 52: tokenizer.ggml.end_header_token_id u32 = 128007 +llama_model_loader: - kv 53: tokenizer.ggml.eom_token_id u32 = 128008 +llama_model_loader: - kv 54: tokenizer.ggml.eot_token_id u32 = 128009 +llama_model_loader: - kv 55: tokenizer.ggml.python_tag_token_id u32 = 128010 +llama_model_loader: - kv 56: tokenizer.ggml.image_token_id u32 = 128256 +llama_model_loader: - kv 57: tokenizer.chat_template str = {{- bos_token }}\n{%- if custom_tools ... +llama_model_loader: - kv 58: general.quantization_version u32 = 2 +llama_model_loader: - type f32: 386 tensors +llama_model_loader: - type q4_K: 429 tensors +llama_model_loader: - type q6_K: 93 tensors +using mllama +llm_tokenizer_bpe: using default regex for BPE tokenization pre-processing +special token: 'tokenizer.ggml.bos_token_id' = 128000 +special token: 'tokenizer.ggml.eos_token_id' = 128009 +special token: 'tokenizer.ggml.eot_token_id' = 128009 +special token: 'tokenizer.ggml.eom_token_id' = 128008 +special token: 'tokenizer.ggml.padding_token_id' = 128004 +special token: 'tokenizer.ggml.image_token_id' = 128256 +llm_load_vocab: control token: 128256 '<|image|>' is not marked as EOG +llm_load_vocab: control token: 128255 '<|reserved_special_token_246|>' is not marked as EOG +llm_load_vocab: control token: 128250 '<|reserved_special_token_241|>' is not marked as EOG +llm_load_vocab: control token: 128247 '<|reserved_special_token_238|>' is not marked as EOG +llm_load_vocab: control token: 128244 '<|reserved_special_token_235|>' is not marked as EOG +llm_load_vocab: control token: 128243 '<|reserved_special_token_234|>' is not marked as EOG +llm_load_vocab: control token: 128242 '<|reserved_special_token_233|>' is not marked as EOG +llm_load_vocab: control token: 128241 '<|reserved_special_token_232|>' is not marked as EOG +llm_load_vocab: control token: 128236 '<|reserved_special_token_227|>' is not marked as EOG +llm_load_vocab: control token: 128232 '<|reserved_special_token_223|>' is not marked as EOG +llm_load_vocab: control token: 128231 '<|reserved_special_token_222|>' is not marked as EOG +llm_load_vocab: control token: 128229 '<|reserved_special_token_220|>' is not marked as EOG +llm_load_vocab: control token: 128226 '<|reserved_special_token_217|>' is not marked as EOG +llm_load_vocab: control token: 128219 '<|reserved_special_token_210|>' is not marked as EOG +llm_load_vocab: control token: 128215 '<|reserved_special_token_206|>' is not marked as EOG +llm_load_vocab: control token: 128214 '<|reserved_special_token_205|>' is not marked as EOG +llm_load_vocab: control token: 128208 '<|reserved_special_token_199|>' is not marked as EOG +llm_load_vocab: control token: 128207 '<|reserved_special_token_198|>' is not marked as EOG +llm_load_vocab: control token: 128205 '<|reserved_special_token_196|>' is not marked as EOG +llm_load_vocab: control token: 128201 '<|reserved_special_token_192|>' is not marked as EOG +llm_load_vocab: control token: 128200 '<|reserved_special_token_191|>' is not marked as EOG +llm_load_vocab: control token: 128199 '<|reserved_special_token_190|>' is not marked as EOG +llm_load_vocab: control token: 128197 '<|reserved_special_token_188|>' is not marked as EOG +llm_load_vocab: control token: 128195 '<|reserved_special_token_186|>' is not marked as EOG +llm_load_vocab: control token: 128194 '<|reserved_special_token_185|>' is not marked as EOG +llm_load_vocab: control token: 128189 '<|reserved_special_token_180|>' is not marked as EOG +llm_load_vocab: control token: 128188 '<|reserved_special_token_179|>' is not marked as EOG +llm_load_vocab: control token: 128186 '<|reserved_special_token_177|>' is not marked as EOG +llm_load_vocab: control token: 128185 '<|reserved_special_token_176|>' is not marked as EOG +llm_load_vocab: control token: 128181 '<|reserved_special_token_172|>' is not marked as EOG +llm_load_vocab: control token: 128180 '<|reserved_special_token_171|>' is not marked as EOG +llm_load_vocab: control token: 128179 '<|reserved_special_token_170|>' is not marked as EOG +llm_load_vocab: control token: 128178 '<|reserved_special_token_169|>' is not marked as EOG +llm_load_vocab: control token: 128177 '<|reserved_special_token_168|>' is not marked as EOG +llm_load_vocab: control token: 128176 '<|reserved_special_token_167|>' is not marked as EOG +llm_load_vocab: control token: 128172 '<|reserved_special_token_163|>' is not marked as EOG +llm_load_vocab: control token: 128171 '<|reserved_special_token_162|>' is not marked as EOG +llm_load_vocab: control token: 128170 '<|reserved_special_token_161|>' is not marked as EOG +llm_load_vocab: control token: 128169 '<|reserved_special_token_160|>' is not marked as EOG +llm_load_vocab: control token: 128166 '<|reserved_special_token_157|>' is not marked as EOG +llm_load_vocab: control token: 128163 '<|reserved_special_token_154|>' is not marked as EOG +llm_load_vocab: control token: 128159 '<|reserved_special_token_150|>' is not marked as EOG +llm_load_vocab: control token: 128157 '<|reserved_special_token_148|>' is not marked as EOG +llm_load_vocab: control token: 128156 '<|reserved_special_token_147|>' is not marked as EOG +llm_load_vocab: control token: 128155 '<|reserved_special_token_146|>' is not marked as EOG +llm_load_vocab: control token: 128152 '<|reserved_special_token_143|>' is not marked as EOG +llm_load_vocab: control token: 128150 '<|reserved_special_token_141|>' is not marked as EOG +llm_load_vocab: control token: 128148 '<|reserved_special_token_139|>' is not marked as EOG +llm_load_vocab: control token: 128147 '<|reserved_special_token_138|>' is not marked as EOG +llm_load_vocab: control token: 128145 '<|reserved_special_token_136|>' is not marked as EOG +llm_load_vocab: control token: 128143 '<|reserved_special_token_134|>' is not marked as EOG +llm_load_vocab: control token: 128142 '<|reserved_special_token_133|>' is not marked as EOG +llm_load_vocab: control token: 128139 '<|reserved_special_token_130|>' is not marked as EOG +llm_load_vocab: control token: 128137 '<|reserved_special_token_128|>' is not marked as EOG +llm_load_vocab: control token: 128136 '<|reserved_special_token_127|>' is not marked as EOG +llm_load_vocab: control token: 128135 '<|reserved_special_token_126|>' is not marked as EOG +llm_load_vocab: control token: 128134 '<|reserved_special_token_125|>' is not marked as EOG +llm_load_vocab: control token: 128132 '<|reserved_special_token_123|>' is not marked as EOG +llm_load_vocab: control token: 128129 '<|reserved_special_token_120|>' is not marked as EOG +llm_load_vocab: control token: 128125 '<|reserved_special_token_116|>' is not marked as EOG +llm_load_vocab: control token: 128124 '<|reserved_special_token_115|>' is not marked as EOG +llm_load_vocab: control token: 128123 '<|reserved_special_token_114|>' is not marked as EOG +llm_load_vocab: control token: 128120 '<|reserved_special_token_111|>' is not marked as EOG +llm_load_vocab: control token: 128116 '<|reserved_special_token_107|>' is not marked as EOG +llm_load_vocab: control token: 128113 '<|reserved_special_token_104|>' is not marked as EOG +llm_load_vocab: control token: 128111 '<|reserved_special_token_102|>' is not marked as EOG +llm_load_vocab: control token: 128110 '<|reserved_special_token_101|>' is not marked as EOG +llm_load_vocab: control token: 128109 '<|reserved_special_token_100|>' is not marked as EOG +llm_load_vocab: control token: 128107 '<|reserved_special_token_98|>' is not marked as EOG +llm_load_vocab: control token: 128104 '<|reserved_special_token_95|>' is not marked as EOG +llm_load_vocab: control token: 128103 '<|reserved_special_token_94|>' is not marked as EOG +llm_load_vocab: control token: 128102 '<|reserved_special_token_93|>' is not marked as EOG +llm_load_vocab: control token: 128098 '<|reserved_special_token_89|>' is not marked as EOG +llm_load_vocab: control token: 128092 '<|reserved_special_token_83|>' is not marked as EOG +llm_load_vocab: control token: 128091 '<|reserved_special_token_82|>' is not marked as EOG +llm_load_vocab: control token: 128090 '<|reserved_special_token_81|>' is not marked as EOG +llm_load_vocab: control token: 128088 '<|reserved_special_token_79|>' is not marked as EOG +llm_load_vocab: control token: 128086 '<|reserved_special_token_77|>' is not marked as EOG +llm_load_vocab: control token: 128082 '<|reserved_special_token_73|>' is not marked as EOG +llm_load_vocab: control token: 128079 '<|reserved_special_token_70|>' is not marked as EOG +llm_load_vocab: control token: 128077 '<|reserved_special_token_68|>' is not marked as EOG +llm_load_vocab: control token: 128076 '<|reserved_special_token_67|>' is not marked as EOG +llm_load_vocab: control token: 128074 '<|reserved_special_token_65|>' is not marked as EOG +llm_load_vocab: control token: 128069 '<|reserved_special_token_60|>' is not marked as EOG +llm_load_vocab: control token: 128068 '<|reserved_special_token_59|>' is not marked as EOG +llm_load_vocab: control token: 128066 '<|reserved_special_token_57|>' is not marked as EOG +llm_load_vocab: control token: 128064 '<|reserved_special_token_55|>' is not marked as EOG +llm_load_vocab: control token: 128063 '<|reserved_special_token_54|>' is not marked as EOG +llm_load_vocab: control token: 128061 '<|reserved_special_token_52|>' is not marked as EOG +llm_load_vocab: control token: 128060 '<|reserved_special_token_51|>' is not marked as EOG +llm_load_vocab: control token: 128058 '<|reserved_special_token_49|>' is not marked as EOG +llm_load_vocab: control token: 128055 '<|reserved_special_token_46|>' is not marked as EOG +llm_load_vocab: control token: 128047 '<|reserved_special_token_38|>' is not marked as EOG +llm_load_vocab: control token: 128046 '<|reserved_special_token_37|>' is not marked as EOG +llm_load_vocab: control token: 128045 '<|reserved_special_token_36|>' is not marked as EOG +llm_load_vocab: control token: 128044 '<|reserved_special_token_35|>' is not marked as EOG +llm_load_vocab: control token: 128039 '<|reserved_special_token_30|>' is not marked as EOG +llm_load_vocab: control token: 128037 '<|reserved_special_token_28|>' is not marked as EOG +llm_load_vocab: control token: 128036 '<|reserved_special_token_27|>' is not marked as EOG +llm_load_vocab: control token: 128033 '<|reserved_special_token_24|>' is not marked as EOG +llm_load_vocab: control token: 128029 '<|reserved_special_token_20|>' is not marked as EOG +llm_load_vocab: control token: 128028 '<|reserved_special_token_19|>' is not marked as EOG +llm_load_vocab: control token: 128025 '<|reserved_special_token_16|>' is not marked as EOG +llm_load_vocab: control token: 128024 '<|reserved_special_token_15|>' is not marked as EOG +llm_load_vocab: control token: 128023 '<|reserved_special_token_14|>' is not marked as EOG +llm_load_vocab: control token: 128022 '<|reserved_special_token_13|>' is not marked as EOG +llm_load_vocab: control token: 128019 '<|reserved_special_token_10|>' is not marked as EOG +llm_load_vocab: control token: 128017 '<|reserved_special_token_8|>' is not marked as EOG +llm_load_vocab: control token: 128016 '<|reserved_special_token_7|>' is not marked as EOG +llm_load_vocab: control token: 128014 '<|reserved_special_token_5|>' is not marked as EOG +llm_load_vocab: control token: 128012 '<|reserved_special_token_3|>' is not marked as EOG +llm_load_vocab: control token: 128011 '<|reserved_special_token_2|>' is not marked as EOG +llm_load_vocab: control token: 128004 '<|finetune_right_pad_id|>' is not marked as EOG +llm_load_vocab: control token: 128002 '<|reserved_special_token_0|>' is not marked as EOG +llm_load_vocab: control token: 128253 '<|reserved_special_token_244|>' is not marked as EOG +llm_load_vocab: control token: 128191 '<|reserved_special_token_182|>' is not marked as EOG +llm_load_vocab: control token: 128184 '<|reserved_special_token_175|>' is not marked as EOG +llm_load_vocab: control token: 128138 '<|reserved_special_token_129|>' is not marked as EOG +llm_load_vocab: control token: 128183 '<|reserved_special_token_174|>' is not marked as EOG +llm_load_vocab: control token: 128041 '<|reserved_special_token_32|>' is not marked as EOG +llm_load_vocab: control token: 128049 '<|reserved_special_token_40|>' is not marked as EOG +llm_load_vocab: control token: 128093 '<|reserved_special_token_84|>' is not marked as EOG +llm_load_vocab: control token: 128216 '<|reserved_special_token_207|>' is not marked as EOG +llm_load_vocab: control token: 128108 '<|reserved_special_token_99|>' is not marked as EOG +llm_load_vocab: control token: 128209 '<|reserved_special_token_200|>' is not marked as EOG +llm_load_vocab: control token: 128146 '<|reserved_special_token_137|>' is not marked as EOG +llm_load_vocab: control token: 128032 '<|reserved_special_token_23|>' is not marked as EOG +llm_load_vocab: control token: 128130 '<|reserved_special_token_121|>' is not marked as EOG +llm_load_vocab: control token: 128202 '<|reserved_special_token_193|>' is not marked as EOG +llm_load_vocab: control token: 128075 '<|reserved_special_token_66|>' is not marked as EOG +llm_load_vocab: control token: 128096 '<|reserved_special_token_87|>' is not marked as EOG +llm_load_vocab: control token: 128187 '<|reserved_special_token_178|>' is not marked as EOG +llm_load_vocab: control token: 128144 '<|reserved_special_token_135|>' is not marked as EOG +llm_load_vocab: control token: 128230 '<|reserved_special_token_221|>' is not marked as EOG +llm_load_vocab: control token: 128007 '<|end_header_id|>' is not marked as EOG +llm_load_vocab: control token: 128056 '<|reserved_special_token_47|>' is not marked as EOG +llm_load_vocab: control token: 128057 '<|reserved_special_token_48|>' is not marked as EOG +llm_load_vocab: control token: 128062 '<|reserved_special_token_53|>' is not marked as EOG +llm_load_vocab: control token: 128154 '<|reserved_special_token_145|>' is not marked as EOG +llm_load_vocab: control token: 128153 '<|reserved_special_token_144|>' is not marked as EOG +llm_load_vocab: control token: 128213 '<|reserved_special_token_204|>' is not marked as EOG +llm_load_vocab: control token: 128173 '<|reserved_special_token_164|>' is not marked as EOG +llm_load_vocab: control token: 128161 '<|reserved_special_token_152|>' is not marked as EOG +llm_load_vocab: control token: 128042 '<|reserved_special_token_33|>' is not marked as EOG +llm_load_vocab: control token: 128182 '<|reserved_special_token_173|>' is not marked as EOG +llm_load_vocab: control token: 128095 '<|reserved_special_token_86|>' is not marked as EOG +llm_load_vocab: control token: 128119 '<|reserved_special_token_110|>' is not marked as EOG +llm_load_vocab: control token: 128237 '<|reserved_special_token_228|>' is not marked as EOG +llm_load_vocab: control token: 128149 '<|reserved_special_token_140|>' is not marked as EOG +llm_load_vocab: control token: 128043 '<|reserved_special_token_34|>' is not marked as EOG +llm_load_vocab: control token: 128140 '<|reserved_special_token_131|>' is not marked as EOG +llm_load_vocab: control token: 128174 '<|reserved_special_token_165|>' is not marked as EOG +llm_load_vocab: control token: 128240 '<|reserved_special_token_231|>' is not marked as EOG +llm_load_vocab: control token: 128158 '<|reserved_special_token_149|>' is not marked as EOG +llm_load_vocab: control token: 128053 '<|reserved_special_token_44|>' is not marked as EOG +llm_load_vocab: control token: 128027 '<|reserved_special_token_18|>' is not marked as EOG +llm_load_vocab: control token: 128003 '<|reserved_special_token_1|>' is not marked as EOG +llm_load_vocab: control token: 128020 '<|reserved_special_token_11|>' is not marked as EOG +llm_load_vocab: control token: 128117 '<|reserved_special_token_108|>' is not marked as EOG +llm_load_vocab: control token: 128162 '<|reserved_special_token_153|>' is not marked as EOG +llm_load_vocab: control token: 128227 '<|reserved_special_token_218|>' is not marked as EOG +llm_load_vocab: control token: 128160 '<|reserved_special_token_151|>' is not marked as EOG +llm_load_vocab: control token: 128013 '<|reserved_special_token_4|>' is not marked as EOG +llm_load_vocab: control token: 128089 '<|reserved_special_token_80|>' is not marked as EOG +llm_load_vocab: control token: 128164 '<|reserved_special_token_155|>' is not marked as EOG +llm_load_vocab: control token: 128114 '<|reserved_special_token_105|>' is not marked as EOG +llm_load_vocab: control token: 128251 '<|reserved_special_token_242|>' is not marked as EOG +llm_load_vocab: control token: 128126 '<|reserved_special_token_117|>' is not marked as EOG +llm_load_vocab: control token: 128054 '<|reserved_special_token_45|>' is not marked as EOG +llm_load_vocab: control token: 128225 '<|reserved_special_token_216|>' is not marked as EOG +llm_load_vocab: control token: 128248 '<|reserved_special_token_239|>' is not marked as EOG +llm_load_vocab: control token: 128252 '<|reserved_special_token_243|>' is not marked as EOG +llm_load_vocab: control token: 128217 '<|reserved_special_token_208|>' is not marked as EOG +llm_load_vocab: control token: 128006 '<|start_header_id|>' is not marked as EOG +llm_load_vocab: control token: 128212 '<|reserved_special_token_203|>' is not marked as EOG +llm_load_vocab: control token: 128078 '<|reserved_special_token_69|>' is not marked as EOG +llm_load_vocab: control token: 128238 '<|reserved_special_token_229|>' is not marked as EOG +llm_load_vocab: control token: 128087 '<|reserved_special_token_78|>' is not marked as EOG +llm_load_vocab: control token: 128228 '<|reserved_special_token_219|>' is not marked as EOG +llm_load_vocab: control token: 128059 '<|reserved_special_token_50|>' is not marked as EOG +llm_load_vocab: control token: 128101 '<|reserved_special_token_92|>' is not marked as EOG +llm_load_vocab: control token: 128210 '<|reserved_special_token_201|>' is not marked as EOG +llm_load_vocab: control token: 128085 '<|reserved_special_token_76|>' is not marked as EOG +llm_load_vocab: control token: 128072 '<|reserved_special_token_63|>' is not marked as EOG +llm_load_vocab: control token: 128071 '<|reserved_special_token_62|>' is not marked as EOG +llm_load_vocab: control token: 128050 '<|reserved_special_token_41|>' is not marked as EOG +llm_load_vocab: control token: 128198 '<|reserved_special_token_189|>' is not marked as EOG +llm_load_vocab: control token: 128073 '<|reserved_special_token_64|>' is not marked as EOG +llm_load_vocab: control token: 128000 '<|begin_of_text|>' is not marked as EOG +llm_load_vocab: control token: 128224 '<|reserved_special_token_215|>' is not marked as EOG +llm_load_vocab: control token: 128218 '<|reserved_special_token_209|>' is not marked as EOG +llm_load_vocab: control token: 128112 '<|reserved_special_token_103|>' is not marked as EOG +llm_load_vocab: control token: 128204 '<|reserved_special_token_195|>' is not marked as EOG +llm_load_vocab: control token: 128052 '<|reserved_special_token_43|>' is not marked as EOG +llm_load_vocab: control token: 128031 '<|reserved_special_token_22|>' is not marked as EOG +llm_load_vocab: control token: 128118 '<|reserved_special_token_109|>' is not marked as EOG +llm_load_vocab: control token: 128010 '<|python_tag|>' is not marked as EOG +llm_load_vocab: control token: 128239 '<|reserved_special_token_230|>' is not marked as EOG +llm_load_vocab: control token: 128203 '<|reserved_special_token_194|>' is not marked as EOG +llm_load_vocab: control token: 128133 '<|reserved_special_token_124|>' is not marked as EOG +llm_load_vocab: control token: 128249 '<|reserved_special_token_240|>' is not marked as EOG +llm_load_vocab: control token: 128168 '<|reserved_special_token_159|>' is not marked as EOG +llm_load_vocab: control token: 128128 '<|reserved_special_token_119|>' is not marked as EOG +llm_load_vocab: control token: 128106 '<|reserved_special_token_97|>' is not marked as EOG +llm_load_vocab: control token: 128040 '<|reserved_special_token_31|>' is not marked as EOG +llm_load_vocab: control token: 128233 '<|reserved_special_token_224|>' is not marked as EOG +llm_load_vocab: control token: 128167 '<|reserved_special_token_158|>' is not marked as EOG +llm_load_vocab: control token: 128131 '<|reserved_special_token_122|>' is not marked as EOG +llm_load_vocab: control token: 128115 '<|reserved_special_token_106|>' is not marked as EOG +llm_load_vocab: control token: 128235 '<|reserved_special_token_226|>' is not marked as EOG +llm_load_vocab: control token: 128192 '<|reserved_special_token_183|>' is not marked as EOG +llm_load_vocab: control token: 128065 '<|reserved_special_token_56|>' is not marked as EOG +llm_load_vocab: control token: 128141 '<|reserved_special_token_132|>' is not marked as EOG +llm_load_vocab: control token: 128097 '<|reserved_special_token_88|>' is not marked as EOG +llm_load_vocab: control token: 128099 '<|reserved_special_token_90|>' is not marked as EOG +llm_load_vocab: control token: 128193 '<|reserved_special_token_184|>' is not marked as EOG +llm_load_vocab: control token: 128094 '<|reserved_special_token_85|>' is not marked as EOG +llm_load_vocab: control token: 128151 '<|reserved_special_token_142|>' is not marked as EOG +llm_load_vocab: control token: 128223 '<|reserved_special_token_214|>' is not marked as EOG +llm_load_vocab: control token: 128234 '<|reserved_special_token_225|>' is not marked as EOG +llm_load_vocab: control token: 128221 '<|reserved_special_token_212|>' is not marked as EOG +llm_load_vocab: control token: 128035 '<|reserved_special_token_26|>' is not marked as EOG +llm_load_vocab: control token: 128034 '<|reserved_special_token_25|>' is not marked as EOG +llm_load_vocab: control token: 128254 '<|reserved_special_token_245|>' is not marked as EOG +llm_load_vocab: control token: 128196 '<|reserved_special_token_187|>' is not marked as EOG +llm_load_vocab: control token: 128100 '<|reserved_special_token_91|>' is not marked as EOG +llm_load_vocab: control token: 128190 '<|reserved_special_token_181|>' is not marked as EOG +llm_load_vocab: control token: 128211 '<|reserved_special_token_202|>' is not marked as EOG +llm_load_vocab: control token: 128175 '<|reserved_special_token_166|>' is not marked as EOG +llm_load_vocab: control token: 128084 '<|reserved_special_token_75|>' is not marked as EOG +llm_load_vocab: control token: 128081 '<|reserved_special_token_72|>' is not marked as EOG +llm_load_vocab: control token: 128105 '<|reserved_special_token_96|>' is not marked as EOG +llm_load_vocab: control token: 128083 '<|reserved_special_token_74|>' is not marked as EOG +llm_load_vocab: control token: 128220 '<|reserved_special_token_211|>' is not marked as EOG +llm_load_vocab: control token: 128018 '<|reserved_special_token_9|>' is not marked as EOG +llm_load_vocab: control token: 128005 '<|step_id|>' is not marked as EOG +llm_load_vocab: control token: 128051 '<|reserved_special_token_42|>' is not marked as EOG +llm_load_vocab: control token: 128206 '<|reserved_special_token_197|>' is not marked as EOG +llm_load_vocab: control token: 128048 '<|reserved_special_token_39|>' is not marked as EOG +llm_load_vocab: control token: 128165 '<|reserved_special_token_156|>' is not marked as EOG +llm_load_vocab: control token: 128021 '<|reserved_special_token_12|>' is not marked as EOG +llm_load_vocab: control token: 128070 '<|reserved_special_token_61|>' is not marked as EOG +llm_load_vocab: control token: 128246 '<|reserved_special_token_237|>' is not marked as EOG +llm_load_vocab: control token: 128122 '<|reserved_special_token_113|>' is not marked as EOG +llm_load_vocab: control token: 128080 '<|reserved_special_token_71|>' is not marked as EOG +llm_load_vocab: control token: 128038 '<|reserved_special_token_29|>' is not marked as EOG +llm_load_vocab: control token: 128245 '<|reserved_special_token_236|>' is not marked as EOG +llm_load_vocab: control token: 128030 '<|reserved_special_token_21|>' is not marked as EOG +llm_load_vocab: control token: 128222 '<|reserved_special_token_213|>' is not marked as EOG +llm_load_vocab: control token: 128067 '<|reserved_special_token_58|>' is not marked as EOG +llm_load_vocab: control token: 128121 '<|reserved_special_token_112|>' is not marked as EOG +llm_load_vocab: control token: 128015 '<|reserved_special_token_6|>' is not marked as EOG +llm_load_vocab: control token: 128026 '<|reserved_special_token_17|>' is not marked as EOG +llm_load_vocab: control token: 128127 '<|reserved_special_token_118|>' is not marked as EOG +llm_load_vocab: special tokens cache size = 257 +llm_load_vocab: token to piece cache size = 0.7999 MB +llm_load_print_meta: format = GGUF V3 (latest) +llm_load_print_meta: arch = mllama +llm_load_print_meta: vocab type = BPE +llm_load_print_meta: n_vocab = 128256 +llm_load_print_meta: n_merges = 280147 +llm_load_print_meta: vocab_only = 0 +llm_load_print_meta: n_ctx_train = 131072 +llm_load_print_meta: n_embd = 4096 +llm_load_print_meta: n_layer = 40 +llm_load_print_meta: n_head = 32 +llm_load_print_meta: n_head_kv = 8 +llm_load_print_meta: n_rot = 128 +llm_load_print_meta: n_swa = 0 +llm_load_print_meta: n_embd_head_k = 128 +llm_load_print_meta: n_embd_head_v = 128 +llm_load_print_meta: n_gqa = 4 +llm_load_print_meta: n_embd_k_gqa = 1024 +llm_load_print_meta: n_embd_v_gqa = 1024 +llm_load_print_meta: f_norm_eps = 0.0e+00 +llm_load_print_meta: f_norm_rms_eps = 1.0e-05 +llm_load_print_meta: f_clamp_kqv = 0.0e+00 +llm_load_print_meta: f_max_alibi_bias = 0.0e+00 +llm_load_print_meta: f_logit_scale = 0.0e+00 +llm_load_print_meta: n_ff = 14336 +llm_load_print_meta: n_expert = 0 +llm_load_print_meta: n_expert_used = 0 +llm_load_print_meta: causal attn = 1 +llm_load_print_meta: pooling type = 0 +llm_load_print_meta: rope type = 0 +llm_load_print_meta: rope scaling = linear +llm_load_print_meta: freq_base_train = 500000.0 +llm_load_print_meta: freq_scale_train = 1 +llm_load_print_meta: n_ctx_orig_yarn = 131072 +llm_load_print_meta: rope_finetuned = unknown +llm_load_print_meta: ssm_d_conv = 0 +llm_load_print_meta: ssm_d_inner = 0 +llm_load_print_meta: ssm_d_state = 0 +llm_load_print_meta: ssm_dt_rank = 0 +llm_load_print_meta: ssm_dt_b_c_rms = 0 +llm_load_print_meta: model type = 11B +llm_load_print_meta: model ftype = Q4_K - Medium +llm_load_print_meta: model params = 10.67 B +llm_load_print_meta: model size = 6.68 GiB (5.38 BPW) +llm_load_print_meta: general.name = Llama 3.2 11B Vision Instruct +llm_load_print_meta: BOS token = 128000 '<|begin_of_text|>' +llm_load_print_meta: EOS token = 128009 '<|eot_id|>' +llm_load_print_meta: EOT token = 128009 '<|eot_id|>' +llm_load_print_meta: EOM token = 128008 '<|eom_id|>' +llm_load_print_meta: PAD token = 128004 '<|finetune_right_pad_id|>' +llm_load_print_meta: LF token = 128 'Ä' +llm_load_print_meta: EOG token = 128001 '<|end_of_text|>' +llm_load_print_meta: EOG token = 128008 '<|eom_id|>' +llm_load_print_meta: EOG token = 128009 '<|eot_id|>' +llm_load_print_meta: max token length = 256 +llm_load_tensors: tensor 'token_embd.weight' (q4_K) (and 108 others) cannot be used with preferred buffer type CUDA_Host, using CPU instead +llm_load_tensors: offloading 30 repeating layers to GPU +llm_load_tensors: offloaded 30/41 layers to GPU +llm_load_tensors: CUDA0 model buffer size = 4208.35 MiB +llm_load_tensors: CPU_Mapped model buffer size = 2886.00 MiB +......................................................................................... +llama_new_context_with_model: n_seq_max = 1 +llama_new_context_with_model: n_ctx = 4096 +llama_new_context_with_model: n_ctx_per_seq = 4096 +llama_new_context_with_model: n_batch = 2048 +llama_new_context_with_model: n_ubatch = 512 +llama_new_context_with_model: flash_attn = 0 +llama_new_context_with_model: freq_base = 500000.0 +llama_new_context_with_model: freq_scale = 1 +llama_new_context_with_model: n_ctx_per_seq (4096) < n_ctx_train (131072) -- the full capacity of the model will not be utilized +llama_kv_cache_init: CUDA0 KV buffer size = 684.19 MiB +llama_kv_cache_init: CPU KV buffer size = 228.06 MiB +llama_new_context_with_model: KV self size = 912.25 MiB, K (f16): 456.12 MiB, V (f16): 456.12 MiB +llama_new_context_with_model: CPU output buffer size = 0.49 MiB +llama_new_context_with_model: CUDA0 compute buffer size = 669.48 MiB +llama_new_context_with_model: CUDA_Host compute buffer size = 16.01 MiB +llama_new_context_with_model: graph nodes = 1030 +llama_new_context_with_model: graph splits = 93 (with bs=512), 3 (with bs=1) +token = 128006 +token = 882 +token = 128007 +token = 271 +token = 128256 +token = 3923 +token = 1587 +token = 279 +token = 2217 +token = 1501 +token = 30 +token = 128009 +token = 128006 +token = 78191 +token = 128007 +token = 271 +The image shows a close-up of the Eiffel Tower in Paris, France. The tower is made of metal and has a dark gray color. It is shaped like a square with four sides, and it has a flat top. The background of the image is a light gray color. + +* The +main: decoded 60 tokens in 16.38 s, speed: 3.66 t/s + + +llama_perf_context_print: load time = 6654.38 ms +llama_perf_context_print: prompt eval time = 1160.81 ms / 17 tokens ( 68.28 ms per token, 14.64 tokens per second) +llama_perf_context_print: eval time = 16096.27 ms / 59 runs ( 272.82 ms per token, 3.67 tokens per second) +llama_perf_context_print: total time = 23033.59 ms / 76 tokens +``` + +
diff --git a/examples/simple-vision-mllama/simple-vision.cpp b/examples/simple-vision-mllama/simple-vision.cpp new file mode 100644 index 0000000000000..0489f49f7f512 --- /dev/null +++ b/examples/simple-vision-mllama/simple-vision.cpp @@ -0,0 +1,189 @@ +#include "arg.h" +#include "common.h" +#include "sampling.h" +#include "log.h" +#include "llama.h" +#include "vision.h" +#include "llama-vision.h" + +#include +#include + +static void print_usage(int, char ** argv) { + LOG("\nexample usage:\n"); + LOG("\n %s -m model.gguf --image some-image.jpg -c 2048\n", argv[0]); + LOG("\n"); +} + +int main(int argc, char ** argv) { + common_params params; + + // Currently it is not possible to pass both tokens and image patch token + // embeddings in one single batch. There is an open issue to address this: + // https://github.com/ggerganov/llama.cpp/issues/10381 + // + // So this example works by first decoding the an intial prompt prefix, + // followed by a batch containing image patch embeddings, and then a + // final prompt with contains the <|image|> special token and the rest + // of the prompt. + std::string prefix_prompt = "<|start_header_id|>user<|end_header_id|>\n\n"; + params.prompt = "<|image|>What does the image show?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"; + params.n_predict = 60; + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_VISION, print_usage)) { + return 1; + } + + common_init(); + + const int n_predict = params.n_predict; + + llama_backend_init(); + llama_numa_init(params.numa); + + llama_model_params model_params = common_model_params_to_llama(params); + llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params); + + if (model == NULL) { + fprintf(stderr , "%s: error: unable to load model\n" , __func__); + return 1; + } + + llama_context_params ctx_params = common_context_params_to_llama(params); + llama_context * ctx = llama_new_context_with_model(model, ctx_params); + if (ctx == NULL) { + fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__); + return 1; + } + + llama_set_cross_attention(ctx, true); + + auto & sparams = params.sampling; + sparams.top_k = 40; + sparams.top_p = 0.9f; + sparams.seed = 4294967295; + sparams.temp = 0.0f; + auto smpl = common_sampler_init(model, sparams); + + // Tokenize the initial prompt + std::vector tokens_prefix; + tokens_prefix = common_tokenize(ctx, prefix_prompt, false, true) ; + for (auto id : tokens_prefix) { + printf("token = %d\n", id); + } + + // Tokenize the prompt + std::vector tokens_prompt; + tokens_prompt = common_tokenize(ctx, params.prompt, false, true) ; + for (auto id : tokens_prompt) { + printf("token = %d\n", id); + } + + // Load the image image and create a batch for . + llama_batch_img img_batch = llama_batch_img_init(1); + img_batch.imgs[0] = mllama_load_image_from_file(params.image[0].c_str()); + img_batch.imgs[0]->nx = 560; + img_batch.imgs[0]->ny = 560; + + if (llama_encode_vision(ctx, img_batch) != 0) { + LOG("%s: llama_encode_vision() failed\n", __func__); + return 1; + } + + llama_batch batch1 = llama_batch_init(tokens_prefix.size(), 0, 1); + int pos = 0; + for (auto id : tokens_prefix) { + common_batch_add(batch1, id, pos, { 0 }, false); + pos++; + } + + if (llama_decode(ctx, batch1) != 0) { + printf("%s: llama_decode() failed\n", __func__); + return 1; + } + + auto n_img_tokens = 15052800; + printf("n_img_tokens = %d\n", n_img_tokens); + float * patch_embeddings = _test_get_img_embd(ctx); + llama_batch embd_batch = llama_batch_init(1, n_img_tokens, 1); + embd_batch.embd = patch_embeddings; + embd_batch.n_tokens = 1; + embd_batch.pos[0] = pos++; + embd_batch.n_seq_id[0] = 1; + embd_batch.seq_id[0][0] = 0; + embd_batch.logits[0] = true; + + if (llama_decode(ctx, embd_batch) != 0) { + printf("%s: llama_decode() failed\n", __func__); + return 1; + } + + llama_batch batch2 = llama_batch_init(tokens_prompt.size(), 0, 1); + for (auto id : tokens_prompt) { + common_batch_add(batch2, id, pos, { 0 }, false); + pos++; + } + batch2.logits[batch2.n_tokens-1] = true; + + if (llama_decode(ctx, batch2) != 0) { + printf("%s: llama_decode() failed\n", __func__); + return 1; + } + + int n_decode = 0; + const auto t_main_start = ggml_time_us(); + for (int i = 0; i < n_predict; i++) { + // Sample the next token + { + const llama_token new_token_id = common_sampler_sample(smpl, ctx, -1); + + // Is it an end of generation? + if (llama_token_is_eog(model, new_token_id)) { + printf("\n"); + + break; + } + + printf("%s", common_token_to_piece(ctx, new_token_id, true).c_str()); + fflush(stdout); + + common_batch_clear(batch2); + + // Push this new token for next evaluation + common_batch_add(batch2, new_token_id, pos, { 0 }, true); + + n_decode += 1; + } + + pos += 1; + + // Evaluate the current batch with the transformer model + if (llama_decode(ctx, batch2)) { + LOG_ERR("%s : failed to eval, return code %d\n", __func__, 1); + return 1; + } + fflush(stdout); + } + + printf("\n"); + + const auto t_main_end = ggml_time_us(); + + printf("%s: decoded %d tokens in %.2f s, speed: %.2f t/s\n", + __func__, n_decode, (t_main_end - t_main_start) / 1000000.0f, n_decode / ((t_main_end - t_main_start) / 1000000.0f)); + + printf("\n"); + llama_perf_context_print(ctx); + + printf("\n"); + + llama_batch_free(batch2); + llama_batch_free(batch1); + common_sampler_free(smpl); + llama_free(ctx); + llama_free_model(model); + + llama_backend_free(); + + return 0; +} diff --git a/examples/simple-vision/CMakeLists.txt b/examples/simple-vision/CMakeLists.txt new file mode 100644 index 0000000000000..92796a78f911e --- /dev/null +++ b/examples/simple-vision/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-simple-vision) +add_executable(${TARGET} simple-vision.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) diff --git a/examples/simple-vision/README.md b/examples/simple-vision/README.md new file mode 100644 index 0000000000000..45f26706d74d4 --- /dev/null +++ b/examples/simple-vision/README.md @@ -0,0 +1,26 @@ +### Simple Vision Example +This example demonstrates how to use the vision-api. + +_The Vision API is currently in development_ + +### Building +This can be build with cmake using the following commands: +```console +$ cmake -S . -B build -DGGML_CUDA=On +$ cmake --build build --target llama-simple-vision -- -j 8 +``` +And with make using the following commands: +```console +$ make -j8 GGML_CUDA=1 LLAMA_DEBUG=1 llama-simple-vision +``` + +### Running the example: +The binaries for cmake and make are located in different directories, cmake +are located in the `build/bin` directory and the make binaries are located in +project root directory so use the binary for the build system you used above. + +The example requires that a model is passed in which should be a CLIP model, and +an image to be processed: +```console +$ ./llama-simple-vision -m models/llava-1.5.7b-hf.gguf -c 1024 -v -ngl 20 --image models/eiffel-tower-3349075_1280.jpg +``` diff --git a/examples/simple-vision/simple-vision.cpp b/examples/simple-vision/simple-vision.cpp new file mode 100644 index 0000000000000..9a18125310c29 --- /dev/null +++ b/examples/simple-vision/simple-vision.cpp @@ -0,0 +1,224 @@ +#include "arg.h" +#include "common.h" +#include "log.h" +#include "llama.h" +#include "vision.h" + +#include + +static void print_usage(int, char ** argv) { + LOG("\nexample usage:\n"); + LOG("\n %s -m model.gguf -p \"Hello my name is\" -n 32\n", argv[0]); + LOG("\n"); +} + +int main(int argc, char ** argv) { + common_params params; + + //params.prompt = "Hello my name is"; + params.prompt = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n" + "USER:\nwhat did you see?\nASSISTANT:"; + params.n_predict = 32; + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_VISION, print_usage)) { + return 1; + } + + common_init(); + + // total length of the sequence including the prompt + const int n_predict = params.n_predict; + + // init LLM + + llama_backend_init(); + llama_numa_init(params.numa); + + // initialize the model + + llama_model_params model_params = common_model_params_to_llama(params); + + llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params); + + if (model == NULL) { + fprintf(stderr , "%s: error: unable to load model\n" , __func__); + return 1; + } + + // initialize the context + + llama_context_params ctx_params = common_context_params_to_llama(params); + + llama_context * ctx = llama_new_context_with_model(model, ctx_params); + + if (ctx == NULL) { + fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__); + return 1; + } + + auto sparams = llama_sampler_chain_default_params(); + + sparams.no_perf = false; + + llama_sampler * smpl = llama_sampler_chain_init(sparams); + + llama_sampler_chain_add(smpl, llama_sampler_init_greedy()); + + // tokenize the prompt + + std::vector tokens_list; + tokens_list = ::llama_tokenize_with_img(ctx, params.prompt, true); + + const int n_ctx = llama_n_ctx(ctx); + const int n_kv_req = tokens_list.size() + (n_predict - tokens_list.size()); + + LOG("\n"); + LOG_INF("%s: n_predict = %d, n_ctx = %d, n_kv_req = %d\n", __func__, n_predict, n_ctx, n_kv_req); + + // make sure the KV cache is big enough to hold all the prompt and generated tokens + if (n_kv_req > n_ctx) { + LOG_ERR("%s: error: n_kv_req > n_ctx, the required KV cache size is not big enough\n", __func__); + LOG_ERR("%s: either reduce n_predict or increase n_ctx\n", __func__); + return 1; + } + + // print the prompt token-by-token + + LOG("\n"); + + for (auto id : tokens_list) { + if (id == TOKEN_IMG_PLACEMENT) { + LOG(""); + } else { + LOG("%s", common_token_to_piece(ctx, id).c_str()); + } + } + + LOG("\n\n"); + + // load image + llama_batch_img img_batch = llama_batch_img_init(1); + img_batch.imgs[0] = load_image_from_file(params.image[0].c_str()); + + // create a llama_batch with size 512 + // we use this object to submit token data for decoding + + llama_batch batch = llama_batch_init(512, 0, 1); + + // evaluate the initial prompt + int n_cur = 0; + int i_img = 0; + for (auto id : tokens_list) { + if (id == TOKEN_IMG_PLACEMENT) { + img_batch.pos[i_img] = n_cur; + n_cur += llama_img_n_tokens(ctx, img_batch.imgs[i_img]); + i_img++; + } else { + common_batch_add(batch, id, n_cur, { 0 }, false); + printf("pos %d tok %d --> %s\n", n_cur, id, common_token_to_piece(ctx, id).c_str()); + n_cur++; + } + } + + // llama_decode will output logits only for the last token of the prompt + batch.logits[batch.n_tokens - 1] = true; + + if (llama_encode_vision(ctx, img_batch) != 0) { + LOG("%s: llama_encode_vision() failed\n", __func__); + return 1; + } + + n_cur = 0; + { + auto t1 = ::common_tokenize(ctx, "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:", false); + auto t2 = ::common_tokenize(ctx, "\nwhat did you see?\nASSISTANT:", false); + t1.insert(t1.begin(), 1); + + n_cur = 0; + common_batch_clear(batch); + common_batch_add(batch, 1, 0, { 0 }, false); + llama_decode(ctx, batch); + + n_cur = t1.size(); + common_batch_clear(batch); + llama_batch batch0 = {int32_t(576), nullptr, _test_get_img_embd(ctx), 0, nullptr, nullptr, nullptr, nullptr }; + llama_decode(ctx, batch0); + + n_cur = 0; + common_batch_clear(batch); + for (auto t : t1) { common_batch_add(batch, t, n_cur, { 0 }, false); n_cur++; } + llama_decode(ctx, batch); + + n_cur = t1.size() + 576; + common_batch_clear(batch); + printf("pos %d\n", n_cur); + for (auto t : t2) { common_batch_add(batch, t, n_cur, { 0 }, false); n_cur++; } + batch.logits[batch.n_tokens - 1] = true; + } + + if (llama_decode(ctx, batch) != 0) { + LOG("%s: llama_decode() failed\n", __func__); + return 1; + } + + // main loop + + int n_decode = 0; + + const auto t_main_start = ggml_time_us(); + + for (int i = 0; i < n_predict; i++) { + // sample the next token + { + const llama_token new_token_id = llama_sampler_sample(smpl, ctx, -1); + + // is it an end of generation? + if (llama_token_is_eog(model, new_token_id)) { + LOG("\n"); + + break; + } + + LOG("%s", common_token_to_piece(ctx, new_token_id).c_str()); + fflush(stdout); + + // prepare the next batch + common_batch_clear(batch); + + // push this new token for next evaluation + common_batch_add(batch, new_token_id, n_cur, { 0 }, true); + + n_decode += 1; + } + + n_cur += 1; + + // evaluate the current batch with the transformer model + if (llama_decode(ctx, batch)) { + LOG_ERR("%s : failed to eval, return code %d\n", __func__, 1); + return 1; + } + } + + LOG("\n"); + + const auto t_main_end = ggml_time_us(); + + LOG_INF("%s: decoded %d tokens in %.2f s, speed: %.2f t/s\n", + __func__, n_decode, (t_main_end - t_main_start) / 1000000.0f, n_decode / ((t_main_end - t_main_start) / 1000000.0f)); + + LOG("\n"); + llama_perf_sampler_print(smpl); + llama_perf_context_print(ctx); + + LOG("\n"); + + llama_batch_free(batch); + llama_sampler_free(smpl); + llama_free(ctx); + llama_free_model(model); + + llama_backend_free(); + + return 0; +} diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp index bbe13e7310032..3288c0250a001 100644 --- a/examples/simple/simple.cpp +++ b/examples/simple/simple.cpp @@ -2,7 +2,6 @@ #include #include #include -#include "vision.h" #include static void print_usage(int, char ** argv) { @@ -21,10 +20,7 @@ int main(int argc, char ** argv) { // number of tokens to predict int n_predict = 32; - //params.prompt = "Hello my name is"; - params.prompt = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n" - "USER:\nwhat did you see?\nASSISTANT:"; - params.n_predict = 32; + // parse command line arguments { int i = 1; @@ -131,107 +127,25 @@ int main(int argc, char ** argv) { llama_sampler_chain_add(smpl, llama_sampler_init_greedy()); - // tokenize the prompt - - std::vector tokens_list; - tokens_list = ::llama_tokenize_with_img(ctx, params.prompt, true); - - const int n_ctx = llama_n_ctx(ctx); - const int n_kv_req = tokens_list.size() + (n_predict - tokens_list.size()); - - LOG("\n"); - LOG_INF("%s: n_predict = %d, n_ctx = %d, n_kv_req = %d\n", __func__, n_predict, n_ctx, n_kv_req); - - // make sure the KV cache is big enough to hold all the prompt and generated tokens - if (n_kv_req > n_ctx) { - LOG_ERR("%s: error: n_kv_req > n_ctx, the required KV cache size is not big enough\n", __func__); - LOG_ERR("%s: either reduce n_predict or increase n_ctx\n", __func__); - return 1; - } - // print the prompt token-by-token - LOG("\n"); - - for (auto id : tokens_list) { - if (id == TOKEN_IMG_PLACEMENT) { - LOG(""); - } else { - LOG("%s", llama_token_to_piece(ctx, id).c_str()); - } - } - - LOG("\n\n"); - - // load image - llama_batch_img img_batch = llama_batch_img_init(1); - img_batch.imgs[0] = load_image_from_file("../models/eiffel-tower-3349075_1280.jpg"); - - // create a llama_batch with size 512 - // we use this object to submit token data for decoding - - llama_batch batch = llama_batch_init(512, 0, 1); - - // evaluate the initial prompt - int n_cur = 0; - int i_img = 0; - for (auto id : tokens_list) { - if (id == TOKEN_IMG_PLACEMENT) { - img_batch.pos[i_img] = n_cur; - n_cur += llama_img_n_tokens(ctx, img_batch.imgs[i_img]); - i_img++; - } else { - llama_batch_add(batch, id, n_cur, { 0 }, false); - printf("pos %d tok %d --> %s\n", n_cur, id, llama_token_to_piece(ctx, id).c_str()); - n_cur++; + for (auto id : prompt_tokens) { + char buf[128]; + int n = llama_token_to_piece(model, id, buf, sizeof(buf), 0, true); + if (n < 0) { + fprintf(stderr, "%s: error: failed to convert token to piece\n", __func__); + return 1; } + std::string s(buf, n); + printf("%s", s.c_str()); } - // llama_decode will output logits only for the last token of the prompt - batch.logits[batch.n_tokens - 1] = true; - - if (llama_encode_vision(ctx, img_batch) != 0) { - LOG("%s: llama_encode_vision() failed\n", __func__); - return 1; - } - - n_cur = 0; - { - auto t1 = ::llama_tokenize(ctx, "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:", false); - auto t2 = ::llama_tokenize(ctx, "\nwhat did you see?\nASSISTANT:", false); - t1.insert(t1.begin(), 1); - - n_cur = 0; - llama_batch_clear(batch); - llama_batch_add(batch, 1, 0, { 0 }, false); - llama_decode(ctx, batch); - - n_cur = t1.size(); - llama_batch_clear(batch); - llama_batch batch0 = {int32_t(576), nullptr, _test_get_img_embd(ctx), nullptr, nullptr, nullptr, nullptr, n_cur, 1, 0, }; - llama_decode(ctx, batch0); - - n_cur = 0; - llama_batch_clear(batch); - for (auto t : t1) { llama_batch_add(batch, t, n_cur, { 0 }, false); n_cur++; } - llama_decode(ctx, batch); - - n_cur = t1.size() + 576; - llama_batch_clear(batch); - printf("pos %d\n", n_cur); - for (auto t : t2) { llama_batch_add(batch, t, n_cur, { 0 }, false); n_cur++; } - batch.logits[batch.n_tokens - 1] = true; - } + // prepare a batch for the prompt - if (llama_decode(ctx, batch) != 0) { - LOG("%s: llama_decode() failed\n", __func__); - return 1; - } + llama_batch batch = llama_batch_get_one(prompt_tokens.data(), prompt_tokens.size()); // main loop - int n_decode = 0; - const auto t_main_start = ggml_time_us(); int n_decode = 0; llama_token new_token_id; @@ -245,15 +159,12 @@ int main(int argc, char ** argv) { n_pos += batch.n_tokens; - for (int i = 0; i < n_predict; i++) { // sample the next token { new_token_id = llama_sampler_sample(smpl, ctx, -1); // is it an end of generation? if (llama_token_is_eog(model, new_token_id)) { - LOG("\n"); - break; } diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index c714fc8c837bb..1bc50fcae5003 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -499,6 +499,7 @@ extern "C" { GGML_OP_UPSCALE, // nearest interpolate GGML_OP_PAD, GGML_OP_PAD_REFLECT_1D, + GGML_OP_UNPAD, GGML_OP_ARANGE, GGML_OP_TIMESTEP_EMBEDDING, GGML_OP_ARGSORT, @@ -1735,6 +1736,15 @@ extern "C" { int p0, int p1); + // unpad each dimension: [x, ..., x, y, ..., y] -> [x, ..., x] + GGML_API struct ggml_tensor * ggml_unpad( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151 // timesteps: [N,] // return: [N, dim] diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c index 8dc8226ac4932..008bc54fa7270 100644 --- a/ggml/src/ggml-alloc.c +++ b/ggml/src/ggml-alloc.c @@ -820,14 +820,14 @@ static bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_t static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) { if (galloc->n_nodes != graph->n_nodes) { #ifndef NDEBUG - GGML_LOG_DEBUG("%s: graph has different number of nodes\n", __func__); + //GGML_LOG_DEBUG("%s: graph has different number of nodes\n", __func__); #endif return true; } if (galloc->n_leafs != graph->n_leafs) { #ifndef NDEBUG - GGML_LOG_DEBUG("%s: graph has different number of leafs\n", __func__); + //GGML_LOG_DEBUG("%s: graph has different number of leafs\n", __func__); #endif return true; } @@ -838,7 +838,7 @@ static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph if (!ggml_gallocr_node_needs_realloc(galloc, node, &node_alloc->dst)) { #ifndef NDEBUG - GGML_LOG_DEBUG("%s: node %s is not valid\n", __func__, node->name); + //GGML_LOG_DEBUG("%s: node %s is not valid\n", __func__, node->name); #endif return true; } @@ -850,7 +850,7 @@ static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph } if (!ggml_gallocr_node_needs_realloc(galloc, src, &node_alloc->src[j])) { #ifndef NDEBUG - GGML_LOG_DEBUG("%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name); + //GGML_LOG_DEBUG("%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name); #endif return true; } @@ -871,7 +871,7 @@ bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) } } else { #ifndef NDEBUG - GGML_LOG_DEBUG("%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__); + //GGML_LOG_DEBUG("%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__); #endif return false; } diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 67e67a08968a1..bebff207c6899 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -10588,6 +10588,59 @@ static void ggml_compute_forward_pad_reflect_1d( } } +static void ggml_compute_forward_unpad_f32( + const struct ggml_compute_params *params, + struct ggml_tensor *dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(src0->nb[0] == sizeof(float)); + GGML_ASSERT( dst->nb[0] == sizeof(float)); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_UNARY_OP_LOCALS + + float * dst_ptr = (float *) dst->data; + + // TODO: optimize + + for (int64_t i2 = 0; i2 < ne2; ++i2) { + for (int64_t i1 = ith; i1 < ne1; i1 += nth) { + for (int64_t i0 = 0; i0 < ne0; ++i0) { + for (int64_t i3 = 0; i3 < ne3; ++i3) { + const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0; + + const float * src_ptr = (const float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + + if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { + dst_ptr[dst_idx] = *src_ptr; + } + } + } + } + } +} + +static void ggml_compute_forward_unpad( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_unpad_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_arange static void ggml_compute_forward_arange_f32( @@ -12690,6 +12743,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_pad_reflect_1d(params, tensor); } break; + case GGML_OP_UNPAD: + { + ggml_compute_forward_unpad(params, tensor); + } break; case GGML_OP_ARANGE: { ggml_compute_forward_arange(params, tensor); @@ -13033,6 +13090,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_UPSCALE: case GGML_OP_PAD: case GGML_OP_PAD_REFLECT_1D: + case GGML_OP_UNPAD: case GGML_OP_ARANGE: case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_ARGSORT: diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 2bbe5f48257b2..8348e7f302284 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -954,6 +954,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "UPSCALE", "PAD", "PAD_REFLECT_1D", + "UNPAD", "ARANGE", "TIMESTEP_EMBEDDING", "ARGSORT", @@ -987,7 +988,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "OPT_STEP_ADAMW", }; -static_assert(GGML_OP_COUNT == 82, "GGML_OP_COUNT != 82"); +static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1050,6 +1051,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "upscale(x)", "pad(x)", "pad_reflect_1d(x)", + "unpad(x)", "arange(start, stop, step)", "timestep_embedding(timesteps, dim, max_period)", "argsort(x)", @@ -1083,7 +1085,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "adamw(x)", }; -static_assert(GGML_OP_COUNT == 82, "GGML_OP_COUNT != 82"); +static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -4214,6 +4216,24 @@ struct ggml_tensor * ggml_pad_reflect_1d( return result; } +// ggml_unpad + +struct ggml_tensor * ggml_unpad( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, int p1, int p2, int p3) { + struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type, + a->ne[0] - p0, + a->ne[1] - p1, + a->ne[2] - p2, + a->ne[3] - p3); + + result->op = GGML_OP_UNPAD; + result->src[0] = a; + + return result; +} + // ggml_arange struct ggml_tensor * ggml_arange( diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2ec978fe22d96..c7c9d2e8e1e8f 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -113,6 +113,9 @@ class LLM: TIME_DECAY_EXTRA_DIM = "{arch}.time_decay_extra_dim" RESIDUAL_SCALE = "{arch}.residual_scale" EMBEDDING_SCALE = "{arch}.embedding_scale" + # TODO(danbev) : Not really sure where I should define these yet. + CROSS_ATTENTION_LAYERS = "{arch}.cross_attention_layers" + IMAGE_TOKEN_INDEX = "{arch}.image_token_index" class Attention: HEAD_COUNT = "{arch}.attention.head_count" @@ -183,6 +186,10 @@ class Tokenizer: PAD_ID = "tokenizer.ggml.padding_token_id" CLS_ID = "tokenizer.ggml.cls_token_id" MASK_ID = "tokenizer.ggml.mask_token_id" + IMAGE_ID = "tokenizer.ggml.image_token_id" + START_HEADER_ID = "tokenizer.ggml.start_header_token_id" + END_HEADER_ID = "tokenizer.ggml.end_header_token_id" + PYTHON_TAG_ID = "tokenizer.ggml.python_tag_token_id" ADD_BOS = "tokenizer.ggml.add_bos_token" ADD_EOS = "tokenizer.ggml.add_eos_token" ADD_PREFIX = "tokenizer.ggml.add_space_prefix" @@ -220,8 +227,15 @@ class Vision: PATCH_SIZE = "vision.patch_size" IMAGE_MEAN = "vision.image_mean" IMAGE_STD = "vision.image_std" + EMBEDDING_LENGTH = "vision.embedding_length" + BLOCK_COUNT = "vision.block_count" + LAYERNORM_RMS_EPS = "vision.attention.layer_norm_rms_epsilon" + FEED_FORWARD_LENGTH = "vision.feed_forward_length" + HEAD_COUNT = "vision.attention.head_count" + ARCHITECTURE = "vision.architecture" class Clip: + # projection-based architectures (LLaVA-style) ARCHITECTURE = "vision.clip.architecture" CONTEXT_LENGTH = "vision.clip.context_length" EMBEDDING_LENGTH = "vision.clip.embedding_length" @@ -238,6 +252,17 @@ class Clip: HEAD_COUNT = "vision.clip.attention.head_count" LAYERNORM_EPS = "vision.clip.attention.layer_norm_epsilon" + class Cross: + # cross-attention based architectures (MLlama, Flamingo) + GLOBAL_BLOCK_COUNT = "vision.cross.{arch}.global_block_count" + MAX_NUM_TILES = "vision.cross.{arch}.max_num_tiles" + CHANNELS_COUNT = "vision.cross.{arch}.channels_count" + INTERMEDIATE_LAYERS_INDICES = "vision.cross.{arch}.intermediate_layers_indices" + OUTPUT_DIM = "vision.cross.{arch}.output_dim" + SUPPORTED_ASPECT_RATIOS = "vision.cross.{arch}.supported_aspect_ratios" + MODEL_TYPE = "vision.cross.{arch}.model_type" + ACTIVATION_FUNCTION = "vision.cross.{arch}.activation_function" + # # recommended mapping of model tensor names for storage in gguf # @@ -304,6 +329,7 @@ class MODEL_ARCH(IntEnum): WAVTOKENIZER_DEC = auto() # vision models LLAVA_VISION = auto() + MLLAMA = auto() class MODEL_TENSOR(IntEnum): @@ -445,6 +471,59 @@ class MODEL_TENSOR(IntEnum): V_PRE_NORM = auto() V_POST_NORM = auto() + # Multi-Modal tensor. TODO(danbev): Consolidate these with vision tensors + MM_V_CLASS_EMBD = auto() + MM_V_POS_EMBD = auto() + MM_V_POS_GATE = auto() + MM_V_POS_EMBD_TILE_EMBD = auto() + MM_V_POS_EMBD_TILE_GATE = auto() + + MM_V_GLOBAL_ATTN_GATE = auto() + MM_V_GLOBAL_INPUT_NORM = auto() + MM_V_GLOBAL_FFN_UP = auto() + MM_V_GLOBAL_FFN_DOWN = auto() + MM_V_GLOBAL_FFN_GATE = auto() + MM_V_GLOBAL_POST_ATTN_NORM = auto() + MM_V_GLOBAL_ATTN_K = auto() + MM_V_GLOBAL_ATTN_OUT = auto() + MM_V_GLOBAL_ATTN_Q = auto() + MM_V_GLOBAL_ATTN_V = auto() + + + MM_V_POST_NORM = auto() + MM_V_PRE_NORM = auto() + MM_V_PATCH_EMBD = auto() + MM_V_POST_TILE_POS_EMBD = auto() + MM_V_POST_TILE_POS_GATE = auto() + MM_V_PRE_TILE_POS_EMBD = auto() + MM_V_PRE_TILE_POS_GATE = auto() + MM_V_TILE_POS_EMBD = auto() + MM_V_TILE_POS_GATE = auto() + MM_V_INPUT_NORM = auto() + MM_V_FFN_UP = auto() + MM_V_FFN_DOWN = auto() + MM_V_FFN_GATE = auto() + MM_V_ATTN_K = auto() + MM_V_ATTN_OUT = auto() + MM_V_ATTN_OUT_NORM = auto() + MM_V_ATTN_Q = auto() + MM_V_ATTN_V = auto() + MM_V_ATTN_GATE = auto() + MM_ATTN_NORM = auto() + + MM_CROSS_ATTN_K_NORM = auto() + MM_CROSS_ATTN_K = auto() + MM_CROSS_ATTN_Q_NORM = auto() + MM_CROSS_ATTN_Q = auto() + MM_CROSS_ATTN_OUT = auto() + MM_CROSS_ATTN_V = auto() + MM_CROSS_ATTN_GATE = auto() + MM_CROSS_ATTN_FFN_GATE = auto() + MM_CROSS_ATTN_NORM = auto() + #MM_LM_HEAD = auto() + #MM_LM_NORM = auto() + MM_PROJECTOR = auto() + #MM_ATTN_V = auto() MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.LLAMA: "llama", @@ -502,6 +581,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec", # vision MODEL_ARCH.LLAVA_VISION: "llava", + MODEL_ARCH.MLLAMA: "mllama", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -634,7 +714,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_ENC_EMBD_POS: "v.enc.embd.pos", MODEL_TENSOR.V_ENC_ATTN_Q: "v.enc.blk.{bid}.attn_q", MODEL_TENSOR.V_ENC_ATTN_K: "v.enc.blk.{bid}.attn_k", - MODEL_TENSOR.V_ENC_ATTN_V: "v.enc.blk.{bid}.attn_v", + MODEL_TENSOR.V_ENC_ATTN_V: "v.enc.blk.{bid}.attn_v_proj", MODEL_TENSOR.V_ENC_INPUT_NORM: "v.enc.blk.{bid}.input_norm", MODEL_TENSOR.V_ENC_OUTPUT: "v.enc.blk.{bid}.output", MODEL_TENSOR.V_ENC_OUTPUT_NORM: "v.enc.blk.{bid}.output_norm", @@ -642,6 +722,73 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_ENC_FFN_DOWN: "v.enc.blk.{bid}.ffn_down", MODEL_TENSOR.V_PRE_NORM: "v.pre_norm", MODEL_TENSOR.V_POST_NORM: "v.post_norm", + + # Multi-Modal tensor. + MODEL_TENSOR.MM_V_CLASS_EMBD: "v.enc.embd.cls", + MODEL_TENSOR.MM_V_PATCH_EMBD: "v.enc.embd.patch", + + MODEL_TENSOR.MM_V_POS_EMBD: "v.enc.embd.pos", + MODEL_TENSOR.MM_V_POS_GATE: "v.enc.embd.pos_gate", + + MODEL_TENSOR.MM_V_POST_NORM: "v.enc.post_ln", + MODEL_TENSOR.MM_V_PRE_NORM: "v.enc.pre_ln", + + MODEL_TENSOR.MM_V_TILE_POS_EMBD: "v.enc.tile_pos_embd", + MODEL_TENSOR.MM_V_TILE_POS_GATE: "v.enc.tile_pos_gate", + + MODEL_TENSOR.MM_V_PRE_TILE_POS_EMBD: "v.enc.pre_tile_pos_embd", + MODEL_TENSOR.MM_V_PRE_TILE_POS_GATE: "v.enc.pre_tile_pos_gate", + + MODEL_TENSOR.MM_V_POST_TILE_POS_EMBD: "v.enc.post_tile_pos_embd", + MODEL_TENSOR.MM_V_POST_TILE_POS_GATE: "v.enc.post_tile_pos_gate", + + MODEL_TENSOR.MM_PROJECTOR: "v.enc.mmproj", + + ## Global Layers + MODEL_TENSOR.MM_V_GLOBAL_INPUT_NORM: "v.enc.global.blk.{bid}.attn_norm", + + MODEL_TENSOR.MM_V_GLOBAL_ATTN_K: "v.enc.global.blk.{bid}.attn_k", + MODEL_TENSOR.MM_V_GLOBAL_ATTN_Q: "v.enc.global.blk.{bid}.attn_q", + MODEL_TENSOR.MM_V_GLOBAL_ATTN_V: "v.enc.global.blk.{bid}.attn_v", + MODEL_TENSOR.MM_V_GLOBAL_ATTN_GATE: "v.enc.global.blk.{bid}.attn_gate", + + MODEL_TENSOR.MM_V_GLOBAL_FFN_UP: "v.enc.global.blk.{bid}.ffn_up", + MODEL_TENSOR.MM_V_GLOBAL_FFN_GATE: "v.enc.global.blk.{bid}.ffn_gate", + MODEL_TENSOR.MM_V_GLOBAL_FFN_DOWN: "v.enc.global.blk.{bid}.ffn_down", + + MODEL_TENSOR.MM_V_GLOBAL_ATTN_OUT: "v.enc.global.blk.{bid}.attn_out", + MODEL_TENSOR.MM_V_GLOBAL_POST_ATTN_NORM: "v.enc.global.blk.{bid}.post_attn_norm", + + ## Local (Non-Global) Layers + MODEL_TENSOR.MM_V_INPUT_NORM: "v.enc.blk.{bid}.input_norm", + + MODEL_TENSOR.MM_V_ATTN_K: "v.enc.blk.{bid}.attn_k", + MODEL_TENSOR.MM_V_ATTN_Q: "v.enc.blk.{bid}.attn_q", + MODEL_TENSOR.MM_V_ATTN_V: "v.enc.blk.{bid}.attn_v", + MODEL_TENSOR.MM_V_ATTN_GATE: "v.enc.blk.{bid}.attn_gate", + + MODEL_TENSOR.MM_V_FFN_UP: "v.enc.blk.{bid}.ffn_up", + MODEL_TENSOR.MM_V_FFN_GATE: "v.enc.blk.{bid}.ffn_gate", + MODEL_TENSOR.MM_V_FFN_DOWN: "v.enc.blk.{bid}.ffn_down", + MODEL_TENSOR.MM_V_ATTN_OUT: "v.enc.blk.{bid}.attn_out", + MODEL_TENSOR.MM_V_ATTN_OUT_NORM: "v.enc.blk.{bid}.attn_out_norm", + + # Cross-Attention layers + MODEL_TENSOR.MM_CROSS_ATTN_K_NORM: "blk.{bid}.cross_attn_k_norm", + MODEL_TENSOR.MM_CROSS_ATTN_K: "blk.{bid}.cross_attn_k_proj", + MODEL_TENSOR.MM_CROSS_ATTN_Q_NORM: "blk.{bid}.cross_attn_q_norm", + MODEL_TENSOR.MM_CROSS_ATTN_Q: "blk.{bid}.cross_attn_q_proj", + MODEL_TENSOR.MM_CROSS_ATTN_OUT: "blk.{bid}.cross_attn_o_proj", + MODEL_TENSOR.MM_CROSS_ATTN_V: "blk.{bid}.cross_attn_v_proj", + MODEL_TENSOR.MM_CROSS_ATTN_GATE: "blk.{bid}.cross_attn_attn_gate", + MODEL_TENSOR.MM_CROSS_ATTN_FFN_GATE: "blk.{bid}.cross_attn_mlp_gate", + MODEL_TENSOR.MM_CROSS_ATTN_NORM: "blk.{bid}.cross_attn_norm", + + #MODEL_TENSOR.MM_ATTN_V: "blk.{bid}.attn_v_proj", + + #MODEL_TENSOR.MM_LM_HEAD: "v.enc.lm_head", + #MODEL_TENSOR.MM_LM_NORM: "v.enc.lm_norm", + } MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { @@ -1518,6 +1665,90 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_PRE_NORM, MODEL_TENSOR.V_POST_NORM, ], + MODEL_ARCH.MLLAMA: [ + # Language Model Tensors + MODEL_TENSOR.TOKEN_EMBD, + + MODEL_TENSOR.ATTN_NORM, + ## Cross Attention layer Tensors + MODEL_TENSOR.MM_CROSS_ATTN_Q, + MODEL_TENSOR.MM_CROSS_ATTN_Q_NORM, + MODEL_TENSOR.MM_CROSS_ATTN_K, + MODEL_TENSOR.MM_CROSS_ATTN_K_NORM, + MODEL_TENSOR.MM_CROSS_ATTN_V, + MODEL_TENSOR.MM_CROSS_ATTN_OUT, + MODEL_TENSOR.MM_CROSS_ATTN_GATE, + + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.MM_CROSS_ATTN_FFN_GATE, + + ## Attention layer Tensors + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + + MODEL_TENSOR.OUTPUT_NORM, + + MODEL_TENSOR.OUTPUT, + + # Vision Model Tensors + MODEL_TENSOR.MM_V_CLASS_EMBD, + MODEL_TENSOR.MM_V_PATCH_EMBD, + + MODEL_TENSOR.MM_V_POS_EMBD, + MODEL_TENSOR.MM_V_POS_GATE, + + MODEL_TENSOR.MM_V_PRE_NORM, + MODEL_TENSOR.MM_V_POST_NORM, + + MODEL_TENSOR.MM_V_TILE_POS_GATE, + MODEL_TENSOR.MM_V_TILE_POS_EMBD, + + MODEL_TENSOR.MM_V_PRE_TILE_POS_EMBD, + MODEL_TENSOR.MM_V_PRE_TILE_POS_GATE, + + MODEL_TENSOR.MM_V_POST_TILE_POS_EMBD, + MODEL_TENSOR.MM_V_POST_TILE_POS_GATE, + + MODEL_TENSOR.MM_PROJECTOR, + + ## Global Layers + MODEL_TENSOR.MM_V_GLOBAL_INPUT_NORM, + MODEL_TENSOR.MM_V_GLOBAL_ATTN_K, + MODEL_TENSOR.MM_V_GLOBAL_ATTN_Q, + MODEL_TENSOR.MM_V_GLOBAL_ATTN_V, + MODEL_TENSOR.MM_V_GLOBAL_FFN_UP, + MODEL_TENSOR.MM_V_GLOBAL_ATTN_GATE, + MODEL_TENSOR.MM_V_GLOBAL_FFN_DOWN, + MODEL_TENSOR.MM_V_GLOBAL_ATTN_OUT, + MODEL_TENSOR.MM_V_GLOBAL_FFN_GATE, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.MM_V_GLOBAL_FFN_UP, + MODEL_TENSOR.MM_V_GLOBAL_POST_ATTN_NORM, + MODEL_TENSOR.MM_V_INPUT_NORM, + MODEL_TENSOR.MM_V_FFN_UP, + MODEL_TENSOR.MM_V_FFN_DOWN, + MODEL_TENSOR.MM_V_ATTN_K, + MODEL_TENSOR.MM_V_ATTN_OUT, + MODEL_TENSOR.MM_V_ATTN_Q, + MODEL_TENSOR.MM_V_ATTN_V, + MODEL_TENSOR.MM_V_ATTN_GATE, + MODEL_TENSOR.MM_V_ATTN_OUT_NORM, + ], # TODO } diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 6118c6380c65d..08d88f5bf0564 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -855,6 +855,18 @@ def add_cls_token_id(self, id: int) -> None: def add_mask_token_id(self, id: int) -> None: self.add_uint32(Keys.Tokenizer.MASK_ID, id) + def add_image_token_id(self, id: int) -> None: + self.add_uint32(Keys.Tokenizer.IMAGE_ID, id) + + def add_start_header_token_id(self, id: int) -> None: + self.add_uint32(Keys.Tokenizer.START_HEADER_ID, id) + + def add_end_header_token_id(self, id: int) -> None: + self.add_uint32(Keys.Tokenizer.END_HEADER_ID, id) + + def add_python_tag_token_id(self, id: int) -> None: + self.add_uint32(Keys.Tokenizer.PYTHON_TAG_ID, id) + def add_add_bos_token(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.ADD_BOS, value) @@ -903,6 +915,9 @@ def add_vision_clip_max_position_embeddings(self, value: int) -> None: def add_vision_clip_projector_type(self, value: CLIPProjectorType) -> None: self.add_string(Keys.Vision.Clip.PROJECTOR_TYPE, value.value) + def add_vision_clip_projection_dimension(self, dim: int) -> None: + self.add_uint32(Keys.Vision.Clip.PROJECTION_DIM, dim) + def add_vision_clip_max_slices(self, value: int) -> None: self.add_uint32(Keys.Vision.Clip.MAX_SLICES, value) @@ -921,6 +936,56 @@ def add_vision_clip_image_mean(self, value: Sequence[float]) -> None: def add_vision_clip_image_std(self, value: Sequence[float]) -> None: self.add_array(Keys.Vision.IMAGE_STD, value) + # Multi-Modal Vision methods. TODO: consolidate with Vision methods + def add_image_token_index(self, idx: int) -> None: + self.add_uint32(Keys.LLM.IMAGE_TOKEN_INDEX.format(arch=self.arch), idx) + + def add_cross_attention_layers(self, layers: Sequence[int]) -> None: + self.add_array(Keys.LLM.CROSS_ATTENTION_LAYERS.format(arch=self.arch), layers) + + def add_vision_block_count(self, heads: int) -> None: + self.add_uint32(Keys.Vision.BLOCK_COUNT, heads) + + def add_vision_layer_norm_rms_eps(self, value: float) -> None: + self.add_float32(Keys.Vision.LAYERNORM_RMS_EPS, value) + + def add_vision_embedding_length(self, size: int) -> None: + self.add_uint32(Keys.Vision.EMBEDDING_LENGTH, size) + + def add_vision_feed_forward_length(self, length: int) -> None: + self.add_uint32(Keys.Vision.FEED_FORWARD_LENGTH, length) + + def add_vision_global_block_count(self, size: int) -> None: + self.add_uint32(Keys.Vision.Cross.GLOBAL_BLOCK_COUNT.format(arch=self.arch), size) + + def add_vision_max_num_tiles(self, value: int) -> None: + self.add_uint32(Keys.Vision.Cross.MAX_NUM_TILES.format(arch=self.arch), value) + + def add_vision_channels_count(self, channels: int) -> None: + self.add_uint32(Keys.Vision.Cross.CHANNELS_COUNT.format(arch=self.arch), channels) + + def add_vision_intermediate_layer_indices(self, indices: Sequence[int]) -> None: + self.add_array(Keys.Vision.Cross.INTERMEDIATE_LAYERS_INDICES.format(arch=self.arch), indices) + + def add_vision_attention_head_count(self, count: int) -> None: + self.add_uint32(Keys.Vision.HEAD_COUNT, count) + + def add_vision_output_dimension(self, dim: int) -> None: + self.add_uint32(Keys.Vision.Cross.OUTPUT_DIM.format(arch=self.arch), dim) + + def add_vision_supported_aspect_ratios(self, ratios: Sequence[Sequence[int]]) -> None: + flattened_ratios = [ratio for sequence in ratios for ratio in sequence] + self.add_array(Keys.Vision.Cross.SUPPORTED_ASPECT_RATIOS.format(arch=self.arch), flattened_ratios) + + def add_vision_model_type(self, model_type: str) -> None: + self.add_string(Keys.Vision.Cross.MODEL_TYPE.format(arch=self.arch), model_type) + + def add_vision_activation_function(self, act: str) -> None: + self.add_string(Keys.Vision.Cross.ACTIVATION_FUNCTION.format(arch=self.arch), act) + + def add_vision_architecture(self, value: str) -> None: + self.add_string(Keys.Vision.ARCHITECTURE, value) + def add_chat_template(self, value: str | Sequence[Mapping[str, str]]) -> None: if not isinstance(value, str): template_default = None diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 7e4a3ec373f8a..127f7596c2939 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -28,6 +28,7 @@ class TensorNameMap: "transformer.token_embeddings", # openelm "shared", # t5 "rwkv.embeddings", # rwkv + "language_model.model.embed_tokens", # mllama ), # Token type embeddings @@ -62,6 +63,7 @@ class TensorNameMap: "output_layer", # chatglm "head", # rwkv "head.out", # wavtokenizer + "language_model.lm_head", # mllama ), # Output norm @@ -83,6 +85,7 @@ class TensorNameMap: "model.norm", # nemotron "rwkv.ln_out", # rwkv "backbone.final_layer_norm", # wavtokenizer + "language_model.model.norm" # mllama ), # Rope frequencies @@ -97,6 +100,59 @@ class TensorNameMap: MODEL_TENSOR.CONV1D: ( "backbone.embed", # roberta ), + + # Multi-modal tensor names + MODEL_TENSOR.MM_V_CLASS_EMBD: ( + "vision_model.class_embedding", # mllama + ), + + MODEL_TENSOR.MM_V_POS_EMBD: ( + "vision_model.gated_positional_embedding.embedding", # mllama + ), + + MODEL_TENSOR.MM_V_POS_GATE: ( + "vision_model.gated_positional_embedding.gate", # mllama + ), + + MODEL_TENSOR.MM_V_TILE_POS_EMBD: ( + "vision_model.gated_positional_embedding.tile_embedding", # mllama + ), + + MODEL_TENSOR.MM_V_TILE_POS_GATE: ( + "vision_model.gated_positional_embedding.tile_embedding.gate", # mllama + ), + + MODEL_TENSOR.MM_V_POST_NORM: ( + "vision_model.layernorm_post", # mllama + ), + + MODEL_TENSOR.MM_V_PRE_NORM: ( + "vision_model.layernorm_pre", # mllama + ), + + MODEL_TENSOR.MM_V_PATCH_EMBD: ( + "vision_model.patch_embedding", # mllama + ), + + MODEL_TENSOR.MM_V_POST_TILE_POS_EMBD: ( + "vision_model.post_tile_positional_embedding.embedding", # mllama + ), + + MODEL_TENSOR.MM_V_POST_TILE_POS_GATE: ( + "vision_model.post_tile_positional_embedding.gate", # mllama + ), + + MODEL_TENSOR.MM_V_PRE_TILE_POS_EMBD: ( + "vision_model.pre_tile_positional_embedding.embedding", # mllama + ), + + MODEL_TENSOR.MM_V_PRE_TILE_POS_GATE: ( + "vision_model.pre_tile_positional_embedding.gate", # mllama + ), + + MODEL_TENSOR.MM_PROJECTOR: ( + "multi_modal_projector", # mllama + ), } block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { @@ -123,6 +179,7 @@ class TensorNameMap: "encoder.layers.{bid}.input_layernorm", # chatglm "transformer.layers.{bid}.attn_norm", # openelm "rwkv.blocks.{bid}.ln1", # rwkv + "language_model.model.layers.{bid}.input_layernorm", # mlama ), # Attention norm 2 @@ -161,6 +218,7 @@ class TensorNameMap: "model.layers.{bid}.attention.wq", # internlm2 "transformer.decoder_layer.{bid}.multi_head_attention.query",# Grok "transformer.h.{bid}.attn.attention.q_proj", # exaone + "language_model.model.layers.{bid}.self_attn.q_proj", # mllama ), # Attention key @@ -175,6 +233,7 @@ class TensorNameMap: "model.layers.{bid}.attention.wk", # internlm2 "transformer.decoder_layer.{bid}.multi_head_attention.key",# Grok "transformer.h.{bid}.attn.attention.k_proj", # exaone + "language_model.model.layers.{bid}.self_attn.k_proj", # mllama ), # Attention value @@ -188,6 +247,7 @@ class TensorNameMap: "model.layers.{bid}.attention.wv", # internlm2 "transformer.decoder_layer.{bid}.multi_head_attention.value",# Grok "transformer.h.{bid}.attn.attention.v_proj", # exaone + "language_model.model.layers.{bid}.self_attn.v_proj", # mllama ), # Attention output @@ -213,6 +273,7 @@ class TensorNameMap: "encoder.layers.{bid}.self_attention.dense", # chatglm "transformer.layers.{bid}.attn.out_proj", # openelm "transformer.h.{bid}.attn.attention.out_proj", # exaone + "language_model.model.layers.{bid}.self_attn.o_proj", # mllama ), # Attention output norm @@ -250,6 +311,7 @@ class TensorNameMap: "transformer.decoder_layer.{bid}.rms_norm_2", # Grok "encoder.layers.{bid}.post_attention_layernorm", # chatglm "transformer.layers.{bid}.ffn_norm", # openelm + "language_model.model.layers.{bid}.post_attention_layernorm", # mllama ), # Post feed-forward norm @@ -302,6 +364,7 @@ class TensorNameMap: "model.layers.{bid}.residual_mlp.w3", # arctic "encoder.layers.{bid}.mlp.dense_h_to_4h", # chatglm "transformer.h.{bid}.mlp.c_fc_1", # exaone + "language_model.model.layers.{bid}.mlp.up_proj", # mllama ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -334,6 +397,7 @@ class TensorNameMap: "transformer.h.{bid}.mlp.linear_1", # refact "model.layers.{bid}.residual_mlp.w1", # arctic "transformer.h.{bid}.mlp.c_fc_0", # exaone + "language_model.model.layers.{bid}.mlp.gate_proj", # mllama ), MODEL_TENSOR.FFN_GATE_EXP: ( @@ -374,6 +438,7 @@ class TensorNameMap: "encoder.layer.{bid}.mlp.down_layer", # jina-bert-v2 "encoder.layers.{bid}.mlp.dense_4h_to_h", # chatglm "model.layers.h.{bid}.mlp.c_proj", # exaone + "language_model.model.layers.{bid}.mlp.down_proj", # mllama ), MODEL_TENSOR.FFN_DOWN_EXP: ( @@ -701,9 +766,6 @@ class TensorNameMap: MODEL_TENSOR.CLS_OUT: ( "classifier.out_proj", # roberta - - MODEL_TENSOR.V_MMPROJ: ( - "multi_modal_projector.linear_{bid}", ), MODEL_TENSOR.V_MMPROJ: ( @@ -822,6 +884,111 @@ class TensorNameMap: MODEL_TENSOR.POSNET_ATTN_OUT: ( "backbone.posnet.{bid}.proj_out", # wavtokenizer ), + + MODEL_TENSOR.MM_CROSS_ATTN_K_NORM: ( + "language_model.model.layers.{bid}.cross_attn.k_norm", # mllama + ), + + MODEL_TENSOR.MM_CROSS_ATTN_K: ( + "language_model.model.layers.{bid}.cross_attn.k_proj", # mllama + ), + + MODEL_TENSOR.MM_CROSS_ATTN_OUT: ( + "language_model.model.layers.{bid}.cross_attn.o_proj", # mllama + ), + + MODEL_TENSOR.MM_CROSS_ATTN_Q_NORM: ( + "language_model.model.layers.{bid}.cross_attn.q_norm", # mllama + ), + + MODEL_TENSOR.MM_CROSS_ATTN_Q: ( + "language_model.model.layers.{bid}.cross_attn.q_proj", # mllama + ), + + MODEL_TENSOR.MM_CROSS_ATTN_V: ( + "language_model.model.layers.{bid}.cross_attn.v_proj", # mllama + ), + + MODEL_TENSOR.MM_CROSS_ATTN_GATE: ( + "language_model.model.layers.{bid}.cross_attn_attn_gate", # mllama + ), + + MODEL_TENSOR.MM_CROSS_ATTN_FFN_GATE: ( + "language_model.model.layers.{bid}.cross_attn_mlp_gate", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_INPUT_NORM: ( + "vision_model.global_transformer.layers.{bid}.input_layernorm", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_FFN_UP: ( + "vision_model.global_transformer.layers.{bid}.mlp.fc1", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_FFN_DOWN: ( + "vision_model.global_transformer.layers.{bid}.mlp.fc2", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_POST_ATTN_NORM: ( + "vision_model.global_transformer.layers.{bid}.post_attention_layernorm", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_ATTN_K: ( + "vision_model.global_transformer.layers.{bid}.self_attn.k_proj", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_ATTN_OUT: ( + "vision_model.global_transformer.layers.{bid}.self_attn.o_proj", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_ATTN_Q: ( + "vision_model.global_transformer.layers.{bid}.self_attn.q_proj", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_ATTN_V: ( + "vision_model.global_transformer.layers.{bid}.self_attn.v_proj", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_ATTN_GATE: ( + "vision_model.global_transformer.layers.{bid}.gate_attn", # mllama + ), + + MODEL_TENSOR.MM_V_GLOBAL_FFN_GATE: ( + "vision_model.global_transformer.layers.{bid}.gate_ffn", # mllama + ), + + MODEL_TENSOR.MM_V_INPUT_NORM: ( + "vision_model.transformer.layers.{bid}.input_layernorm", # mllama + ), + + MODEL_TENSOR.MM_V_FFN_UP: ( + "vision_model.transformer.layers.{bid}.mlp.fc1", # mllama + ), + + MODEL_TENSOR.MM_V_FFN_DOWN: ( + "vision_model.transformer.layers.{bid}.mlp.fc2", # mllama + ), + + MODEL_TENSOR.MM_V_ATTN_OUT_NORM: ( + "vision_model.transformer.layers.{bid}.post_attention_layernorm", # mllama + ), + + MODEL_TENSOR.MM_V_ATTN_K: ( + "vision_model.transformer.layers.{bid}.self_attn.k_proj", # mllama + ), + + MODEL_TENSOR.MM_V_ATTN_OUT: ( + "vision_model.transformer.layers.{bid}.self_attn.o_proj", # mllama + ), + + MODEL_TENSOR.MM_V_ATTN_Q: ( + "vision_model.transformer.layers.{bid}.self_attn.q_proj", # mllama + ), + + MODEL_TENSOR.MM_V_ATTN_V: ( + "vision_model.transformer.layers.{bid}.self_attn.v_proj", # mllama + ), + } # architecture-specific block mappings diff --git a/include/llama.h b/include/llama.h index bc014bac8f660..e80bef3387a44 100644 --- a/include/llama.h +++ b/include/llama.h @@ -105,6 +105,7 @@ extern "C" { LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, + LLAMA_VOCAB_PRE_TYPE_MLLAMA = 28, }; enum llama_rope_type { @@ -238,7 +239,7 @@ extern "C" { uint32_t ny; unsigned char * data; } llama_img; - + // Input data for llama_vision_decode typedef struct llama_batch_img { int32_t n_imgs; @@ -264,6 +265,7 @@ extern "C" { llama_token * token; float * embd; + int32_t n_embd; llama_pos * pos; int32_t * n_seq_id; llama_seq_id ** seq_id; @@ -431,6 +433,10 @@ extern "C" { const char * path_model, struct llama_model_params params); + LLAMA_API void llama_set_cross_attention(struct llama_context * ctx, bool cross_attn_state); + + LLAMA_API void llama_set_n_tiles(struct llama_context * ctx, uint32_t n_tiles); + LLAMA_API void llama_free_model(struct llama_model * model); // TODO: rename to llama_init_from_model diff --git a/models/ggml-vocab-mllama.gguf b/models/ggml-vocab-mllama.gguf new file mode 100644 index 0000000000000..9297336fbea7f Binary files /dev/null and b/models/ggml-vocab-mllama.gguf differ diff --git a/src/llama-vision.cpp b/src/llama-vision.cpp index 93f9e4b529402..86e037fd2cec7 100644 --- a/src/llama-vision.cpp +++ b/src/llama-vision.cpp @@ -76,6 +76,12 @@ int clip_n_patches(const clip_context & ctx) { return n_patches; } +int ca_n_patches(const ca_context & ctx) { + auto & hparams = ctx.model->hparams; + int n_patches = (hparams.image_size / hparams.patch_size) * (hparams.image_size / hparams.patch_size); + return n_patches; +} + int clip_n_mmproj_embd(const clip_context & ctx) { if (ctx.model->hparams.proj_type == CLIP_PROJECTOR_TYPE_MLP) { return ctx.model->mm_2_b->ne[0]; @@ -84,6 +90,10 @@ int clip_n_mmproj_embd(const clip_context & ctx) { } } +int ca_n_mmproj_embd(const ca_context & ctx) { + return ctx.model->mm_1_b->ne[0]; +} + /** * Selects the best resolution from a list of possible resolutions based on the original size. * @@ -408,10 +418,10 @@ static ggml_cgraph * clip_image_build_graph(clip_context & ctx, int batch_size, const int d_head = hidden_size / n_head; const int patch_size = hparams.patch_size; const float eps = hparams.eps; - const int num_patches = ((image_size.width / patch_size) * (image_size.height / patch_size)); - const int num_positions = num_patches + (model.class_embedding ? 1 : 0); + const int n_patches = ((image_size.width / patch_size) * (image_size.height / patch_size)); + const int n_positions = n_patches + (model.class_embedding ? 1 : 0); - LLAMA_LOG_DEBUG("%s: num_patches = %d\n", __func__, num_patches); + LLAMA_LOG_DEBUG("%s: n_patches = %d\n", __func__, n_patches); struct ggml_init_params params = { /*.mem_size =*/ ctx.buf_compute_meta.size(), @@ -430,9 +440,13 @@ static ggml_cgraph * clip_image_build_graph(clip_context & ctx, int batch_size, ggml_set_input(inp_raw); struct ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + ggml_set_name(inp, "inp_conv2d"); + ggml_set_name(model.patch_embeddings, "model_patch_embeddings"); - inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size); + inp = ggml_reshape_3d(ctx0, inp, n_patches, hidden_size, batch_size); + ggml_set_name(inp, "inp_after_reshape"); inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); + ggml_set_name(inp, "inp_after_cont"); if (model.patch_bias) { inp = ggml_add(ctx0, inp, model.patch_bias); @@ -441,7 +455,7 @@ static ggml_cgraph * clip_image_build_graph(clip_context & ctx, int batch_size, embeddings = inp; if (model.class_embedding) { - embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size); + embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, n_positions, batch_size); ggml_set_name(embeddings, "embeddings"); ggml_set_input(embeddings); embeddings = ggml_acc(ctx0, embeddings, model.class_embedding, @@ -450,7 +464,7 @@ static ggml_cgraph * clip_image_build_graph(clip_context & ctx, int batch_size, embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]); } - struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions); + struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_positions); ggml_set_name(positions, "positions"); ggml_set_input(positions); @@ -487,33 +501,33 @@ static ggml_cgraph * clip_image_build_graph(clip_context & ctx, int batch_size, model.layers[il].q_b); Q = ggml_scale_inplace(ctx0, Q, 1.0f / sqrt((float)d_head)); - Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, num_positions, batch_size); + Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, n_positions, batch_size); Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3)); - Q = ggml_reshape_3d(ctx0, Q, d_head, num_positions, n_head * batch_size); + Q = ggml_reshape_3d(ctx0, Q, d_head, n_positions, n_head * batch_size); struct ggml_tensor * K = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].k_w, cur), model.layers[il].k_b); - K = ggml_reshape_4d(ctx0, K, d_head, n_head, num_positions, batch_size); + K = ggml_reshape_4d(ctx0, K, d_head, n_head, n_positions, batch_size); K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3)); - K = ggml_reshape_3d(ctx0, K, d_head, num_positions, n_head * batch_size); + K = ggml_reshape_3d(ctx0, K, d_head, n_positions, n_head * batch_size); struct ggml_tensor * V = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].v_w, cur), model.layers[il].v_b); - V = ggml_reshape_4d(ctx0, V, d_head, n_head, num_positions, batch_size); + V = ggml_reshape_4d(ctx0, V, d_head, n_head, n_positions, batch_size); V = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3)); - V = ggml_reshape_3d(ctx0, V, num_positions, d_head, n_head * batch_size); + V = ggml_reshape_3d(ctx0, V, n_positions, d_head, n_head * batch_size); struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); KQ = ggml_soft_max_inplace(ctx0, KQ); struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ); - KQV = ggml_reshape_4d(ctx0, KQV, d_head, num_positions, n_head, batch_size); + KQV = ggml_reshape_4d(ctx0, KQV, d_head, n_positions, n_head, batch_size); KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3); - cur = ggml_cont_3d(ctx0, KQV, hidden_size, num_positions, batch_size); + cur = ggml_cont_3d(ctx0, KQV, hidden_size, n_positions, batch_size); } // attention output @@ -562,7 +576,7 @@ static ggml_cgraph * clip_image_build_graph(clip_context & ctx, int batch_size, { embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]); - struct ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_patches); + struct ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); ggml_set_name(patches, "patches"); ggml_set_input(patches); @@ -599,11 +613,11 @@ static int32_t clip_image_batch_encode(clip_context & ctx, const clip_image_f32_ clip_image_size image_size{(int)hparams.image_size, (int)hparams.image_size}; const int patch_size = hparams.patch_size; - const int num_patches = ((image_size.width / patch_size) * (image_size.height / patch_size)); - const int num_positions = num_patches + (model.class_embedding ? 1 : 0); + const int n_patches = ((image_size.width / patch_size) * (image_size.height / patch_size)); + const int n_positions = n_patches + (model.class_embedding ? 1 : 0); LLAMA_LOG_DEBUG("%s: image_size = %d\n", __func__, hparams.image_size); - LLAMA_LOG_DEBUG("%s: num_positions = %d\n", __func__, num_positions); + LLAMA_LOG_DEBUG("%s: n_positions = %d\n", __func__, n_positions); // build the inference graph ggml_cgraph * gf = clip_image_build_graph(ctx, batch_size, image_size); @@ -648,11 +662,12 @@ static int32_t clip_image_batch_encode(clip_context & ctx, const clip_image_f32_ free(zero_mem); } + { struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions"); int* positions_data = (int*)malloc(ggml_nbytes(positions)); - for (int i = 0; i < num_positions; i++) { + for (int i = 0; i < n_positions; i++) { positions_data[i] = i; } ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions)); @@ -662,16 +677,110 @@ static int32_t clip_image_batch_encode(clip_context & ctx, const clip_image_f32_ { struct ggml_tensor * patches = ggml_graph_get_tensor(gf, "patches"); int* patches_data = (int*)malloc(ggml_nbytes(patches)); - for (int i = 0; i < num_patches; i++) { + for (int i = 0; i < n_patches; i++) { patches_data[i] = i + 1; } ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches)); free(patches_data); } + { + struct ggml_tensor* next_out = model.patch_embeddings; + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get_async(next_bk, next_out, next_buffer, 0, sizeof(next_buffer)); + for (int i = 0; i < 10; i++) { + printf("model_patch_embeddings[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + + { + struct ggml_tensor* next_out = ggml_graph_get_tensor(gf, "inp_raw"); + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get_async(next_bk, next_out, next_buffer, 0, sizeof(next_buffer)); + //ggml_backend_sched_synchronize(ctx.sched); + for (int i = 0; i < 10; i++) { + printf("inp_raw[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + + { + struct ggml_tensor* next_out = ggml_graph_get_tensor(gf, "inp_conv2d"); + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get(next_out, next_buffer, 0, sizeof(next_buffer)); + for (int i = 0; i < 10; i++) { + printf("inp_conv2d[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + + { + struct ggml_tensor* next_out = model.patch_embeddings; + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get_async(next_bk, next_out, next_buffer, 0, sizeof(next_buffer)); + for (int i = 0; i < 10; i++) { + printf("patch_embeddings[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + + printf("-------------------- Compute graph --------------------\n"); + // compute ggml_backend_sched_graph_compute_async(ctx.sched, gf); + { + struct ggml_tensor* next_out = model.patch_embeddings; + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get_async(next_bk, next_out, next_buffer, 0, sizeof(next_buffer)); + for (int i = 0; i < 10; i++) { + printf("model_patch_embeddings[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + + { + struct ggml_tensor* next_out = ggml_graph_get_tensor(gf, "inp_raw"); + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get_async(next_bk, next_out, next_buffer, 0, sizeof(next_buffer)); + //ggml_backend_sched_synchronize(ctx.sched); + for (int i = 0; i < 10; i++) { + printf("inp_raw[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + + { + struct ggml_tensor* next_out = ggml_graph_get_tensor(gf, "inp_conv2d"); + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get_async(next_bk, next_out, next_buffer, 0, sizeof(next_buffer)); + //ggml_backend_sched_synchronize(ctx.sched); + for (int i = 0; i < 10; i++) { + printf("inp_conv2d[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + + { + struct ggml_tensor* next_out = ggml_graph_get_tensor(gf, "inp_after_reshape"); + float next_buffer[10]; + ggml_backend_t next_bk = ggml_backend_sched_get_tensor_backend(ctx.sched, next_out); + printf("Backend type: %s\n", ggml_backend_name(next_bk)); + ggml_backend_tensor_get_async(next_bk, next_out, next_buffer, 0, sizeof(next_buffer)); + //ggml_backend_sched_synchronize(ctx.sched); + for (int i = 0; i < 10; i++) { + printf("inp_after_reshape[%d] = %f (isnan=%d)\n", i, next_buffer[i], std::isnan(next_buffer[i])); + } + } + // the last node is the embedding tensor struct ggml_tensor * embeddings = ggml_graph_node(gf, -1); ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(ctx.sched, embeddings); @@ -726,6 +835,437 @@ static int32_t encode_image_with_clip(clip_context & ctx, const llama_img img, s return 0; } +// TODO(danbev) Incorporate the following into the llama_img struct +// or something similar later. This is only to try to get something working +// and I'm currently cutting corners to not spend too much time on this. +struct mllama_image { + const llama_img * img; + + int n_channels = 3; + int n_tiles = 4; + int aspect_ratio_id; +}; + +struct mllama_image_batch { + struct mllama_image * data; + size_t size; +}; + + +static ggml_tensor * mllama_image_build_encoder_layer( + struct ggml_context * ctx0, const size_t il, const struct ca_layer & layer, struct ggml_tensor * embeddings, + const float eps, const int hidden_size, const int batch_size, const int n_head, const int d_head) { + struct ggml_tensor *cur = embeddings; + + { + // layernorm1 + cur = ggml_norm(ctx0, cur, eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.norm_in_w), layer.norm_in_b); + ggml_format_name(cur, "layer_norm_in-%ld", il); + } + + { + // self-attention + struct ggml_tensor *Q = ggml_mul_mat(ctx0, layer.q_w, cur); + if (layer.q_b != nullptr) { + Q = ggml_add(ctx0, Q, layer.q_b); + } + + Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, Q->ne[1], batch_size); + Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3)); + + struct ggml_tensor *K = ggml_mul_mat(ctx0, layer.k_w, cur); + if (layer.k_b != nullptr) { + K = ggml_add(ctx0, K, layer.k_b); + } + + K = ggml_reshape_4d(ctx0, K, d_head, n_head, K->ne[1], batch_size); + K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3)); + + struct ggml_tensor *V = ggml_mul_mat(ctx0, layer.v_w, cur); + if (layer.v_b != nullptr) { + V = ggml_add(ctx0, V, layer.v_b); + } + + V = ggml_reshape_4d(ctx0, V, d_head, n_head, V->ne[1], batch_size); + V = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3)); + + struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q); + KQ = ggml_scale(ctx0, KQ, 1.0f / sqrtf((float)d_head)); + KQ = ggml_soft_max(ctx0, KQ); + + struct ggml_tensor *KQV = ggml_mul_mat(ctx0, V, KQ); + KQV = ggml_reshape_4d(ctx0, KQV, d_head, KQV->ne[1], n_head, batch_size); + KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3); + KQV = ggml_cont_3d(ctx0, KQV, hidden_size, KQV->ne[2], batch_size); + + cur = ggml_mul_mat(ctx0, layer.output_w, KQV); + if (layer.output_b != nullptr) { + cur = ggml_add(ctx0, cur, layer.output_b); + } + + if (layer.attn_gate != nullptr) { + cur = ggml_mul(ctx0, cur, layer.attn_gate); + } + } + + cur = ggml_add(ctx0, cur, embeddings); + + embeddings = cur; + + { + // layernorm2 + cur = ggml_norm(ctx0, cur, eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.norm_out_w), layer.norm_out_b); + } + + { + // feed forward + cur = ggml_add(ctx0, ggml_mul_mat(ctx0, layer.ffn_up_w, cur), layer.ffn_up_b); + cur = ggml_gelu(ctx0, cur); + cur = ggml_add(ctx0, ggml_mul_mat(ctx0, layer.ffn_down_w, cur), layer.ffn_down_b); + + if (layer.ffn_gate != nullptr) { + cur = ggml_mul(ctx0, cur, layer.ffn_gate); + } + } + + // residual 2 + cur = ggml_add(ctx0, cur, embeddings); + + embeddings = cur; + + return embeddings; +} + +using ca_image_batch = std::vector; + +static ggml_cgraph * mllama_image_build_graph(ca_context * ctx, const ca_image_batch img_batch) { + const auto & model = *ctx->model; + const auto &hparams = model.hparams; + + const int image_size = hparams.image_size; + const int image_size_width = image_size; + const int image_size_height = image_size; + + const int patch_size = hparams.patch_size; + const int n_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); + const int n_positions = n_patches + (model.class_embedding == nullptr ? 0 : 1); + const int hidden_size = hparams.hidden_size; + const int n_head = hparams.n_head; + const int d_head = hidden_size / n_head; + + const int batch_size = img_batch.size(); + GGML_ASSERT(img_batch.size() == 1); + + const int n_tiles = img_batch[0].n_tiles > 0 ? img_batch[0].n_tiles : 4; + const int n_channels = img_batch[0].n_channels > 0 ? img_batch[0].n_channels : 3; + + + struct ggml_init_params params = { + /* mem_size */ ctx->buf_compute_meta.size(), + /* mem buffer */ ctx->buf_compute_meta.data(), + /* no_alloc */ true, + }; + + struct ggml_context * ctx0 = ggml_init(params); + struct ggml_cgraph * gf = ggml_new_graph(ctx0); + + struct ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size_width, image_size_height, n_channels, n_tiles); + ggml_set_input(inp_raw); + ggml_set_name(inp_raw, "inp_raw"); + + // TODO:(danbev) Removing this and one a bit further down causes the output + // to be incorrect. The output will be something like: + //"The image shows a gray background..." + // + // I initially thought it was because the image patch embeddings were not + // being generated correctly but when I checked the output image patch + // embeddings (uncommenting the code in encode_image_with_ca_vision) using: + // $ sha256sum image_patch_embeddings.bin + // 319cc0572866e3b165d5f59dc3e5709b87ec503ff6af10e3cd487d21e2ad17ab image_patch_embeddings.bin + // + // They are the same with this code commmented out or not. + // + // The same output will also be generated if the number of layers to offload + // to the GPU is greater than 35 (the number of layers in the model is 40). + struct ggml_tensor * inp_raw_copy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size_width, image_size_height, n_channels, n_tiles); + ggml_set_input(inp_raw_copy); + ggml_set_name(inp_raw_copy, "inp_raw_copy"); + ggml_build_forward_expand(gf, inp_raw_copy); + + struct ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + inp = ggml_reshape_3d(ctx0, inp, n_patches, hidden_size, n_tiles); + inp = ggml_permute(ctx0, inp, 1, 0, 2, 3); + inp = ggml_cont(ctx0, inp); + + struct ggml_tensor * aspect_ratios = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, img_batch.size()); + ggml_set_name(aspect_ratios, "aspect_ratios"); + ggml_set_input(aspect_ratios); + ggml_build_forward_expand(gf, aspect_ratios); + + if (model.pre_tile_position_embeddings != nullptr) { + struct ggml_tensor *pre_tile_position_embeddings = ggml_get_rows(ctx0, model.pre_tile_position_embeddings, aspect_ratios); + + pre_tile_position_embeddings = ggml_reshape_3d(ctx0, pre_tile_position_embeddings, hidden_size, 1, n_tiles); + if (model.pre_tile_position_embeddings_gate != nullptr) { + pre_tile_position_embeddings = ggml_mul(ctx0, pre_tile_position_embeddings, model.pre_tile_position_embeddings_gate); + } + + inp = ggml_add(ctx0, inp, pre_tile_position_embeddings); + } + ggml_set_name(inp, "after_pre_tile_position_embeddings"); + + struct ggml_tensor * embeddings = inp; + + if (model.class_embedding != nullptr) { + embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, n_positions, n_tiles); + ggml_set_input(embeddings); + ggml_set_name(embeddings, "embeddings"); + + for (int i = 0; i < n_tiles; ++i) { + // repeat class embeddings for each tile + embeddings = ggml_acc(ctx0, embeddings, model.class_embedding, embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], i * embeddings->nb[2]); + } + + embeddings = ggml_acc(ctx0, embeddings, inp, embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]); + } + + struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_positions); + ggml_set_name(positions, "positions"); + ggml_set_input(positions); + + struct ggml_tensor * position_embd = ggml_get_rows(ctx0, model.position_embeddings, positions); + ggml_set_name(position_embd, "position_embd"); + + if (model.position_embeddings_gate != nullptr) { + position_embd = ggml_mul(ctx0, position_embd, model.position_embeddings_gate); + } + + embeddings = ggml_add(ctx0, embeddings, position_embd); + + if (model.tile_position_embeddings != nullptr) { + struct ggml_tensor * tile_position_embeddings = ggml_get_rows(ctx0, model.tile_position_embeddings, aspect_ratios); + ggml_set_name(tile_position_embeddings, "tile_position_embeddings"); + + tile_position_embeddings = ggml_reshape_3d(ctx0, tile_position_embeddings, hidden_size, n_positions, n_tiles); + ggml_build_forward_expand(gf, tile_position_embeddings); + + // TODO(danbev) See previous comment about this and why it is here for + // now. + struct ggml_tensor * tile_pos_embd_reshape_copy = ggml_reshape_3d(ctx0, tile_position_embeddings, hidden_size, n_positions, n_tiles); + ggml_set_name(tile_pos_embd_reshape_copy, "tile_pos_embd_reshaped_copy"); + ggml_build_forward_expand(gf, tile_pos_embd_reshape_copy); + + if (model.tile_position_embeddings_gate != nullptr) { + tile_position_embeddings = ggml_mul(ctx0, tile_position_embeddings, model.tile_position_embeddings_gate); + } + + embeddings = ggml_add(ctx0, embeddings, tile_position_embeddings); + } + + // pre-layernorm + if (model.pre_norm_w != nullptr) { + embeddings = ggml_mul(ctx0, ggml_norm(ctx0, embeddings, hparams.eps), model.pre_norm_w); + if (model.pre_norm_b != nullptr) { + embeddings = ggml_add(ctx0, embeddings, model.pre_norm_b); + } + + ggml_set_name(embeddings, "pre layernorm"); + } + + const int num_padding_patches = 8 - (embeddings->ne[1] % 8) % 8; + + embeddings = ggml_pad(ctx0, embeddings, 0, num_padding_patches, 0, 0); + + embeddings = ggml_view_3d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1] * embeddings->ne[2], batch_size, embeddings->nb[1], embeddings->nb[2] * embeddings->ne[3], 0); + + std::vector intermediate_embeddings; + + // encoder + for (size_t il = 0, iil = 0; il < model.layers.size(); il++) { + if (hparams.intermediate_layers[iil] == (int) il) { + intermediate_embeddings.push_back(embeddings); + iil++; + } + + embeddings = mllama_image_build_encoder_layer( + ctx0, il, model.layers[il], embeddings, + hparams.eps, hidden_size, batch_size, n_head, d_head); + } + + struct ggml_tensor * embeddings_after_layers = ggml_dup(ctx0, embeddings); + ggml_set_name(embeddings_after_layers, "embeddings_after_layers"); + + // post-layernorm + if (model.post_norm_w != nullptr) { + embeddings = ggml_mul(ctx0, ggml_norm(ctx0, embeddings, hparams.eps), model.post_norm_w); + if (model.post_norm_b != nullptr) { + embeddings = ggml_add(ctx0, embeddings, model.post_norm_b); + } + + ggml_set_name(embeddings, "post layernorm"); + } + + embeddings = ggml_reshape_3d(ctx0, embeddings, hidden_size, n_positions + num_padding_patches, n_tiles); + + if (model.post_tile_position_embeddings != nullptr) { + struct ggml_tensor * post_tile_position_embeddings = ggml_get_rows(ctx0, model.post_tile_position_embeddings, aspect_ratios); + ggml_set_name(post_tile_position_embeddings, "post_tile_position_embeddings"); + + post_tile_position_embeddings = ggml_reshape_3d(ctx0, post_tile_position_embeddings, hidden_size, 1, n_tiles); + if (model.post_tile_position_embeddings_gate != nullptr) { + post_tile_position_embeddings = ggml_mul(ctx0, post_tile_position_embeddings, model.post_tile_position_embeddings_gate); + } + + embeddings = ggml_add(ctx0, embeddings, post_tile_position_embeddings); + } + + embeddings = ggml_reshape_3d(ctx0, embeddings, hidden_size, n_tiles * (n_positions + num_padding_patches), 1); + + // global encoder + for (size_t il = 0; il < model.global_layers.size(); il++) { + embeddings = mllama_image_build_encoder_layer( + ctx0, il, model.global_layers[il], embeddings, + hparams.eps, hidden_size, batch_size, n_head, d_head); + } + + struct ggml_tensor * stacked_embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 0, hidden_size, (n_positions + num_padding_patches) * n_tiles); + for (size_t i = 0; i < intermediate_embeddings.size(); ++i) { + stacked_embeddings = ggml_concat(ctx0, stacked_embeddings, ggml_reshape_3d(ctx0, intermediate_embeddings[i], 1, intermediate_embeddings[i]->ne[0], intermediate_embeddings[i]->ne[1]), 0); + } + + stacked_embeddings = ggml_reshape_4d(ctx0, stacked_embeddings, intermediate_embeddings.size() * hidden_size, n_positions + num_padding_patches, n_tiles, batch_size); + stacked_embeddings = ggml_unpad(ctx0, stacked_embeddings, 0, num_padding_patches, 0, 0); + + embeddings = ggml_reshape_3d(ctx0, embeddings, hidden_size, n_positions + num_padding_patches, n_tiles); + embeddings = ggml_unpad(ctx0, embeddings, 0, num_padding_patches, 0, 0); + embeddings = ggml_concat(ctx0, embeddings, stacked_embeddings, 0); + + // mllama projector + embeddings = ggml_add(ctx0, ggml_mul_mat(ctx0, model.mm_1_w, embeddings), model.mm_1_b); + ggml_set_name(embeddings, "mmproj"); + + // build the graph + ggml_build_forward_expand(gf, embeddings); + + ggml_free(ctx0); + + return gf; +} + +static int32_t encode_image_with_ca_vision(ca_context & ctx, + llama_img img, std::vector & output) { + const auto & model = *ctx.model; + auto & hparams = model.hparams; + + mllama_image mllama_img = {&img, 3, 4, 6}; + ca_image_batch img_batch{mllama_img}; + + static ggml_cgraph * gf = mllama_image_build_graph(&ctx, img_batch); + ggml_backend_sched_reset(ctx.sched); + bool ok = ggml_backend_sched_alloc_graph(ctx.sched, gf); + if (!ok) { + LLAMA_LOG_ERROR("failed to alloc memory for graph\n"); + return -1; + } + + // Set inputs + const int image_size = hparams.image_size; + int image_size_width = image_size; + int image_size_height = image_size; + + const int patch_size = hparams.patch_size; + const int n_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); + const int n_positions = n_patches + (model.class_embedding == nullptr ? 0 : 1); + + // set raw input + { + struct ggml_tensor * inp_raw = ggml_graph_get_tensor(gf, "inp_raw"); + ggml_backend_tensor_set(inp_raw, img_batch[0].img->data, 0, ggml_nbytes(inp_raw)); + + const char * name = inp_raw->name; + float buf[10]; + ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(ctx.sched, inp_raw); + printf("%s tensor type: %s\n", name, ggml_type_name(inp_raw->type)); + printf("%s backend type: %s\n", name, ggml_backend_name(backend)); + ggml_backend_tensor_get_async(backend, inp_raw, buf, 0, sizeof(buf)); + ggml_backend_sched_synchronize(ctx.sched); + for (int i = 0; i < 10; i++) { + printf("%s[%d] = %f\n", name, i, buf[i]); + } + + for (int i = 0; i < 10; i++) { + printf("inp_raw[%d] = %f\n", i, img_batch[0].img->data[i]); + } + } + + if (model.class_embedding) { + struct ggml_tensor * embeddings = ggml_graph_get_tensor(gf, "embeddings"); + + void* zero_mem = malloc(ggml_nbytes(embeddings)); + memset(zero_mem, 0, ggml_nbytes(embeddings)); + ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings)); + free(zero_mem); + } + + + { + struct ggml_tensor * positions = ggml_graph_get_tensor(gf, "positions"); + if (positions != nullptr) { + int * positions_data = (int *)malloc(ggml_nbytes(positions)); + for (int i = 0; i < n_positions; i++) { + positions_data[i] = i; + } + ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions)); + free(positions_data); + + } + } + + { + struct ggml_tensor * aspect_ratios = ggml_graph_get_tensor(gf, "aspect_ratios"); + if (aspect_ratios != nullptr) { + int * aspect_ratios_data = (int *)malloc(ggml_nbytes(aspect_ratios)); + aspect_ratios_data[0] = img_batch[0].aspect_ratio_id; + ggml_backend_tensor_set(aspect_ratios, aspect_ratios_data, 0, ggml_nbytes(aspect_ratios)); + free(aspect_ratios_data); + } + } + + ggml_backend_sched_graph_compute_async(ctx.sched, gf); + + struct ggml_tensor * embeddings = ggml_graph_get_tensor(gf, "mmproj"); + + size_t out_nbytes = n_positions * embeddings->ne[0] * hparams.n_tiles * sizeof(float); + GGML_ASSERT(out_nbytes == ggml_nbytes(embeddings)); + output.resize(out_nbytes); + + ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(ctx.sched, embeddings); + ggml_backend_tensor_get_async(backend_embd, embeddings, output.data(), 0, ggml_nbytes(embeddings)); + ggml_backend_sched_synchronize(ctx.sched); + + for (int i = 0; i < 10; i++) { + printf("output[%d] = %f\n", i, output[i]); + } + + // Save image patch embeddings to file for debugging. + /* + std::string filename = "image_patch_embeddings.bin"; + std::ofstream file(filename, std::ios::binary); + if (!file) { + printf("Failed to open file %s\n", filename.c_str()); + return 1; + } + file.write(reinterpret_cast(output.data()), output.size() * sizeof(float)); + file.close(); + */ + + ggml_backend_sched_reset(ctx.sched); + return 0; +} + + //////////////////////////////////////////////////////////////////////////////////////// // public API @@ -761,11 +1301,56 @@ int32_t llama_encode_vision_internal(clip_context & ctx, llama_batch_img * batch return 0; } +int32_t ca_llama_encode_vision_internal(ca_context & ctx, llama_batch_img * batch) { + if (batch->n_imgs == 0) { + return 0; + } + const int n_embd = ca_n_mmproj_embd(ctx); + const int n_tokens_per_img = ca_n_patches(ctx); + + const int image_size = ctx.model->hparams.image_size; + int image_size_width = image_size; + int image_size_height = image_size; + const int patch_size = ctx.model->hparams.patch_size; + const int n_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); + const int n_positions = n_patches + (ctx.model->class_embedding == nullptr ? 0 : 1); + + size_t out_ntokens = n_positions * ctx.model->hparams.n_tiles; + size_t out_nbytes = out_ntokens * n_embd * sizeof(float); + ctx.out_embd.resize(out_nbytes); + ctx.out_pos.resize(n_positions); + + for (int i = 0; i < batch->n_imgs; i++) { + std::vector output_single; + int32_t status = encode_image_with_ca_vision(ctx, *batch->imgs[i], output_single); + if (status != 0) { + return status; + } + + // copy output embeddings to result + for (size_t k = 0; k < out_nbytes; k++) { + ctx.out_embd[n_embd * n_tokens_per_img * i + k] = output_single[k]; + } + + // fill position for all output tokens + for (int p = 0; p < n_positions; p++) { + ctx.out_pos[n_tokens_per_img*i + p] = batch->pos[i] + p; + } + } + + return 0; +} + void llama_vision_clear_output(clip_context & ctx) { ctx.out_embd.clear(); ctx.out_pos.clear(); } +void ca_llama_vision_clear_output(ca_context & ctx) { + ctx.out_embd.clear(); + ctx.out_pos.clear(); +} + //////////////////////////////////////////////////////////////////////////////////////// // for debugging #ifndef NDEBUG diff --git a/src/llama-vision.h b/src/llama-vision.h index 844c535d2eced..4feb0d3a22703 100644 --- a/src/llama-vision.h +++ b/src/llama-vision.h @@ -9,6 +9,7 @@ enum vision_arch { VISION_ARCH_UNKNOWN, VISION_ARCH_LLAVA, + VISION_ARCH_MLLAMA, }; enum clip_projector_type { @@ -48,6 +49,7 @@ struct clip_hparams { int32_t image_crop_resolution; }; + struct clip_layer { // attention struct ggml_tensor * k_w = NULL; @@ -116,6 +118,106 @@ struct clip_context { std::vector out_pos; // position of each token }; +// Cross Attention (CA) parameters +struct ca_hparams { + vision_arch arch = VISION_ARCH_UNKNOWN; + + uint32_t image_size; + uint32_t patch_size; + uint32_t hidden_size; + uint32_t n_intermediate; + uint32_t projection_dim; + uint32_t n_head; + uint32_t n_layer; + uint32_t max_pos_embd; + uint32_t proj_n_embd; + + uint32_t n_global_layer; + uint32_t n_tiles; + + std::array image_mean; + std::array image_std; + + float eps; + + std::array intermediate_layers; + std::array aspect_ratios; +}; + +// Cross Attention (CA) layer +struct ca_layer { + // attention + struct ggml_tensor * k_w = NULL; + struct ggml_tensor * k_b = NULL; + struct ggml_tensor * q_w = NULL; + struct ggml_tensor * q_b = NULL; + struct ggml_tensor * v_w = NULL; + struct ggml_tensor * v_b = NULL; + + struct ggml_tensor * output_w = NULL; + struct ggml_tensor * output_b = NULL; + + struct ggml_tensor * attn_gate = NULL; + + // layernorm 1 + struct ggml_tensor * norm_in_w = NULL; + struct ggml_tensor * norm_in_b = NULL; + + // feed-forward + struct ggml_tensor * ffn_up_w = NULL; + struct ggml_tensor * ffn_up_b = NULL; + + struct ggml_tensor * ffn_down_w = NULL; + struct ggml_tensor * ffn_down_b = NULL; + + struct ggml_tensor * ffn_gate = NULL; + + // layernorm 2 + struct ggml_tensor * norm_out_w = NULL; + struct ggml_tensor * norm_out_b = NULL; +}; + +// Cross Attention (CA) model +struct ca_vision_model { + struct ca_hparams hparams; + + // embeddings + struct ggml_tensor * class_embedding = NULL; + struct ggml_tensor * patch_embeddings = NULL; + struct ggml_tensor * position_embeddings = NULL; + struct ggml_tensor * position_embeddings_gate = NULL; + struct ggml_tensor * tile_position_embeddings = NULL; + struct ggml_tensor * tile_position_embeddings_gate = NULL; + struct ggml_tensor * pre_tile_position_embeddings = NULL; + struct ggml_tensor * pre_tile_position_embeddings_gate = NULL; + struct ggml_tensor * post_tile_position_embeddings = NULL; + struct ggml_tensor * post_tile_position_embeddings_gate = NULL; + + struct ggml_tensor * pre_norm_w = NULL; + struct ggml_tensor * pre_norm_b = NULL; + + std::vector global_layers; + std::vector layers; + + struct ggml_tensor * post_norm_w = NULL; + struct ggml_tensor * post_norm_b = NULL; + + struct ggml_tensor * mm_1_w = NULL; + struct ggml_tensor * mm_1_b = NULL; +}; + +struct ca_context { + // memory buffers used to evaluate the model + std::vector buf_compute_meta; + ggml_backend_sched_t sched = nullptr; + + const ca_vision_model * model; + + // temporary output data, to be picked up by llama_decode() + std::vector out_embd; // size == n_tokens * n_embd + std::vector out_pos; // position of each token +}; + mm_patch_merge mm_patch_merge_from_name(std::string & name); clip_projector_type projector_type_from_name(std::string & name); int clip_n_patches(const clip_context & ctx); @@ -123,3 +225,9 @@ int clip_n_mmproj_embd(const clip_context & ctx); int32_t llama_encode_vision_internal(clip_context & ctx, llama_batch_img * batch); void llama_vision_clear_output(clip_context & ctx); + +void ca_llama_vision_clear_output(ca_context & ctx); +int32_t ca_llama_encode_vision_internal(ca_context & ctx, llama_batch_img * batch); + +int ca_n_patches(const ca_context & ctx); +int ca_n_mmproj_embd(const ca_context & ctx); diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 7f2725f94be13..a753aed2a70a6 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -381,6 +381,8 @@ struct llm_tokenizer_bpe : llm_tokenizer { break; case LLAMA_VOCAB_PRE_TYPE_DBRX: case LLAMA_VOCAB_PRE_TYPE_SMAUG: + case LLAMA_VOCAB_PRE_TYPE_MLLAMA: + printf("llm_tokenizer_bpe: using default regex for BPE tokenization pre-processing\n"); regex_exprs = { // same as llama3 "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 4bb16d2e4299f..69131a99536b7 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -47,6 +47,8 @@ struct llama_vocab { id special_pad_id = LLAMA_TOKEN_NULL; id special_cls_id = LLAMA_TOKEN_NULL; id special_mask_id = LLAMA_TOKEN_NULL; + id special_start_header_id = LLAMA_TOKEN_NULL; + id special_end_header_id = LLAMA_TOKEN_NULL; id linefeed_id = 13; @@ -61,6 +63,8 @@ struct llama_vocab { // set of all tokens that cause "end of generation" std::set special_eog_ids; + id special_image_id = LLAMA_TOKEN_NULL; + // tokenizer flags bool tokenizer_add_space_prefix = false; bool tokenizer_add_bos = false; diff --git a/src/llama.cpp b/src/llama.cpp index 13d06d98aceb8..ddffac728e50c 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -80,6 +80,7 @@ // bump if necessary #define LLAMA_MAX_LAYERS 512 #define LLAMA_MAX_EXPERTS 160 // DeepSeekV2 +#define LLAMA_CROSS_ATTN_LAYERS 8 // MLlama // // helpers @@ -199,6 +200,7 @@ enum llm_arch { LLM_ARCH_GRANITE_MOE, LLM_ARCH_CHAMELEON, LLM_ARCH_WAVTOKENIZER_DEC, + LLM_ARCH_MLLAMA, LLM_ARCH_UNKNOWN, }; @@ -256,12 +258,14 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_CHAMELEON, "chameleon" }, { LLM_ARCH_WAVTOKENIZER_DEC, "wavtokenizer-dec" }, + { LLM_ARCH_MLLAMA, "mllama" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; static const std::map VISION_ARCH_NAMES = { - { VISION_ARCH_LLAVA, "llava" }, - { VISION_ARCH_UNKNOWN, "(unknown)" }, + { VISION_ARCH_LLAVA, "llava" }, + { VISION_ARCH_MLLAMA, "mllama_vision_model" }, + { VISION_ARCH_UNKNOWN, "(unknown)" }, }; enum llm_kv { @@ -361,6 +365,7 @@ enum llm_kv { LLM_KV_TOKENIZER_PAD_ID, LLM_KV_TOKENIZER_CLS_ID, LLM_KV_TOKENIZER_MASK_ID, + LLM_KV_TOKENIZER_IMAGE_ID, LLM_KV_TOKENIZER_ADD_BOS, LLM_KV_TOKENIZER_ADD_EOS, LLM_KV_TOKENIZER_ADD_PREFIX, @@ -374,6 +379,8 @@ enum llm_kv { LLM_KV_TOKENIZER_FIM_PAD_ID, LLM_KV_TOKENIZER_FIM_REP_ID, LLM_KV_TOKENIZER_FIM_SEP_ID, + LLM_KV_TOKENIZER_START_HEADER_ID, + LLM_KV_TOKENIZER_END_HEADER_ID, LLM_KV_ADAPTER_TYPE, LLM_KV_ADAPTER_LORA_ALPHA, @@ -395,6 +402,12 @@ enum llm_kv { LLM_KV_VISION_PATCH_SIZE, LLM_KV_VISION_IMAGE_MEAN, LLM_KV_VISION_IMAGE_STD, + LLM_KV_VISION_EMBEDDING_LENGTH, + LLM_KV_VISION_BLOCK_COUNT, + LLM_KV_VISION_LAYERNORM_RMS_EPS, + LLM_KV_VISION_FEED_FORWARD_LENGTH, + LLM_KV_VISION_HEAD_COUNT, + LLM_KV_VISION_ARCHITECTURE, LLM_KV_VISION_CLIP_ARCHITECTURE, LLM_KV_VISION_CLIP_CONTEXT_LENGTH, LLM_KV_VISION_CLIP_EMBEDDING_LENGTH, @@ -410,6 +423,13 @@ enum llm_kv { LLM_KV_VISION_CLIP_PATCH_MERGE_TYPE, LLM_KV_VISION_CLIP_HEAD_COUNT, LLM_KV_VISION_CLIP_LAYERNORM_EPS, + // Cross-Attention Vision (mllama) + LLM_KV_VISION_CROSS_ATTENTION_LAYERS, + LLM_KV_VISION_IMAGE_TOKEN_INDEX, + LLM_KV_VISION_TILES, + LLM_KV_VISION_ASPECT_RATIOS, + LLM_KV_VISION_GLOBAL_BLOCK_COUNT, + LLM_KV_VISION_INTERMEDIATE_LAYERS, }; static const std::map LLM_KV_NAMES = { @@ -515,6 +535,9 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_TOKENIZER_PAD_ID, "tokenizer.ggml.padding_token_id" }, { LLM_KV_TOKENIZER_CLS_ID, "tokenizer.ggml.cls_token_id" }, { LLM_KV_TOKENIZER_MASK_ID, "tokenizer.ggml.mask_token_id" }, + { LLM_KV_TOKENIZER_IMAGE_ID, "tokenizer.ggml.image_token_id" }, + { LLM_KV_TOKENIZER_START_HEADER_ID, "tokenizer.ggml.start_header_token_id" }, + { LLM_KV_TOKENIZER_END_HEADER_ID, "tokenizer.ggml.end_header_token_id" }, { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" }, { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" }, { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" }, @@ -542,6 +565,12 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_VISION_PATCH_SIZE, "vision.patch_size" }, { LLM_KV_VISION_IMAGE_MEAN, "vision.image_mean" }, { LLM_KV_VISION_IMAGE_STD, "vision.image_std" }, + { LLM_KV_VISION_EMBEDDING_LENGTH, "vision.embedding_length" }, + { LLM_KV_VISION_BLOCK_COUNT, "vision.block_count" }, + { LLM_KV_VISION_LAYERNORM_RMS_EPS, "vision.attention.layer_norm_rms_epsilon" }, + { LLM_KV_VISION_FEED_FORWARD_LENGTH, "vision.feed_forward_length" }, + { LLM_KV_VISION_HEAD_COUNT, "vision.attention.head_count" }, + { LLM_KV_VISION_ARCHITECTURE, "vision.architecture" }, { LLM_KV_VISION_CLIP_ARCHITECTURE, "vision.clip.architecture" }, { LLM_KV_VISION_CLIP_CONTEXT_LENGTH, "vision.clip.context_length" }, { LLM_KV_VISION_CLIP_EMBEDDING_LENGTH, "vision.clip.embedding_length" }, @@ -557,6 +586,14 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_VISION_CLIP_PATCH_MERGE_TYPE, "vision.clip.patch_merge_type" }, { LLM_KV_VISION_CLIP_HEAD_COUNT, "vision.clip.attention.head_count" }, { LLM_KV_VISION_CLIP_LAYERNORM_EPS, "vision.clip.attention.layer_norm_epsilon" }, + + { LLM_KV_VISION_CROSS_ATTENTION_LAYERS, "%s.cross_attention_layers" }, + { LLM_KV_VISION_IMAGE_TOKEN_INDEX, "%s.image_token_index" }, + { LLM_KV_VISION_TILES, "vision.cross.%s.max_num_tiles" }, + { LLM_KV_VISION_ASPECT_RATIOS, "vision.cross.%s.supported_aspect_ratios" }, + { LLM_KV_VISION_GLOBAL_BLOCK_COUNT, "vision.cross.%s.global_block_count" }, + { LLM_KV_VISION_INTERMEDIATE_LAYERS, "vision.cross.%s.intermediate_layers_indices"}, + }; struct LLM_KV { @@ -694,6 +731,17 @@ enum llm_tensor { LLM_TENSOR_POS_NET_ATTN_K, LLM_TENSOR_POS_NET_ATTN_V, LLM_TENSOR_POS_NET_ATTN_OUT, + // Cross Attention tensors names + LLM_TENSOR_CA_K_NORM, + LLM_TENSOR_CA_K_PROJ, + LLM_TENSOR_CA_O_PROJ, + LLM_TENSOR_CA_Q_NORM, + LLM_TENSOR_CA_Q_PROJ, + LLM_TENSOR_CA_V_PROJ, + LLM_TENSOR_CA_ATTN_GATE, + LLM_TENSOR_CA_MLP_GATE, + LLM_TENSOR_CA_FFN_UP, + LLM_TENSOR_CA_FFN_DOWN, }; enum vision_tensor { @@ -701,9 +749,12 @@ enum vision_tensor { VISION_TENSOR_ENC_EMBD_CLS, VISION_TENSOR_ENC_EMBD_PATCH, VISION_TENSOR_ENC_EMBD_POS, + VISION_TENSOR_ENC_EMBD_POS_GATE, + VISION_TENSOR_ENC_ATTN_Q, VISION_TENSOR_ENC_ATTN_K, VISION_TENSOR_ENC_ATTN_V, + VISION_TENSOR_ENC_ATTN_GATE, VISION_TENSOR_ENC_INPUT_NORM, VISION_TENSOR_ENC_OUTPUT, VISION_TENSOR_ENC_OUTPUT_NORM, @@ -711,6 +762,32 @@ enum vision_tensor { VISION_TENSOR_ENC_FFN_DOWN, VISION_TENSOR_PRE_NORM, VISION_TENSOR_POST_NORM, + + VISION_TENSOR_CA_MMPROJ, + VISION_TENSOR_ENC_FFN_GATE, + + VISION_TENSOR_ENC_GLOBAL_ATTN_Q, + VISION_TENSOR_ENC_GLOBAL_ATTN_K, + VISION_TENSOR_ENC_GLOBAL_ATTN_V, + VISION_TENSOR_ENC_GLOBAL_ATTN_GATE, + VISION_TENSOR_ENC_GLOBAL_FFN_GATE, + VISION_TENSOR_ENC_GLOBAL_INPUT_NORM, + VISION_TENSOR_ENC_GLOBAL_OUTPUT, + VISION_TENSOR_ENC_GLOBAL_OUTPUT_NORM, + VISION_TENSOR_ENC_GLOBAL_FFN_UP, + VISION_TENSOR_ENC_GLOBAL_FFN_DOWN, + VISION_TENSOR_ENC_LM_HEAD, + VISION_TENSOR_ENC_LM_NORM, + VISION_TENSOR_PRE_GLOBAL_NORM, + VISION_TENSOR_POST_GLOBAL_NORM, + VISION_TENSOR_TILE_POS_EMBD, + VISION_TENSOR_TILE_POS_GATE, + + VISION_TENSOR_POST_TILE_POS_EMBD, + VISION_TENSOR_POST_TILE_POS_GATE, + + VISION_TENSOR_PRE_TILE_POS_EMBD, + VISION_TENSOR_PRE_TILE_POS_GATE, }; static const std::map> LLM_TENSOR_NAMES = { @@ -1718,6 +1795,48 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_POS_NET_ATTN_K, "posnet.%d.attn_k" }, { LLM_TENSOR_POS_NET_ATTN_V, "posnet.%d.attn_v" }, { LLM_TENSOR_POS_NET_ATTN_OUT, "posnet.%d.attn_output" }, + } + }, + { + LLM_ARCH_MLLAMA, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ROPE_FREQS, "rope_freqs" }, + + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + + { LLM_TENSOR_CA_K_NORM, "blk.%d.cross_attn_k_norm" }, + { LLM_TENSOR_CA_Q_NORM, "blk.%d.cross_attn_q_norm" }, + + { LLM_TENSOR_CA_MLP_GATE, "blk.%d.cross_attn_mlp_gate" }, + { LLM_TENSOR_CA_ATTN_GATE, "blk.%d.cross_attn_attn_gate" }, + { LLM_TENSOR_CA_V_PROJ, "blk.%d.cross_attn_v_proj" }, + { LLM_TENSOR_CA_Q_PROJ, "blk.%d.cross_attn_q_proj" }, + { LLM_TENSOR_CA_O_PROJ, "blk.%d.cross_attn_o_proj" }, + { LLM_TENSOR_CA_K_PROJ, "blk.%d.cross_attn_k_proj" }, + + { LLM_TENSOR_CA_Q_PROJ, "blk.%d.cross_attn_q" }, + { LLM_TENSOR_CA_Q_PROJ, "blk.%d.cross_attn_q_proj" }, + + { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" }, + { LLM_TENSOR_FFN_GATE_EXP, "blk.%d.ffn_gate.%d" }, + { LLM_TENSOR_FFN_DOWN_EXP, "blk.%d.ffn_down.%d" }, + { LLM_TENSOR_FFN_UP_EXP, "blk.%d.ffn_up.%d" }, + { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" }, + { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" }, + { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" }, + { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" }, + + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, }, }, { @@ -1810,6 +1929,50 @@ static const std::map> VISION_ { VISION_TENSOR_PRE_NORM, "v.pre_norm" }, { VISION_TENSOR_POST_NORM, "v.post_norm" }, } + }, + { + VISION_ARCH_MLLAMA, + { + { VISION_TENSOR_ENC_EMBD_CLS, "v.enc.embd.cls" }, + { VISION_TENSOR_ENC_EMBD_PATCH, "v.enc.embd.patch" }, + + { VISION_TENSOR_ENC_EMBD_POS, "v.enc.embd.pos" }, + { VISION_TENSOR_ENC_EMBD_POS_GATE, "v.enc.embd.pos_gate" }, + + { VISION_TENSOR_PRE_NORM, "v.enc.pre_ln" }, + { VISION_TENSOR_POST_NORM, "v.enc.post_ln" }, + + {VISION_TENSOR_TILE_POS_EMBD, "v.enc.tile_pos_embd" }, + {VISION_TENSOR_TILE_POS_GATE, "v.enc.tile_pos_gate" }, + + { VISION_TENSOR_ENC_ATTN_Q, "v.enc.blk.%d.attn_q" }, + { VISION_TENSOR_ENC_ATTN_K, "v.enc.blk.%d.attn_k" }, + { VISION_TENSOR_ENC_ATTN_V, "v.enc.blk.%d.attn_v" }, + { VISION_TENSOR_ENC_INPUT_NORM, "v.enc.blk.%d.input_norm" }, + { VISION_TENSOR_ENC_OUTPUT, "v.enc.blk.%d.attn_out" }, + { VISION_TENSOR_ENC_OUTPUT_NORM, "v.enc.blk.%d.attn_out_norm" }, + { VISION_TENSOR_ENC_FFN_UP, "v.enc.blk.%d.ffn_up" }, + { VISION_TENSOR_ENC_FFN_DOWN, "v.enc.blk.%d.ffn_down" }, + { VISION_TENSOR_ENC_ATTN_GATE, "v.enc.blk.%d.attn_gate" }, + + { VISION_TENSOR_ENC_GLOBAL_ATTN_K, "v.enc.global.blk.%d.attn_k" }, + { VISION_TENSOR_ENC_GLOBAL_ATTN_Q, "v.enc.global.blk.%d.attn_q" }, + { VISION_TENSOR_ENC_GLOBAL_ATTN_V, "v.enc.global.blk.%d.attn_v" }, + { VISION_TENSOR_ENC_GLOBAL_ATTN_GATE, "v.enc.global.blk.%d.attn_gate" }, + { VISION_TENSOR_ENC_GLOBAL_FFN_GATE, "v.enc.global.blk.%d.ffn_gate" }, + { VISION_TENSOR_ENC_GLOBAL_INPUT_NORM, "v.enc.global.blk.%d.attn_norm" }, + { VISION_TENSOR_ENC_GLOBAL_OUTPUT, "v.enc.global.blk.%d.attn_out" }, + { VISION_TENSOR_ENC_GLOBAL_OUTPUT_NORM, "v.enc.global.blk.%d.post_attn_norm" }, + { VISION_TENSOR_ENC_GLOBAL_FFN_UP, "v.enc.global.blk.%d.ffn_up" }, + { VISION_TENSOR_ENC_GLOBAL_FFN_DOWN, "v.enc.global.blk.%d.ffn_down" }, + + { VISION_TENSOR_POST_TILE_POS_EMBD, "v.enc.post_tile_pos_embd" }, + { VISION_TENSOR_POST_TILE_POS_GATE, "v.enc.post_tile_pos_gate" }, + { VISION_TENSOR_PRE_TILE_POS_EMBD, "v.enc.pre_tile_pos_embd" }, + { VISION_TENSOR_PRE_TILE_POS_GATE, "v.enc.pre_tile_pos_gate" }, + { VISION_TENSOR_CA_MMPROJ, "v.enc.mmproj" }, + + } } }; @@ -2647,6 +2810,7 @@ enum e_model { MODEL_40B, MODEL_65B, MODEL_70B, + MODEL_90B, MODEL_236B, MODEL_314B, MODEL_SMALL, @@ -2704,6 +2868,7 @@ struct llama_hparams { std::array n_head_arr; std::array n_head_kv_arr; std::array n_ff_arr; + std::array cross_attn_layers; uint32_t n_layer_dense_lead = 0; uint32_t n_lora_q = 0; @@ -2759,6 +2924,9 @@ struct llama_hparams { // ref: https://github.com/ggerganov/llama.cpp/pull/8141 llama_token dec_start_token_id = LLAMA_TOKEN_NULL; + // For vision models like MLlama + uint32_t image_token_index = 0; + enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE; enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE; enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE; @@ -2834,6 +3002,10 @@ struct llama_hparams { // corresponds to Mamba's ssm_states size return ssm_d_state * ssm_d_inner; } + + bool cross_attention_layers(uint32_t il) const { + return std::find(cross_attn_layers.begin(), cross_attn_layers.end(), il) != cross_attn_layers.end(); + } }; static_assert(std::is_trivially_copyable::value, "llama_hparams must be trivially copyable"); @@ -2863,6 +3035,8 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool no_perf; + bool cross_attn = false; + uint32_t n_tiles = 0; enum llama_pooling_type pooling_type; @@ -3069,6 +3243,16 @@ struct llama_layer { struct llama_layer_posnet posnet; struct llama_layer_convnext convnext; + + // Cross Attention + struct ggml_tensor * ca_k_norm; + struct ggml_tensor * ca_k_proj; + struct ggml_tensor * ca_q_norm; + struct ggml_tensor * ca_q_proj; + struct ggml_tensor * ca_v_proj; + struct ggml_tensor * ca_attn_gate; + struct ggml_tensor * ca_mlp_gate; + struct ggml_tensor * ca_o_proj; }; // very similar to llama_batch, @@ -3083,6 +3267,7 @@ struct llama_ubatch { llama_token * token; // [n_tokens] float * embd; // [n_embd, n_tokens] + uint32_t n_embd; llama_pos * pos; // [n_tokens] int32_t * n_seq_id; // [n_seqs] llama_seq_id ** seq_id; // [n_seqs] @@ -3208,6 +3393,7 @@ struct llama_model { std::unordered_map gguf_kv; bool has_vision = false; clip_vision_model clip; + ca_vision_model ca_vision; llama_split_mode split_mode; int main_gpu; @@ -3323,6 +3509,7 @@ struct llama_sbatch { /*n_seqs =*/ 0, /*token =*/ !has_embd ? ubatch_token.data() : nullptr, /*embd =*/ has_embd ? ubatch_embd.data() : nullptr, + /*n_embd =*/ has_embd ? (uint32_t) n_embd : 0, /*pos =*/ ubatch_pos.data(), /*n_seq_id =*/ ubatch_n_seq_id.data(), /*seq_id =*/ ubatch_seq_id.data(), @@ -3667,8 +3854,14 @@ struct llama_context { struct ggml_tensor * inp_embd_enc; // F32 [n_embd, n_outputs_enc] struct ggml_tensor * inp_KQ_mask_cross; // F32 [n_outputs_enc, n_batch] + // Cross Attention Patch Embeddings + struct ggml_tensor * ca_patch_embd; + struct ggml_tensor * ca_softmax_mask; + // vision clip_context clip; + // cross attention vision + ca_context ca_vision; }; struct llama_lora_weight { @@ -3839,12 +4032,23 @@ static bool llama_kv_cache_init( return false; } - ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*kv_size); - ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*kv_size); - ggml_format_name(k, "cache_k_l%d", i); - ggml_format_name(v, "cache_v_l%d", i); - cache.k_l.push_back(k); - cache.v_l.push_back(v); + // for cross attention layers + if (model.arch == LLM_ARCH_MLLAMA && hparams.cross_attention_layers(i)) { + ggml_tensor * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, hparams.n_embd_head_k, 6404, hparams.n_head_kv(i)); + ggml_tensor * v = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, hparams.n_embd_head_v, 6404, hparams.n_head_kv(i)); + ggml_format_name(k, "cache_k_l%d", i); + ggml_format_name(v, "cache_v_l%d", i); + cache.k_l.push_back(k); + cache.v_l.push_back(v); + continue; + } else { + ggml_tensor * k = ggml_new_tensor_1d(ctx, type_k, n_embd_k_gqa*kv_size); + ggml_tensor * v = ggml_new_tensor_1d(ctx, type_v, n_embd_v_gqa*kv_size); + ggml_format_name(k, "cache_k_l%d", i); + ggml_format_name(v, "cache_v_l%d", i); + cache.k_l.push_back(k); + cache.v_l.push_back(v); + } } // allocate tensors and initialize the buffers to avoid NaNs in the padding @@ -5775,6 +5979,10 @@ static void llm_load_hparams( ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false); ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false); + + // Multi-Modal cross attention + std::fill(hparams.cross_attn_layers.begin(), hparams.cross_attn_layers.end(), -1); + ml.get_arr(LLM_KV_VISION_CROSS_ATTENTION_LAYERS, hparams.cross_attn_layers, false); // n_head_kv is optional, default to n_head hparams.n_head_kv_arr = hparams.n_head_arr; @@ -5823,7 +6031,7 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot, false); - if (model.arch == LLM_ARCH_LLAMA || model.arch == LLM_ARCH_FALCON) { + if (model.arch == LLM_ARCH_LLAMA || model.arch == LLM_ARCH_FALCON || model.arch == LLM_ARCH_MLLAMA) { if (hparams.n_rot != hparams.n_embd_head_k) { throw std::runtime_error(format("invalid n_rot: %u, expected %u", hparams.n_rot, hparams.n_embd_head_k)); } @@ -6525,16 +6733,37 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS, hparams.f_norm_group_eps); ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups); ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); + } break; + case LLM_ARCH_MLLAMA: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_arr(LLM_KV_VISION_CROSS_ATTENTION_LAYERS, hparams.cross_attn_layers); + //ml.get_key(LLM_KV_VISION_IMAGE_TOKEN_INDEX, hparams.image_token_index); + + hparams.n_embd_head_k = hparams.n_embd / hparams.n_head(); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false); + + hparams.n_embd_head_v = hparams.n_embd / hparams.n_head(); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false); + + // sanity check for n_rot (optional) + hparams.n_rot = hparams.n_embd_head_k; + + switch (hparams.n_layer) { + case 40: model.type = e_model::MODEL_11B; break; + case 100: model.type = e_model::MODEL_90B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } } break; default: (void)0; } // vision model - auto & vparams = model.clip.hparams; std::string vision_type; ml.get_key(LLM_KV_VISION_TYPE, vision_type, false); if (vision_type == "clip-vit") { model.has_vision = true; + auto & vparams = model.clip.hparams; ml.get_key(LLM_KV_VISION_IMAGE_SIZE, vparams.image_size, true); ml.get_key(LLM_KV_VISION_PATCH_SIZE, vparams.patch_size, true); ml.get_key_or_arr(LLM_KV_VISION_IMAGE_MEAN, vparams.image_mean, 3, true); @@ -6568,18 +6797,48 @@ static void llm_load_hparams( } } } + // arch-specific CLIP hparams + switch (vparams.arch) { + case VISION_ARCH_LLAVA: + { + ml.get_key(LLM_KV_VISION_CLIP_MAX_POS_EMBD, vparams.max_pos_embd, true); + } break; + default: (void)0; + } + } else if (vision_type == "cross-attn") { + model.has_vision = true; + auto & vparams = model.ca_vision.hparams; + + ml.get_key(LLM_KV_VISION_IMAGE_SIZE, vparams.image_size, true); + ml.get_key(LLM_KV_VISION_PATCH_SIZE, vparams.patch_size, true); + ml.get_key(LLM_KV_VISION_EMBEDDING_LENGTH, vparams.hidden_size, true); + ml.get_key(LLM_KV_VISION_FEED_FORWARD_LENGTH, vparams.n_intermediate, true); + ml.get_key(LLM_KV_VISION_CLIP_PROJECTION_DIM, vparams.projection_dim, true); + ml.get_key(LLM_KV_VISION_HEAD_COUNT, vparams.n_head, true); + ml.get_key(LLM_KV_VISION_BLOCK_COUNT, vparams.n_layer, true); + ml.get_key(LLM_KV_VISION_CLIP_MAX_POS_EMBD, vparams.max_pos_embd, true); + ml.get_key(LLM_KV_VISION_GLOBAL_BLOCK_COUNT, vparams.n_global_layer, true); + ml.get_key(LLM_KV_VISION_TILES, vparams.n_tiles, true); + ml.get_key_or_arr(LLM_KV_VISION_IMAGE_MEAN, vparams.image_mean, 3, true); + ml.get_key_or_arr(LLM_KV_VISION_IMAGE_STD, vparams.image_std, 3, true); + ml.get_key(LLM_KV_VISION_LAYERNORM_RMS_EPS, vparams.eps, true); + ml.get_key_or_arr(LLM_KV_VISION_ASPECT_RATIOS, vparams.aspect_ratios, 16, true); + ml.get_key(LLM_KV_EMBEDDING_LENGTH, vparams.proj_n_embd, true); + ml.get_key_or_arr(LLM_KV_VISION_INTERMEDIATE_LAYERS,vparams.intermediate_layers, 5, true); + { + std::string arch; + ml.get_key(LLM_KV_VISION_ARCHITECTURE, arch, true); + for (auto & it : VISION_ARCH_NAMES) { + if (arch == it.second) { + vparams.arch = it.first; + break; + } + } + } } else if (!vision_type.empty()) { throw std::runtime_error(format("unsupported vision type: %s", vision_type.c_str())); } - // arch-specific CLIP hparams - switch (vparams.arch) { - case VISION_ARCH_LLAVA: - { - ml.get_key(LLM_KV_VISION_CLIP_MAX_POS_EMBD, vparams.max_pos_embd, true); - } break; - default: (void)0; - } model.ftype = ml.ftype; @@ -6801,6 +7060,13 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "smaug-bpe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG; + vocab.tokenizer_ignore_merges = false; + vocab.tokenizer_clean_spaces = true; + } else if ( + tokenizer_pre == "mllama") { + printf("using mllama\n"); + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_MLLAMA; + vocab.tokenizer_ignore_merges = false; } else if ( tokenizer_pre == "poro-chat") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_PORO; @@ -6973,12 +7239,14 @@ static void llm_load_vocab( { LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id }, { LLM_KV_TOKENIZER_CLS_ID, vocab.special_cls_id }, { LLM_KV_TOKENIZER_MASK_ID, vocab.special_mask_id }, + { LLM_KV_TOKENIZER_IMAGE_ID, vocab.special_image_id }, { LLM_KV_TOKENIZER_FIM_PRE_ID, vocab.special_fim_pre_id }, { LLM_KV_TOKENIZER_FIM_SUF_ID, vocab.special_fim_suf_id }, { LLM_KV_TOKENIZER_FIM_MID_ID, vocab.special_fim_mid_id }, { LLM_KV_TOKENIZER_FIM_PAD_ID, vocab.special_fim_pad_id }, { LLM_KV_TOKENIZER_FIM_REP_ID, vocab.special_fim_rep_id }, { LLM_KV_TOKENIZER_FIM_SEP_ID, vocab.special_fim_sep_id }, + { LLM_KV_TOKENIZER_FIM_SEP_ID, vocab.special_fim_sep_id }, // deprecated { LLM_KV_TOKENIZER_PREFIX_ID, vocab.special_fim_pre_id }, @@ -6998,6 +7266,7 @@ static void llm_load_vocab( LLAMA_LOG_WARN("%s: bad special token: '%s' = %ud, using default id %d\n", __func__, key.c_str(), new_id, id); } else { + printf("special token: '%s' = %d\n", key.c_str(), new_id); id = new_id; } } @@ -7178,6 +7447,7 @@ static void llm_load_vocab( || t.first == "<|endoftext|>" || t.first == "<|eom_id|>" || t.first == "" + || t.first == "<|end_of_text|>" ) { vocab.special_eog_ids.insert(t.second); if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { @@ -7194,6 +7464,7 @@ static void llm_load_vocab( } } + // sanity checks if (vocab.special_eos_id != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(vocab.special_eos_id) == 0) { vocab.special_eog_ids.insert(vocab.special_eos_id); @@ -7603,24 +7874,59 @@ static const std::map llm_tensor_info_mapping = { {LLM_TENSOR_CONVNEXT_PW1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_CONVNEXT_PW2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_CONVNEXT_GAMMA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + // MLLama tensor mappings + {LLM_TENSOR_CA_K_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_CA_K_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CA_O_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CA_Q_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_CA_Q_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CA_V_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CA_ATTN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_CA_MLP_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_CA_FFN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CA_FFN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, }; static const std::map vision_tensor_info_mapping = { - {VISION_TENSOR_MMPROJ_A, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, - {VISION_TENSOR_MMPROJ_B, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {VISION_TENSOR_MMPROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_NONE}}, {VISION_TENSOR_ENC_EMBD_CLS, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, - {VISION_TENSOR_ENC_EMBD_PATCH, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {VISION_TENSOR_ENC_EMBD_PATCH, {LLM_TENSOR_LAYER_INPUT, GGML_OP_CONV_TRANSPOSE_2D}}, {VISION_TENSOR_ENC_EMBD_POS, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {VISION_TENSOR_ENC_EMBD_POS_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, {VISION_TENSOR_ENC_ATTN_Q, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {VISION_TENSOR_ENC_ATTN_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {VISION_TENSOR_ENC_ATTN_V, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, - {VISION_TENSOR_ENC_INPUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_INPUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {VISION_TENSOR_ENC_OUTPUT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, - {VISION_TENSOR_ENC_OUTPUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_OUTPUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {VISION_TENSOR_ENC_FFN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {VISION_TENSOR_ENC_FFN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, - {VISION_TENSOR_PRE_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {VISION_TENSOR_POST_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_PRE_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {VISION_TENSOR_POST_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + + {VISION_TENSOR_CA_MMPROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_ATTN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {VISION_TENSOR_ENC_FFN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + + {VISION_TENSOR_ENC_GLOBAL_ATTN_Q, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_GLOBAL_ATTN_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_GLOBAL_ATTN_V, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_GLOBAL_ATTN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_GLOBAL_FFN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_GLOBAL_INPUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {VISION_TENSOR_ENC_GLOBAL_OUTPUT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {VISION_TENSOR_ENC_GLOBAL_OUTPUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {VISION_TENSOR_ENC_GLOBAL_FFN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {VISION_TENSOR_ENC_GLOBAL_FFN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + + {VISION_TENSOR_TILE_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {VISION_TENSOR_TILE_POS_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + + {VISION_TENSOR_POST_TILE_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {VISION_TENSOR_POST_TILE_POS_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + + {VISION_TENSOR_PRE_TILE_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, + {VISION_TENSOR_PRE_TILE_POS_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, }; // checks if the weight tensor can be used with the specified buffer type and device @@ -7652,7 +7958,7 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w } break; case GGML_OP_MUL_MAT: { - ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]); + ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 4096, w->ne[2], w->ne[3]); op_tensor = ggml_mul_mat(ctx, w, b); } break; case GGML_OP_MUL_MAT_ID: @@ -7730,6 +8036,17 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w const int n_embd = hparams.n_embd; ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd, w->ne[1], 1, 1); op_tensor = ggml_im2col(ctx, w, b, 1, 0, 0, 0, 1, 0, false, GGML_TYPE_F16); + } break; + case GGML_OP_CONV_TRANSPOSE_2D: + { + int stride_x = 1; + int stride_y = 1; + int pad_x = 0; + int pad_y = 0; + int dil_x = 1; + int dil_y = 1; + struct ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 4096, w->ne[2], w->ne[3]); + op_tensor = ggml_conv_2d(ctx, w, b, stride_x, stride_y, pad_x, pad_y, dil_x, dil_y); } break; default: GGML_ABORT("%s: missing test for op %s for tensor %s", __func__, ggml_op_name(op), w->name); @@ -9798,6 +10115,67 @@ static bool llm_load_tensors( model.output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hparams.convnext.n_embd, n_embd}, 0); model.output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"), {n_embd}, 0); + } break; + case LLM_ARCH_MLLAMA: + { + // The actual shape of this tensor in model-00001-of-00005.safetensors is: + // language_model.model.embed_tokens.weight: torch.Size([128264, 4096]) + // + // There are 8 special characters: + // <|begin_of_text|> + // <|end_of_text|> + // <|finetune_right_pad_id|> + // <|step_id|> + // <|start_header_id|> + // <|end_header_id|> + // <|python_tag|> + // <|image|> + // + // Which I think is the reason for these additional tokens in this tensor. + model.tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab + 8}, 0); + + { + model.output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + model.output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); + + // if output is NULL, init from the input tok embed + if (model.output == NULL) { + model.output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); + } + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + if (hparams.cross_attention_layers(i)) { + layer.ca_q_norm = create_tensor(tn(LLM_TENSOR_CA_Q_NORM, "weight", i), {n_embd_head_k}, 0); + layer.ca_q_proj = create_tensor(tn(LLM_TENSOR_CA_Q_PROJ, "weight", i), {n_embd, n_embd}, 0); + + layer.ca_k_norm = create_tensor(tn(LLM_TENSOR_CA_K_NORM, "weight", i), {n_embd_head_k}, 0); + layer.ca_k_proj = create_tensor(tn(LLM_TENSOR_CA_K_PROJ, "weight", i), {n_embd, n_embd_k_gqa}, 0); + + layer.ca_v_proj = create_tensor(tn(LLM_TENSOR_CA_V_PROJ, "weight", i), {n_embd, n_embd_v_gqa}, 0); + + layer.ca_o_proj = create_tensor(tn(LLM_TENSOR_CA_O_PROJ, "weight", i), {n_embd, n_embd}, 0); + + layer.ca_attn_gate = create_tensor(tn(LLM_TENSOR_CA_ATTN_GATE, i), {1}, 0); + + layer.ca_mlp_gate = create_tensor(tn(LLM_TENSOR_CA_MLP_GATE, i), {1}, 0); + } else { + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0)); + + } + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff , n_embd}, 0); + } } break; default: throw std::runtime_error("unknown llm architecture"); @@ -9811,20 +10189,17 @@ static bool llm_load_tensors( // create tensors for the vision weights { // load tensors for vision model - auto & vparams = model.clip.hparams; if (model.has_vision) { - const int64_t n_layer = vparams.n_layer; - const int64_t n_embd = vparams.hidden_size; - const int64_t n_ff = vparams.n_intermediate; - const int64_t max_pos_embd = vparams.max_pos_embd; - const int64_t n_channel = 3; // always RGB - const int64_t patch_size = vparams.patch_size; int n_moved_tensors = 0; ggml_tensor * first_moved_tensor = nullptr; ggml_backend_buffer_type_t first_moved_from_buft = nullptr; ggml_backend_buffer_type_t first_moved_to_buft = nullptr; - const auto tn = VISION_TN(vparams.arch); + const auto arch = model.arch == LLM_ARCH_MLLAMA ? VISION_ARCH_MLLAMA : VISION_ARCH_LLAVA; + const auto tn = VISION_TN(arch); + // TODO(danbev) This is duplication of the above create_tensor + // lambda but it should be possible when C++17 can be used to make + // 'tn' generic and just have one lambda. auto create_tensor = [&](const VISION_TN_IMPL & tn, const std::initializer_list & ne, int flags) -> ggml_tensor * { ggml_tensor * t_meta = ml.get_tensor_meta(tn.str().c_str()); @@ -9886,7 +10261,7 @@ static bool llm_load_tensors( // avoid using a host buffer when using mmap auto * buft_dev = ggml_backend_buft_get_device(buft); - if (ml.use_mmap && buft == ggml_backend_dev_host_buffer_type(buft_dev)) { + if (ml.use_mmap && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) { auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); buft = ggml_backend_dev_buffer_type(cpu_dev); } @@ -9911,31 +10286,34 @@ static bool llm_load_tensors( } return ml.create_tensor(ctx, tn, ne, flags); }; - - //ggml_context * ctx_vision = ctx_map.at(model.buft_input.buft); // TODO: make dedicated buft for vision - //auto ctx_for_layer = [&](int i) { return ctx_map.at(model.buft_layer[i].buft); }; - - model.clip.layers.resize(n_layer); - - switch (vparams.arch) { + + switch (arch) { case VISION_ARCH_LLAVA: { - model.clip.mm_a_w = create_tensor(tn(VISION_TENSOR_MMPROJ, "weight", 1), {n_embd, n_ff}, 0); - model.clip.mm_a_b = create_tensor(tn(VISION_TENSOR_MMPROJ, "bias" , 1), {n_ff}, 0); - model.clip.mm_b_w = create_tensor(tn(VISION_TENSOR_MMPROJ, "weight", 2), {n_ff, n_ff}, 0); - model.clip.mm_b_b = create_tensor(tn(VISION_TENSOR_MMPROJ, "bias" , 2), {n_ff}, 0); + auto & vparams = model.clip.hparams; + const int64_t n_layer = vparams.n_layer; + const int64_t n_embd = vparams.hidden_size; + const int64_t n_ff = vparams.n_intermediate; + const int64_t max_pos_embd = vparams.max_pos_embd; + const int64_t n_channel = 3; // always RGB + const int64_t patch_size = vparams.patch_size; + model.clip.layers.resize(n_layer); + + model.clip.mm_1_w = create_tensor(tn(VISION_TENSOR_MMPROJ, "weight", 1), {n_embd, n_ff}, 0); + model.clip.mm_1_b = create_tensor(tn(VISION_TENSOR_MMPROJ, "bias" , 1), {n_ff}, 0); + model.clip.mm_2_w = create_tensor(tn(VISION_TENSOR_MMPROJ, "weight", 2), {n_ff, n_ff}, 0); + model.clip.mm_2_b = create_tensor(tn(VISION_TENSOR_MMPROJ, "bias" , 2), {n_ff}, 0); model.clip.class_embedding = create_tensor(tn(VISION_TENSOR_ENC_EMBD_CLS ), {n_embd}, 0); model.clip.patch_embeddings = create_tensor(tn(VISION_TENSOR_ENC_EMBD_PATCH, "weight"), {patch_size, patch_size, n_channel, n_embd}, 0); model.clip.position_embeddings = create_tensor(tn(VISION_TENSOR_ENC_EMBD_POS, "weight"), {n_embd, max_pos_embd}, 0); model.clip.pre_norm_w = create_tensor(tn(VISION_TENSOR_PRE_NORM, "weight"), {n_embd}, 0); - model.clip.pre_norm_w = create_tensor(tn(VISION_TENSOR_PRE_NORM, "bias" ), {n_embd}, 0); - model.clip.post_norm_w = create_tensor(tn(VISION_TENSOR_POST_NORM, "weight"), {n_embd}, 0); - model.clip.post_norm_w = create_tensor(tn(VISION_TENSOR_POST_NORM, "bias" ), {n_embd}, 0); + model.clip.pre_norm_b = create_tensor(tn(VISION_TENSOR_PRE_NORM, "bias" ), {n_embd}, 0); + model.clip.post_norm_w = create_tensor(tn(VISION_TENSOR_POST_NORM, "weight"), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + model.clip.post_norm_b = create_tensor(tn(VISION_TENSOR_POST_NORM, "bias" ), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); for (int i = 0; i < n_layer; ++i) { - //ggml_context * ctx_layer = ctx_for_layer(i); auto & layer = model.clip.layers[i]; layer.k_w = create_tensor(tn(VISION_TENSOR_ENC_ATTN_K, "weight", i), {n_embd, n_embd}, 0); @@ -9959,12 +10337,116 @@ static bool llm_load_tensors( layer.output_b = create_tensor(tn(VISION_TENSOR_ENC_OUTPUT, "bias" , i), {n_embd}, 0); } } break; + case VISION_ARCH_MLLAMA: + { + auto & vparams = model.ca_vision.hparams; + + const int64_t n_embd = vparams.hidden_size; + const int64_t n_layer = vparams.n_layer; + const int64_t n_global_layer = vparams.n_global_layer; + const int64_t max_pos_embd = vparams.max_pos_embd; + const int64_t n_channel = 3; // RGB + const int64_t patch_size = vparams.patch_size; + const int64_t n_proj_dim = vparams.projection_dim; + const int64_t n_intermediate = vparams.n_intermediate; + const int64_t v_n_embd = vparams.hidden_size; + model.ca_vision.global_layers.resize(n_global_layer); + model.ca_vision.layers.resize(n_layer); + + model.ca_vision.class_embedding = create_tensor(tn(VISION_TENSOR_ENC_EMBD_CLS ), {n_embd}, 0); + model.ca_vision.patch_embeddings = create_tensor(tn(VISION_TENSOR_ENC_EMBD_PATCH, "weight"), {patch_size, patch_size, n_channel, n_embd}, 0); + + model.ca_vision.position_embeddings = create_tensor(tn(VISION_TENSOR_ENC_EMBD_POS ), {n_embd, max_pos_embd}, 0); + model.ca_vision.position_embeddings_gate = create_tensor(tn(VISION_TENSOR_ENC_EMBD_POS_GATE ), {1, 1}, 0); + + model.ca_vision.pre_norm_w = create_tensor(tn(VISION_TENSOR_PRE_NORM, "weight"), {n_embd}, 0); + model.ca_vision.pre_norm_b = create_tensor(tn(VISION_TENSOR_PRE_NORM, "bias" ), {n_embd}, 0); + model.ca_vision.post_norm_w = create_tensor(tn(VISION_TENSOR_POST_NORM, "weight"), {n_embd}, 0); + model.ca_vision.post_norm_b = create_tensor(tn(VISION_TENSOR_POST_NORM, "bias" ), {n_embd}, 0); + + model.ca_vision.tile_position_embeddings = create_tensor(tn(VISION_TENSOR_TILE_POS_EMBD, "weight"), {max_pos_embd*n_intermediate, 9}, 0); + model.ca_vision.tile_position_embeddings_gate = create_tensor(tn(VISION_TENSOR_TILE_POS_GATE ), {1, 1}, 0); + + model.ca_vision.pre_tile_position_embeddings = create_tensor(tn(VISION_TENSOR_PRE_TILE_POS_EMBD, "weight"), {n_intermediate, 9}, 0); + model.ca_vision.pre_tile_position_embeddings_gate = create_tensor(tn(VISION_TENSOR_PRE_TILE_POS_GATE ), {1, 1}, 0); + + model.ca_vision.post_tile_position_embeddings = create_tensor(tn(VISION_TENSOR_POST_TILE_POS_EMBD, "weight"), {n_intermediate, 9}, 0); + model.ca_vision.post_tile_position_embeddings_gate = create_tensor(tn(VISION_TENSOR_POST_TILE_POS_GATE ), {1}, 0); + + model.ca_vision.mm_1_w = create_tensor(tn(VISION_TENSOR_CA_MMPROJ, "weight"), {n_proj_dim, model.hparams.n_embd}, 0); + model.ca_vision.mm_1_b = create_tensor(tn(VISION_TENSOR_CA_MMPROJ, "bias" ), {model.hparams.n_embd}, 0); + + for (int i = 0; i < n_global_layer; i++) { + auto & layer = model.ca_vision.global_layers[i]; + + layer.norm_in_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_INPUT_NORM, "weight", i), {n_embd}, 0); + layer.norm_in_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_INPUT_NORM, "bias" , i), {n_embd}, 0); + + layer.k_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_ATTN_K, "weight", i), {n_embd, n_embd}, 0); + layer.k_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_ATTN_K, "bias" , i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.q_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_ATTN_Q, "weight", i), {n_embd, n_embd}, 0); + layer.q_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_ATTN_Q, "bias" , i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.v_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_ATTN_V, "weight", i), {n_embd, n_embd}, 0); + layer.v_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_ATTN_V, "bias" , i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.attn_gate = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_ATTN_GATE , i), {1}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.ffn_up_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_FFN_UP, "weight", i), {n_embd, n_intermediate}, 0); + layer.ffn_up_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_FFN_UP, "bias" , i), {n_intermediate}, 0); + + layer.ffn_gate = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_FFN_GATE , i), {1}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.ffn_down_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_FFN_DOWN, "weight", i), {n_intermediate, v_n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_FFN_DOWN, "bias" , i), {n_embd}, 0); + + layer.output_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_OUTPUT, "weight", i), {n_embd, n_embd}, 0); + layer.output_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_OUTPUT, "bias" , i), {n_embd, n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.norm_out_w = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_OUTPUT_NORM, "weight", i), {n_embd}, 0); + layer.norm_out_b = create_tensor(tn(VISION_TENSOR_ENC_GLOBAL_OUTPUT_NORM, "bias", i), {n_embd}, 0); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.ca_vision.layers[i]; + + layer.norm_in_w = create_tensor(tn(VISION_TENSOR_ENC_INPUT_NORM, "weight", i), {n_embd}, 0); + layer.norm_in_b = create_tensor(tn(VISION_TENSOR_ENC_INPUT_NORM, "bias" , i), {n_embd}, 0); + + layer.k_w = create_tensor(tn(VISION_TENSOR_ENC_ATTN_K, "weight", i), {n_embd, n_embd}, 0); + layer.k_b = create_tensor(tn(VISION_TENSOR_ENC_ATTN_K, "bias" , i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.q_w = create_tensor(tn(VISION_TENSOR_ENC_ATTN_Q, "weight", i), {n_embd, n_embd}, 0); + layer.q_b = create_tensor(tn(VISION_TENSOR_ENC_ATTN_Q, "bias" , i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.v_w = create_tensor(tn(VISION_TENSOR_ENC_ATTN_V, "weight", i), {n_embd, n_embd}, 0); + layer.v_b = create_tensor(tn(VISION_TENSOR_ENC_ATTN_V, "bias" , i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.attn_gate = create_tensor(tn(VISION_TENSOR_ENC_ATTN_GATE , i), {1}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.ffn_up_w = create_tensor(tn(VISION_TENSOR_ENC_FFN_UP, "weight", i), {n_embd, n_intermediate}, 0); + layer.ffn_up_b = create_tensor(tn(VISION_TENSOR_ENC_FFN_UP, "bias" , i), {n_intermediate}, 0); + + layer.ffn_gate = create_tensor(tn(VISION_TENSOR_ENC_FFN_GATE , i), {1}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.ffn_down_w = create_tensor(tn(VISION_TENSOR_ENC_FFN_DOWN, "weight", i), {n_intermediate, v_n_embd}, 0); + layer.ffn_down_b = create_tensor(tn(VISION_TENSOR_ENC_FFN_DOWN, "bias" , i), {n_embd}, 0); + + layer.output_w = create_tensor(tn(VISION_TENSOR_ENC_OUTPUT, "weight", i), {n_embd, n_embd}, 0); + layer.output_b = create_tensor(tn(VISION_TENSOR_ENC_OUTPUT, "bias" , i), {n_embd, n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.norm_out_w = create_tensor(tn(VISION_TENSOR_ENC_OUTPUT_NORM, "weight", i), {n_embd}, 0); + layer.norm_out_b = create_tensor(tn(VISION_TENSOR_ENC_OUTPUT_NORM, "bias", i), {n_embd}, 0); + } + } break; default: throw std::runtime_error("unknown vision architecture"); } } } + ml.done_getting_tensors(); ml.init_mappings(true, use_mlock ? &model.mlock_mmaps : nullptr); @@ -10125,7 +10607,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam if (model.vocab.type != LLAMA_VOCAB_TYPE_NONE && model.hparams.n_vocab != model.vocab.id_to_token.size()) { - throw std::runtime_error("vocab size mismatch"); + //throw std::runtime_error("vocab size mismatch"); } if (params.vocab_only) { @@ -10209,6 +10691,24 @@ static struct ggml_tensor * llm_build_inp_embd( return inpL; } +static struct ggml_tensor * llm_build_ca_patch_embd( + struct ggml_context * ctx, + struct llama_context & lctx, + const llama_hparams & hparams, + const llm_build_cb & cb) { + const int64_t n_embd = hparams.n_embd; + + // Cross Attention Patch embeddings. + // 1601 is the number of positions per frame, plus one for the CLS token. + // 4 is the number of tiles. 1601 * 4 = 6404 + struct ggml_tensor * ca_patch_embd = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, 1601, 4); + cb(ca_patch_embd, "ca_patch_embd", -1); + ggml_set_input(ca_patch_embd); + lctx.ca_patch_embd = ca_patch_embd; + + return ca_patch_embd; +} + static void llm_build_kv_store( struct ggml_context * ctx, const llama_hparams & hparams, @@ -11182,6 +11682,8 @@ struct llm_build_context { lctx.inp_pos_bucket = nullptr; lctx.inp_embd_enc = nullptr; lctx.inp_KQ_mask_cross = nullptr; + + lctx.ca_patch_embd = nullptr; } void free() { @@ -17687,6 +18189,248 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_mllama() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false); + + // mutable variable, needed during the last layer of the computation to skip unused tokens + int32_t n_tokens = this->n_tokens; + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + //printf("n_embd: %ld, n_embd_head: %ld, k_embd_head: %ld\n", n_embd, n_embd_head, hparams.n_embd_head_k); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + // Cross attention Patch Embeddings + struct ggml_tensor * ca_patch_embd; + + inpL = llm_build_inp_embd(ctx0, lctx, hparams, ubatch, model.tok_embd, cb); + + // ca_patch_embd contains the patch embeddings from the vision encoder. + ca_patch_embd = llm_build_ca_patch_embd(ctx0, lctx, hparams, cb); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = build_inp_pos(); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); + + const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + if (hparams.cross_attention_layers(il)) { + if (!ubatch.embd && !cparams.cross_attn) { + continue; + } + // cross attention layer + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].ca_q_proj, cur); + cb(Qcur, "Qcur", il); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + cb(Qcur, "Qcur", il); + Qcur = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 0, 2, 1, 3)); + cb(Qcur, "Qcur", il); + Qcur = llm_build_norm(ctx0, Qcur, hparams, model.layers[il].ca_q_norm, NULL, LLM_NORM_RMS, cb, il); + cb(Qcur, "Qcur", il); + + struct ggml_tensor * Kcur; + struct ggml_tensor * Vcur; + if (ubatch.embd) { + // The image patch embeddings will be set by llama_set_inputs before computing the graph. + //printf("--------- use ca_patch_embd for K and V and store in kv_cache.layer[%d] ------ \n", il); + Kcur = ggml_mul_mat(ctx0, model.layers[il].ca_k_proj, ca_patch_embd); + cb(Kcur, "Kcur", il); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, 6404); + cb(Kcur, "Kcur", il); + Kcur = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3)); + cb(Kcur, "Kcur", il); + Kcur = llm_build_norm(ctx0, Kcur, hparams, model.layers[il].ca_k_norm, NULL, LLM_NORM_RMS, cb, il); + cb(Kcur, "Kcur", il); + + // Copy K into the KV cache. + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, kv_self.k_l[il])); + + Vcur = ggml_mul_mat(ctx0, model.layers[il].ca_v_proj, ca_patch_embd); + cb(Vcur, "Vcur", il); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, 6404); + cb(Vcur, "Vcur", il); + Vcur = ggml_permute(ctx0, Vcur, 0, 2, 1, 3); + cb(Vcur, "Vcur", il); + + // Copy V into the KV cache. + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, kv_self.v_l[il])); + } else { + // Use KV-Cache values for K and V. + //printf("--------- use values from KV cache for kv_cache.layer[%d] ------ \n", il); + Kcur = ggml_dup_tensor(ctx0, kv_self.k_l[il]); + cb(Kcur, "Kcur", il); + ggml_set_input(Kcur); + + Vcur = ggml_dup_tensor(ctx0, kv_self.v_l[il]); + cb(Vcur, "Vcur", il); + ggml_set_input(Vcur); + } + + struct ggml_tensor * kq = ggml_mul_mat(ctx0, Kcur, Qcur); + cb(kq, "kq", il); + + // TODO(danbev): what should the mask be? + //struct ggml_tensor * kq_soft_max = ggml_soft_max_ext(ctx0, kq, softmax_mask, 1.f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); + struct ggml_tensor * kq_soft_max = ggml_soft_max_ext(ctx0, kq, nullptr, 1.f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); + cb(kq_soft_max, "kq_soft_max", il); + + Vcur = ggml_cont(ctx0, ggml_transpose(ctx0, Vcur)); + cb(Vcur, "Vcur", il); + + struct ggml_tensor * kqv = ggml_mul_mat(ctx0, Vcur, kq_soft_max); + cb(kqv, "kqv", il); + + struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3); + cb(kqv_merged, "kqv_merged", il); + + cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_head_v*n_head, n_tokens); + cb(cur, "kqv_merged_cont", il); + + cur = ggml_mul_mat(ctx0, model.layers[il].ca_o_proj, cur); + cb(cur, "cur", il); + + cur = ggml_mul(ctx0, cur, ggml_tanh(ctx0, model.layers[il].ca_attn_gate)); + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, lctx, cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, ggml_tanh(ctx0, model.layers[il].ca_mlp_gate)), ffn_inp); + cb(cur, "ffn_out", il); + + cur = lctx.cvec.apply_to(ctx0, cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } else { + // self attention layer + + // rope freq factors for llama3; may return nullptr for llama2 and other models + struct ggml_tensor * rope_factors = build_rope_factors(il); + cb(rope_factors, "rope_factors", il); + + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + if (model.layers[il].bq) { + Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq); + cb(Qcur, "Qcur", il); + } + + struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + if (model.layers[il].bk) { + Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk); + cb(Kcur, "Kcur", il); + } + + struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + if (model.layers[il].bv) { + Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); + cb(Vcur, "Vcur_add_bias", il); + } + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur_roped", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur_roped", il); + + cur = llm_build_kv(ctx0, lctx, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, kq_scale, cb, il); + cb(Kcur, "cur-kv", il); + + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + n_tokens = n_outputs; + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, lctx, cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = lctx.cvec.apply_to(ctx0, cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { @@ -17965,6 +18709,10 @@ static struct ggml_cgraph * llama_build_graph( case LLM_ARCH_WAVTOKENIZER_DEC: { result = llm.build_wavtokenizer_dec(); + } break; + case LLM_ARCH_MLLAMA: + { + result = llm.build_mllama(); } break; default: GGML_ABORT("fatal error"); @@ -18044,10 +18792,27 @@ static void llama_set_inputs(llama_context & lctx, const llama_ubatch & ubatch) } if (ubatch.embd) { - const int64_t n_embd = hparams.n_embd; - const int64_t n_tokens = ubatch.n_tokens; + if (lctx.ca_patch_embd && lctx.ca_patch_embd->buffer) { + // Copy/set the image patch embeddings (ubatch.embd) to the + // ca_patch_embd tensor. The ca_patch tensor is created in + // the llm_build_ca_patch_embd function. This tensor will then be + // used for the cross-attention layers to compute the K and V + // matrices using the ca_k_proj and ca_v_proj matrices, and the + // store the result of those operations in kv_self.k_l and + // kv_self.v_l for those layers. And this will make them available + // to the self-attention. + ggml_backend_tensor_set(lctx.ca_patch_embd, ubatch.embd, 0, ggml_nbytes(lctx.ca_patch_embd)); + + float * inp_embd_data = (float *)lctx.inp_embd->data; + for (int i = 0; i < ggml_nelements(lctx.inp_embd); ++i) { + inp_embd_data[i] = 0.0f; + } - ggml_backend_tensor_set(lctx.inp_embd, ubatch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd)); + } else { + const int64_t n_embd = hparams.n_embd; + const int64_t n_tokens = ubatch.n_tokens; + ggml_backend_tensor_set(lctx.inp_embd, ubatch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd)); + } } if (ubatch.pos && lctx.inp_pos) { @@ -18605,7 +19370,7 @@ static int llama_decode_internal( const auto & hparams = model.hparams; const auto & cparams = lctx.cparams; - GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT + //GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT if (batch.token) { for (uint32_t i = 0; i < n_tokens_all; ++i) { @@ -18653,7 +19418,7 @@ static int llama_decode_internal( n_outputs = 1; } - lctx.sbatch.from_batch(batch, n_embd, + lctx.sbatch.from_batch(batch, batch.n_embd, /* simple_split */ !kv_self.recurrent, /* logits_all */ n_outputs == n_tokens_all); @@ -18727,8 +19492,6 @@ static int llama_decode_internal( } } - //printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head); - ggml_backend_sched_reset(lctx.sched.get()); ggml_backend_sched_set_eval_callback(lctx.sched.get(), lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data); @@ -18798,13 +19561,13 @@ static int llama_decode_internal( GGML_ASSERT(backend_res != nullptr); GGML_ASSERT(lctx.logits != nullptr); - float * logits_out = lctx.logits + n_outputs_prev*n_vocab; + float * logits_out = lctx.logits + n_outputs_prev*(n_vocab-1); const int32_t n_outputs_new = lctx.n_outputs; if (n_outputs_new) { GGML_ASSERT( n_outputs_prev + n_outputs_new <= n_outputs); GGML_ASSERT((n_outputs_prev + n_outputs_new)*n_vocab <= (int64_t) lctx.logits_size); - ggml_backend_tensor_get_async(backend_res, res, logits_out, 0, n_outputs_new*n_vocab*sizeof(float)); + ggml_backend_tensor_get_async(backend_res, res, logits_out, 0, n_outputs_new*(n_vocab-1)*sizeof(float)); } } @@ -18961,7 +19724,7 @@ static int llama_encode_internal( const int64_t n_embd = hparams.n_embd; - lctx.sbatch.from_batch(batch, n_embd, /* simple_split */ true, /* logits_all */ true); + lctx.sbatch.from_batch(batch, inp_batch.n_embd, /* simple_split */ true, /* logits_all */ true); const llama_ubatch ubatch = lctx.sbatch.split_simple(n_tokens); @@ -19370,7 +20133,7 @@ static void llama_kv_cache_update_internal(struct llama_context & lctx) { uint32_t n_seqs = 1; // TODO: worst-case number of sequences uint32_t n_tokens = std::min(lctx.cparams.n_ctx, lctx.cparams.n_ubatch); llama_token token = llama_token_bos(&lctx.model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph - llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; + llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, 0, nullptr, nullptr, nullptr, nullptr}; ggml_cgraph * gf = llama_build_graph(lctx, ubatch, true); // initialize scheduler with the worst-case graph @@ -19971,11 +20734,13 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s const struct ggml_tensor * tensor = it->tensor; const std::string name = ggml_get_name(tensor); + printf("tensor name: %s\n", name.c_str()); // TODO: avoid hardcoded tensor names - use the TN_* constants if (name.find("attn_v.weight") != std::string::npos || name.find("attn_qkv.weight") != std::string::npos || - name.find("attn_kv_b.weight")!= std::string::npos) { + name.find("attn_kv_b.weight")!= std::string::npos || + name.find("v_proj.weight")!= std::string::npos) { ++qs.n_attention_wv; } else if (name == LLM_TN(model.arch)(LLM_TENSOR_OUTPUT, "weight")) { qs.has_output = true; @@ -19992,6 +20757,11 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (llama_model_has_encoder(&model)) { n_attn_layer *= 3; } + // TODO(danbev) This is just here to allow quantization to work for + // mllama and should be revisited. + if (model.has_vision) { + n_attn_layer *= 2; + } GGML_ASSERT((qs.n_attention_wv == n_attn_layer) && "n_attention_wv is unexpected"); } @@ -20126,6 +20896,25 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // do not quantize relative position bias (T5) quantize &= name.find("attn_rel_b.weight") == std::string::npos; + // do not quantize mllama's mllama's patch embedding tensors + quantize &= name.find("v.enc.tile_pos_embd") == std::string::npos; + quantize &= name.find("v.enc.tile_pos_gate") == std::string::npos; + quantize &= name.find("v.enc.pre_tile_pos_embd") == std::string::npos; + quantize &= name.find("v.enc.pre_tile_pos_gate") == std::string::npos; + quantize &= name.find("v.enc.post_tile_pos_embd") == std::string::npos; + quantize &= name.find("v.enc.post_tile_pos_gate") == std::string::npos; + quantize &= name.find("v.enc.mmproj.weight") == std::string::npos; + quantize &= name.find("v.enc.embd.patch.weight") == std::string::npos; + //quantize &= name.find("token_embd.weight") == std::string::npos; + //quantize &= name.find("output.weight") == std::string::npos; + quantize &= name.find("cross_attn_k_norm.weight") == std::string::npos; + quantize &= name.find("cross_attn_q_norm.weight") == std::string::npos; + quantize &= name.find("cross_attn_gate.weight") == std::string::npos; + quantize &= name.find("cross_attn_ffn_gate.weight") == std::string::npos; + //quantize &= name.find("attn_norm.weight") == std::string::npos; + //quantize &= name.find("ffn_norm.weight") == std::string::npos; + //quantize &= name.find("output_norm.weight") == std::string::npos; + enum ggml_type new_type; void * new_data; size_t new_size; @@ -21029,7 +21818,7 @@ struct llama_context * llama_new_context_with_model( uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph - llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; + llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, 0, nullptr, nullptr, nullptr, nullptr}; ggml_cgraph * gf_pp = llama_build_graph(*ctx, ubatch_pp, true); // reserve pp graph first so that buffers are only allocated once @@ -21038,7 +21827,7 @@ struct llama_context * llama_new_context_with_model( int n_nodes_pp = ggml_graph_n_nodes(gf_pp); // reserve with tg graph to get the number of splits and nodes - llama_ubatch ubatch_tg = { true, 1, 1, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr}; + llama_ubatch ubatch_tg = { true, 1, 1, n_seqs, &token, nullptr, 0, nullptr, nullptr, nullptr, nullptr}; ggml_cgraph * gf_tg = llama_build_graph(*ctx, ubatch_tg, true); ggml_backend_sched_reserve(ctx->sched.get(), gf_tg); int n_splits_tg = ggml_backend_sched_get_n_splits(ctx->sched.get()); @@ -21078,10 +21867,22 @@ struct llama_context * llama_new_context_with_model( // initialize vision context if (model->has_vision) { - ctx->clip.model = &model->clip; - ctx->clip.sched = ctx->sched.get(); - const size_t max_nodes = llama_model_max_nodes(*model); - ctx->clip.buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false)); + switch (model->arch) { + case LLM_ARCH_MLLAMA: + { + ctx->ca_vision.model = &model->ca_vision; + ctx->ca_vision.sched = ctx->sched.get(); + const size_t max_nodes = llama_model_max_nodes(*model); + ctx->ca_vision.buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false)); + } + break; + default: + ctx->clip.model = &model->clip; + ctx->clip.sched = ctx->sched.get(); + const size_t max_nodes = llama_model_max_nodes(*model); + ctx->clip.buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false)); + break; + } } return ctx; @@ -21174,6 +21975,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_GRANITE: case LLM_ARCH_GRANITE_MOE: case LLM_ARCH_CHAMELEON: + case LLM_ARCH_MLLAMA: return LLAMA_ROPE_TYPE_NORM; // the pairs of head values are offset by n_rot/2 @@ -22579,6 +23381,14 @@ void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn) { ctx->cparams.causal_attn = causal_attn; } +void llama_set_cross_attention(struct llama_context * ctx, bool cross_attention) { + ctx->cparams.cross_attn = cross_attention; +} + +void llama_set_n_tiles(struct llama_context * ctx, uint32_t n_tiles) { + ctx->cparams.n_tiles = n_tiles; +} + struct llama_batch llama_batch_get_one( llama_token * tokens, int32_t n_tokens) { @@ -22586,6 +23396,7 @@ struct llama_batch llama_batch_get_one( /*n_tokens =*/ n_tokens, /*tokens =*/ tokens, /*embd =*/ nullptr, + /*n_embd =*/ 0, /*pos =*/ nullptr, /*n_seq_id =*/ nullptr, /*seq_id =*/ nullptr, @@ -22598,6 +23409,7 @@ struct llama_batch llama_batch_init(int32_t n_tokens_alloc, int32_t embd, int32_ /*n_tokens =*/ 0, /*tokens =*/ nullptr, /*embd =*/ nullptr, + /*n_embd =*/ 0, /*pos =*/ nullptr, /*n_seq_id =*/ nullptr, /*seq_id =*/ nullptr, @@ -22606,6 +23418,7 @@ struct llama_batch llama_batch_init(int32_t n_tokens_alloc, int32_t embd, int32_ if (embd) { batch.embd = (float *) malloc(sizeof(float) * n_tokens_alloc * embd); + batch.n_embd = embd; } else { batch.token = (llama_token *) malloc(sizeof(llama_token) * n_tokens_alloc); } @@ -22648,7 +23461,14 @@ int32_t llama_encode( return ret; } -float * _test_get_img_embd(struct llama_context * ctx) { return ctx->clip.out_embd.data(); } +float * _test_get_img_embd(struct llama_context * ctx) { + if (ctx->clip.sched != nullptr) { + return ctx->clip.out_embd.data(); + } else { + return ctx->ca_vision.out_embd.data(); + } +} + int32_t llama_decode( struct llama_context * ctx, struct llama_batch batch) { @@ -23515,7 +24335,11 @@ void llama_img_free(llama_img * img) { int32_t llama_img_n_tokens(struct llama_context * ctx, llama_img * img) { GGML_UNUSED(img); // reserved for future usage - return clip_n_patches(ctx->clip); + if (ctx->clip.sched != nullptr) { + return clip_n_patches(ctx->clip); + } else { + return ca_n_patches(ctx->ca_vision); + } } llama_batch_img llama_batch_img_init(int n_imgs) { @@ -23534,7 +24358,11 @@ void llama_batch_img_free(llama_batch_img batch) { } int32_t llama_encode_vision(struct llama_context * ctx, llama_batch_img batch) { - return llama_encode_vision_internal(ctx->clip, &batch); + if (ctx->ca_vision.sched != NULL) { + return ca_llama_encode_vision_internal(ctx->ca_vision, &batch); + } else { + return llama_encode_vision_internal(ctx->clip, &batch); + } } //