region decoding: use raw image descriptor in Flutter on decoded bytes…

… from Android
deckerst · Mar 2, 2025 · f850178 · f850178
1 parent 5805bb2
commit f850178
Show file tree

Hide file tree

Showing 13 changed files with 205 additions and 70 deletions.
diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt
@@ -38,7 +38,7 @@ import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
 import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend
 import deckers.thibault.aves.model.FieldMap
 import deckers.thibault.aves.utils.BitmapUtils
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
 import deckers.thibault.aves.utils.LogUtils
 import deckers.thibault.aves.utils.anyCauseIs
 import deckers.thibault.aves.utils.getApplicationInfoCompat
@@ -175,7 +175,7 @@ class AppAdapterHandler(private val context: Context) : MethodCallHandler {
 
                 try {
                     val bitmap = withContext(Dispatchers.IO) { target.get() }
-                    data = bitmap?.getBytes(canHaveAlpha = true, recycle = false)
+                    data = bitmap?.getEncodedBytes(canHaveAlpha = true, recycle = false)
                 } catch (e: Exception) {
                     Log.w(LOG_TAG, "failed to decode app icon for packageName=$packageName", e)
                 }

diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt
@@ -23,7 +23,7 @@ import deckers.thibault.aves.model.FieldMap
 import deckers.thibault.aves.model.provider.ImageProvider
 import deckers.thibault.aves.model.provider.ImageProviderFactory.getProvider
 import deckers.thibault.aves.utils.BitmapUtils
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
 import deckers.thibault.aves.utils.FileUtils.transferFrom
 import deckers.thibault.aves.utils.LogUtils
 import deckers.thibault.aves.utils.MimeTypes
@@ -75,7 +75,7 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
                     val orientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
                     exif.thumbnailBitmap?.let { bitmap ->
                         TransformationUtils.rotateImageExif(BitmapUtils.getBitmapPool(context), bitmap, orientation)?.let {
-                            it.getBytes(canHaveAlpha = false, recycle = false)?.let { bytes -> thumbnails.add(bytes) }
+                            it.getEncodedBytes(canHaveAlpha = false, recycle = false)?.let { bytes -> thumbnails.add(bytes) }
                         }
                     }
                 }

diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt
@@ -3,6 +3,7 @@ package deckers.thibault.aves.channel.calls
 import android.content.Context
 import android.graphics.Rect
 import androidx.core.net.toUri
+import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
 import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend
 import deckers.thibault.aves.channel.calls.fetchers.RegionFetcher
 import deckers.thibault.aves.channel.calls.fetchers.SvgRegionFetcher
@@ -29,7 +30,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
     override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
         when (call.method) {
             "getThumbnail" -> ioScope.launch { safeSuspend(call, result, ::getThumbnail) }
-            "getRegion" -> ioScope.launch { safeSuspend(call, result, ::getRegion) }
+            "getRegion" -> ioScope.launch { safe(call, result, ::getRegion) }
             else -> result.notImplemented()
         }
     }
@@ -68,7 +69,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
         ).fetch()
     }
 
-    private suspend fun getRegion(call: MethodCall, result: MethodChannel.Result) {
+    private fun getRegion(call: MethodCall, result: MethodChannel.Result) {
         val uri = call.argument<String>("uri")?.toUri()
         val mimeType = call.argument<String>("mimeType")
         val pageId = call.argument<Int>("pageId")
@@ -97,13 +98,15 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
                 imageHeight = imageHeight,
                 result = result,
             )
+
             MimeTypes.TIFF -> TiffRegionFetcher(context).fetch(
                 uri = uri,
                 page = pageId ?: 0,
                 sampleSize = sampleSize,
                 regionRect = regionRect,
                 result = result,
             )
+
             else -> regionFetcher.fetch(
                 uri = uri,
                 mimeType = mimeType,

diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt
@@ -4,16 +4,17 @@ import android.content.Context
 import android.graphics.Bitmap
 import android.graphics.BitmapFactory
 import android.graphics.BitmapRegionDecoder
+import android.graphics.ColorSpace
 import android.graphics.Rect
 import android.net.Uri
+import android.os.Build
 import android.util.Log
 import com.bumptech.glide.Glide
 import deckers.thibault.aves.decoder.AvesAppGlideModule
 import deckers.thibault.aves.decoder.MultiPageImage
 import deckers.thibault.aves.utils.BitmapRegionDecoderCompat
 import deckers.thibault.aves.utils.BitmapUtils
-import deckers.thibault.aves.utils.BitmapUtils.ARGB_8888_BYTE_SIZE
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes
 import deckers.thibault.aves.utils.LogUtils
 import deckers.thibault.aves.utils.MathUtils
 import deckers.thibault.aves.utils.MemoryUtils
@@ -33,7 +34,10 @@ class RegionFetcher internal constructor(
 
     private val exportUris = HashMap<Pair<Uri, Int?>, Uri>()
 
-    suspend fun fetch(
+    // return decoded bytes in ARGB_8888, with trailer bytes:
+    // - width (int32)
+    // - height (int32)
+    fun fetch(
         uri: Uri,
         mimeType: String,
         pageId: Int?,
@@ -99,26 +103,37 @@ class RegionFetcher internal constructor(
                 }
             }
 
-            // use `Long` as rect size could be unexpectedly large and go beyond `Int` max
-            val targetBitmapSizeBytes: Long = ARGB_8888_BYTE_SIZE.toLong() * effectiveRect.width() * effectiveRect.height() / effectiveSampleSize
+            val options = BitmapFactory.Options().apply {
+                inSampleSize = effectiveSampleSize
+                // Specifying preferred config and color space avoids the need for conversion afterwards,
+                // but may prevent decoding (e.g. from RGBA_1010102 to ARGB_8888 on some devices).
+                inPreferredConfig = PREFERRED_CONFIG
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                    inPreferredColorSpace = ColorSpace.get(ColorSpace.Named.SRGB)
+                }
+            }
+
+            val pixelCount = effectiveRect.width() * effectiveRect.height() / effectiveSampleSize
+            val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), options.inPreferredConfig)
             if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
                 // decoding a region that large would yield an OOM when creating the bitmap
                 result.error("fetch-large-region", "Region too large for uri=$uri regionRect=$regionRect", null)
                 return
             }
 
-            val options = BitmapFactory.Options().apply {
-                inSampleSize = effectiveSampleSize
-            }
-            val bitmap = decoder.decodeRegion(effectiveRect, options)
-            if (bitmap != null) {
-                val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType)
-                val recycle = false
-                var bytes = bitmap.getBytes(canHaveAlpha, recycle = recycle)
-                if (bytes != null && bytes.isEmpty()) {
-                    bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, recycle = recycle)
+            var bitmap = decoder.decodeRegion(effectiveRect, options)
+            if (bitmap == null) {
+                // retry without specifying config or color space,
+                // falling back to custom byte conversion afterwards
+                options.inPreferredConfig = null
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && options.inPreferredColorSpace != null) {
+                    options.inPreferredColorSpace = null
                 }
-                bitmap.recycle()
+                bitmap = decoder.decodeRegion(effectiveRect, options)
+            }
+
+            val bytes = bitmap?.getDecodedBytes(recycle = true)
+            if (bytes != null) {
                 result.success(bytes)
             } else {
                 result.error("fetch-null", "failed to decode region for uri=$uri regionRect=$regionRect", null)
@@ -173,5 +188,6 @@ class RegionFetcher internal constructor(
 
     companion object {
         private val LOG_TAG = LogUtils.createTag<RegionFetcher>()
+        private val PREFERRED_CONFIG = Bitmap.Config.ARGB_8888
     }
 }
diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/SvgRegionFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/SvgRegionFetcher.kt
@@ -13,8 +13,8 @@ import com.caverock.androidsvg.SVG
 import com.caverock.androidsvg.SVGParseException
 import deckers.thibault.aves.metadata.SVGParserBufferedInputStream
 import deckers.thibault.aves.metadata.SvgHelper.normalizeSize
-import deckers.thibault.aves.utils.BitmapUtils.ARGB_8888_BYTE_SIZE
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils
+import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes
 import deckers.thibault.aves.utils.MemoryUtils
 import deckers.thibault.aves.utils.StorageUtils
 import io.flutter.plugin.common.MethodChannel
@@ -25,7 +25,7 @@ class SvgRegionFetcher internal constructor(
 ) {
     private var lastSvgRef: LastSvgRef? = null
 
-    suspend fun fetch(
+    fun fetch(
         uri: Uri,
         sizeBytes: Long?,
         scale: Int,
@@ -92,25 +92,25 @@ class SvgRegionFetcher internal constructor(
 
             val targetBitmapWidth = regionRect.width()
             val targetBitmapHeight = regionRect.height()
+            val canvasWidth = targetBitmapWidth + bleedX * 2
+            val canvasHeight = targetBitmapHeight + bleedY * 2
 
-            // use `Long` as rect size could be unexpectedly large and go beyond `Int` max
-            val targetBitmapSizeBytes: Long = ARGB_8888_BYTE_SIZE.toLong() * targetBitmapWidth * targetBitmapHeight
+            val config = PREFERRED_CONFIG
+            val pixelCount = canvasWidth * canvasHeight
+            val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), config)
             if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
                 // decoding a region that large would yield an OOM when creating the bitmap
                 result.error("fetch-read-large-region", "SVG region too large for uri=$uri regionRect=$regionRect", null)
                 return
             }
 
-            var bitmap = createBitmap(
-                targetBitmapWidth + bleedX * 2,
-                targetBitmapHeight + bleedY * 2,
-                Bitmap.Config.ARGB_8888
-            )
+            var bitmap = createBitmap(canvasWidth, canvasHeight, config)
             val canvas = Canvas(bitmap)
             svg.renderToCanvas(canvas, renderOptions)
 
             bitmap = Bitmap.createBitmap(bitmap, bleedX, bleedY, targetBitmapWidth, targetBitmapHeight)
-            result.success(bitmap.getBytes(canHaveAlpha = true, recycle = true))
+            val bytes = bitmap.getDecodedBytes(recycle = true)
+            result.success(bytes)
         } catch (e: Exception) {
             result.error("fetch-read-exception", "failed to initialize region decoder for uri=$uri regionRect=$regionRect", e.message)
         }
@@ -120,4 +120,8 @@ class SvgRegionFetcher internal constructor(
         val uri: Uri,
         val svg: SVG,
     )
+
+    companion object {
+        private val PREFERRED_CONFIG = Bitmap.Config.ARGB_8888
+    }
 }
diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt
@@ -17,7 +17,7 @@ import deckers.thibault.aves.decoder.AvesAppGlideModule
 import deckers.thibault.aves.decoder.MultiPageImage
 import deckers.thibault.aves.utils.BitmapUtils
 import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
 import deckers.thibault.aves.utils.MimeTypes
 import deckers.thibault.aves.utils.MimeTypes.SVG
 import deckers.thibault.aves.utils.MimeTypes.isVideo
@@ -81,9 +81,9 @@ class ThumbnailFetcher internal constructor(
         if (bitmap != null) {
             val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType)
             val recycle = false
-            var bytes = bitmap.getBytes(canHaveAlpha, quality, recycle)
+            var bytes = bitmap.getEncodedBytes(canHaveAlpha, quality, recycle)
             if (bytes != null && bytes.isEmpty()) {
-                bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, quality, recycle)
+                bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, quality, recycle)
             }
             result.success(bytes)
         } else {

diff --git a/...oid/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt b/...oid/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt
@@ -3,15 +3,15 @@ package deckers.thibault.aves.channel.calls.fetchers
 import android.content.Context
 import android.graphics.Rect
 import android.net.Uri
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes
 import io.flutter.plugin.common.MethodChannel
 import org.beyka.tiffbitmapfactory.DecodeArea
 import org.beyka.tiffbitmapfactory.TiffBitmapFactory
 
 class TiffRegionFetcher internal constructor(
     private val context: Context,
 ) {
-    suspend fun fetch(
+    fun fetch(
         uri: Uri,
         page: Int,
         sampleSize: Int,
@@ -32,8 +32,9 @@ class TiffRegionFetcher internal constructor(
                     inDecodeArea = DecodeArea(regionRect.left, regionRect.top, regionRect.width(), regionRect.height())
                 }
                 val bitmap = TiffBitmapFactory.decodeFileDescriptor(fd, options)
-                if (bitmap != null) {
-                    result.success(bitmap.getBytes(canHaveAlpha = true, recycle = true))
+                val bytes = bitmap?.getDecodedBytes(recycle = true)
+                if (bytes != null) {
+                    result.success(bytes)
                 } else {
                     result.error("getRegion-tiff-null", "failed to decode region for uri=$uri page=$page regionRect=$regionRect", null)
                 }

diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt
@@ -10,7 +10,7 @@ import com.bumptech.glide.Glide
 import deckers.thibault.aves.decoder.AvesAppGlideModule
 import deckers.thibault.aves.utils.BitmapUtils
 import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
 import deckers.thibault.aves.utils.LogUtils
 import deckers.thibault.aves.utils.MemoryUtils
 import deckers.thibault.aves.utils.MimeTypes
@@ -140,9 +140,9 @@ class ImageByteStreamHandler(private val context: Context, private val arguments
             if (bitmap != null) {
                 val recycle = false
                 val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType)
-                var bytes = bitmap.getBytes(canHaveAlpha, recycle = recycle)
+                var bytes = bitmap.getEncodedBytes(canHaveAlpha, recycle = recycle)
                 if (bytes != null && bytes.isEmpty()) {
-                    bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, recycle = recycle)
+                    bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, recycle = recycle)
                 }
                 if (MemoryUtils.canAllocate(sizeBytes)) {
                     success(bytes)
@@ -168,7 +168,7 @@ class ImageByteStreamHandler(private val context: Context, private val arguments
         try {
             val bitmap = withContext(Dispatchers.IO) { target.get() }
             if (bitmap != null) {
-                val bytes = bitmap.getBytes(canHaveAlpha = false, recycle = false)
+                val bytes = bitmap.getEncodedBytes(canHaveAlpha = false, recycle = false)
                 if (MemoryUtils.canAllocate(sizeBytes)) {
                     success(bytes)
                 } else {

diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt
@@ -20,7 +20,7 @@ import com.bumptech.glide.load.model.MultiModelLoaderFactory
 import com.bumptech.glide.module.LibraryGlideModule
 import com.bumptech.glide.signature.ObjectKey
 import deckers.thibault.aves.utils.BitmapUtils
-import deckers.thibault.aves.utils.BitmapUtils.getBytes
+import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
 import deckers.thibault.aves.utils.MemoryUtils
 import deckers.thibault.aves.utils.StorageUtils.openMetadataRetriever
 import kotlinx.coroutines.CoroutineScope
@@ -112,7 +112,8 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
 
                         // the returned frame is already rotated according to the video metadata
                         val frame = if (dstWidth > 0 && dstHeight > 0 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
-                            val targetBitmapSizeBytes: Long = FORMAT_BYTE_SIZE.toLong() * dstWidth * dstHeight
+                            val pixelCount = dstWidth * dstHeight
+                            val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), getPreferredConfig())
                             if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
                                 throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the scaled frame at $dstWidth x $dstHeight")
                             }
@@ -122,7 +123,8 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
                                 retriever.getScaledFrameAtTime(timeMicros, option, dstWidth, dstHeight)
                             }
                         } else {
-                            val targetBitmapSizeBytes: Long = (FORMAT_BYTE_SIZE.toLong() * videoWidth * videoHeight).toLong()
+                            val pixelCount = videoWidth * videoHeight
+                            val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), getPreferredConfig())
                             if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
                                 throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the full frame at $videoWidth x $videoHeight")
                             }
@@ -132,7 +134,7 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
                                 retriever.getFrameAtTime(timeMicros, option)
                             }
                         }
-                        bytes = frame?.getBytes(canHaveAlpha = false, recycle = false)
+                        bytes = frame?.getEncodedBytes(canHaveAlpha = false, recycle = false)
                     }
 
                     if (bytes != null) {
@@ -151,8 +153,14 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
     }
 
     @RequiresApi(Build.VERSION_CODES.P)
-    private fun getBitmapParams() = MediaMetadataRetriever.BitmapParams().apply {
-        preferredConfig = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+    private fun getBitmapParams(): MediaMetadataRetriever.BitmapParams {
+        val params = MediaMetadataRetriever.BitmapParams()
+        params.preferredConfig = this.getPreferredConfig()
+        return params
+    }
+
+    private fun getPreferredConfig(): Bitmap.Config {
+        return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
             // improved precision with the same memory cost as `ARGB_8888` (4 bytes per pixel)
             // for wide-gamut and HDR content which does not require alpha blending
             Bitmap.Config.RGBA_1010102
@@ -170,9 +178,4 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
     override fun getDataClass(): Class<InputStream> = InputStream::class.java
 
     override fun getDataSource(): DataSource = DataSource.LOCAL
-
-    companion object {
-        // same for either `ARGB_8888` or `RGBA_1010102`
-        private const val FORMAT_BYTE_SIZE = BitmapUtils.ARGB_8888_BYTE_SIZE
-    }
 }