diff --git a/.github/workflows/bridge.yml b/.github/workflows/bridge.yml index 1c0fec8d21f9..9d56fbe6f2b5 100644 --- a/.github/workflows/bridge.yml +++ b/.github/workflows/bridge.yml @@ -73,7 +73,7 @@ jobs: - name: Install flutter rust bridge deps shell: bash run: | - cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" + cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" --locked pushd flutter && sed -i -e 's/extended_text: 14.0.0/extended_text: 13.0.0/g' pubspec.yaml && flutter pub get && popd - name: Run flutter rust bridge diff --git a/.github/workflows/flutter-build.yml b/.github/workflows/flutter-build.yml index fd39302163f3..b295e70f6f87 100644 --- a/.github/workflows/flutter-build.yml +++ b/.github/workflows/flutter-build.yml @@ -1032,7 +1032,7 @@ jobs: ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }} run: | rustup target add ${{ matrix.job.target }} - cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} + cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} --locked case ${{ matrix.job.target }} in aarch64-linux-android) ./flutter/ndk_arm64.sh diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml index 843e07835cdb..9d4d42cb3eda 100644 --- a/.github/workflows/playground.yml +++ b/.github/workflows/playground.yml @@ -149,7 +149,7 @@ jobs: shell: bash run: | sed -i '' 's/3.1.0/2.17.0/g' flutter/pubspec.yaml; - cargo install flutter_rust_bridge_codegen --version ${{ matrix.job.bridge }} --features "uuid" + cargo install flutter_rust_bridge_codegen --version ${{ matrix.job.bridge }} --features "uuid" --locked # below works for mac to make buildable on 3.13.9 # pushd flutter/lib; find . -name "*.dart" | xargs -I{} sed -i '' 's/textScaler: TextScaler.linear(\(.*\)),/textScaleFactor: \1,/g' {}; popd; pushd flutter && flutter pub get && popd @@ -302,7 +302,7 @@ jobs: - name: Install flutter rust bridge deps run: | git config --global core.longpaths true - cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" + cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" --locked sed -i 's/uni_links_desktop/#uni_links_desktop/g' flutter/pubspec.yaml pushd flutter/lib; find . | grep dart | xargs sed -i 's/textScaler: TextScaler.linear(\(.*\)),/textScaleFactor: \1,/g'; popd; pushd flutter ; flutter pub get ; popd @@ -347,7 +347,7 @@ jobs: ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }} run: | rustup target add ${{ matrix.job.target }} - cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} + cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} --locked case ${{ matrix.job.target }} in aarch64-linux-android) ./flutter/ndk_arm64.sh diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index fdcd28f8d04c..e5cde939fb76 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -1268,7 +1268,9 @@ class ImageModel with ChangeNotifier { rgba, rect?.width.toInt() ?? 0, rect?.height.toInt() ?? 0, - isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888, + isWeb | isWindows | isLinux + ? ui.PixelFormat.rgba8888 + : ui.PixelFormat.bgra8888, ); if (parent.target?.id != pid) return; await update(image); diff --git a/flutter/pubspec.lock b/flutter/pubspec.lock index 7c60e037a57d..31f04ce4076d 100644 --- a/flutter/pubspec.lock +++ b/flutter/pubspec.lock @@ -1277,10 +1277,11 @@ packages: texture_rgba_renderer: dependency: "direct main" description: - name: texture_rgba_renderer - sha256: cb048abdd800468ca40749ca10d1db9d1e6a055d1cde6234c05191293f0c7d61 - url: "https://pub.dev" - source: hosted + path: "." + ref: "42797e0f03141dc2b585f76c64a13974508058b4" + resolved-ref: "42797e0f03141dc2b585f76c64a13974508058b4" + url: "https://github.com/rustdesk-org/flutter_texture_rgba_renderer" + source: git version: "0.0.16" timing: dependency: transitive diff --git a/flutter/pubspec.yaml b/flutter/pubspec.yaml index afe09a0dc72e..bec58a4d6ced 100644 --- a/flutter/pubspec.yaml +++ b/flutter/pubspec.yaml @@ -91,7 +91,10 @@ dependencies: password_strength: ^0.2.0 flutter_launcher_icons: ^0.13.1 flutter_keyboard_visibility: ^5.4.0 - texture_rgba_renderer: ^0.0.16 + texture_rgba_renderer: + git: + url: https://github.com/rustdesk-org/flutter_texture_rgba_renderer + ref: 42797e0f03141dc2b585f76c64a13974508058b4 percent_indicator: ^4.2.2 dropdown_button2: ^2.0.0 flutter_gpu_texture_renderer: diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index d929024d84eb..3d56472eedc4 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -193,15 +193,11 @@ impl EncoderApi for HwRamEncoder { } fn support_abr(&self) -> bool { - ["qsv", "vaapi", "mediacodec", "videotoolbox"] - .iter() - .all(|&x| !self.config.name.contains(x)) + ["qsv", "vaapi"].iter().all(|&x| !self.config.name.contains(x)) } fn support_changing_quality(&self) -> bool { - ["vaapi", "mediacodec", "videotoolbox"] - .iter() - .all(|&x| !self.config.name.contains(x)) + ["vaapi"].iter().all(|&x| !self.config.name.contains(x)) } fn latency_free(&self) -> bool { diff --git a/res/vcpkg/ffmpeg/patch/0004-videotoolbox-changing-bitrate.patch b/res/vcpkg/ffmpeg/patch/0004-videotoolbox-changing-bitrate.patch new file mode 100644 index 000000000000..a0b337c5bae5 --- /dev/null +++ b/res/vcpkg/ffmpeg/patch/0004-videotoolbox-changing-bitrate.patch @@ -0,0 +1,84 @@ +From 7f12898fe8fd12c1042c98b34825ab2eda89e54d Mon Sep 17 00:00:00 2001 +From: 21pages +Date: Sun, 24 Nov 2024 12:58:39 +0800 +Subject: [PATCH 1/2] videotoolbox changing bitrate + +Signed-off-by: 21pages +--- + libavcodec/videotoolboxenc.c | 39 ++++++++++++++++++++++++++++++++++++ + 1 file changed, 39 insertions(+) + +diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c +index 5ea9afee22..89c927cdcc 100644 +--- a/libavcodec/videotoolboxenc.c ++++ b/libavcodec/videotoolboxenc.c +@@ -278,6 +278,8 @@ typedef struct VTEncContext { + int max_slice_bytes; + int power_efficient; + int max_ref_frames; ++ ++ int last_bit_rate; + } VTEncContext; + + static int vt_dump_encoder(AVCodecContext *avctx) +@@ -1174,6 +1176,7 @@ static int vtenc_create_encoder(AVCodecContext *avctx, + int64_t one_second_value = 0; + void *nums[2]; + ++ vtctx->last_bit_rate = bit_rate; + int status = VTCompressionSessionCreate(kCFAllocatorDefault, + avctx->width, + avctx->height, +@@ -2618,6 +2621,41 @@ static int vtenc_send_frame(AVCodecContext *avctx, + return 0; + } + ++static void update_config(AVCodecContext *avctx) ++{ ++ VTEncContext *vtctx = avctx->priv_data; ++ ++ if (avctx->codec_id != AV_CODEC_ID_PRORES) { ++ if (avctx->bit_rate != vtctx->last_bit_rate) { ++ av_log(avctx, AV_LOG_INFO, "Setting bit rate to %d\n", avctx->bit_rate); ++ vtctx->last_bit_rate = avctx->bit_rate; ++ SInt32 bit_rate = avctx->bit_rate; ++ CFNumberRef bit_rate_num = CFNumberCreate(kCFAllocatorDefault, ++ kCFNumberSInt32Type, ++ &bit_rate); ++ if (!bit_rate_num) return; ++ ++ if (vtctx->constant_bit_rate) { ++ int status = VTSessionSetProperty(vtctx->session, ++ compat_keys.kVTCompressionPropertyKey_ConstantBitRate, ++ bit_rate_num); ++ if (status == kVTPropertyNotSupportedErr) { ++ av_log(avctx, AV_LOG_ERROR, "Error: -constant_bit_rate true is not supported by the encoder.\n"); ++ } ++ } else { ++ int status = VTSessionSetProperty(vtctx->session, ++ kVTCompressionPropertyKey_AverageBitRate, ++ bit_rate_num); ++ if (!status) { ++ av_log(avctx, AV_LOG_ERROR, "Error: cannot set average bit rate: %d\n", status); ++ } ++ } ++ ++ CFRelease(bit_rate_num); ++ } ++ } ++} ++ + static av_cold int vtenc_frame( + AVCodecContext *avctx, + AVPacket *pkt, +@@ -2630,6 +2668,7 @@ static av_cold int vtenc_frame( + CMSampleBufferRef buf = NULL; + ExtraSEI *sei = NULL; + ++ update_config(avctx); + if (frame) { + status = vtenc_send_frame(avctx, vtctx, frame); + +-- +2.43.0.windows.1 + diff --git a/res/vcpkg/ffmpeg/patch/0005-mediacodec-changing-bitrate.patch b/res/vcpkg/ffmpeg/patch/0005-mediacodec-changing-bitrate.patch new file mode 100644 index 000000000000..1f70a5659308 --- /dev/null +++ b/res/vcpkg/ffmpeg/patch/0005-mediacodec-changing-bitrate.patch @@ -0,0 +1,264 @@ +From 51ac90d8084f7b153eac5133765fa9d0365aa239 Mon Sep 17 00:00:00 2001 +From: 21pages +Date: Sun, 24 Nov 2024 14:17:39 +0800 +Subject: [PATCH 1/4] mediacodec changing bitrate + +Signed-off-by: 21pages +--- + libavcodec/mediacodec_wrapper.c | 101 ++++++++++++++++++++++++++++++++ + libavcodec/mediacodec_wrapper.h | 7 +++ + libavcodec/mediacodecenc.c | 18 ++++++ + 3 files changed, 126 insertions(+) + +diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c +index 306359071e..1ab4e673f6 100644 +--- a/libavcodec/mediacodec_wrapper.c ++++ b/libavcodec/mediacodec_wrapper.c +@@ -35,6 +35,8 @@ + #include "ffjni.h" + #include "mediacodec_wrapper.h" + ++#define PARAMETER_KEY_VIDEO_BITRATE "video-bitrate" ++ + struct JNIAMediaCodecListFields { + + jclass mediacodec_list_class; +@@ -195,6 +197,8 @@ struct JNIAMediaCodecFields { + jmethodID set_input_surface_id; + jmethodID signal_end_of_input_stream_id; + ++ jmethodID set_parameters_id; ++ + jclass mediainfo_class; + + jmethodID init_id; +@@ -248,6 +252,8 @@ static const struct FFJniField jni_amediacodec_mapping[] = { + { "android/media/MediaCodec", "setInputSurface", "(Landroid/view/Surface;)V", FF_JNI_METHOD, OFFSET(set_input_surface_id), 0 }, + { "android/media/MediaCodec", "signalEndOfInputStream", "()V", FF_JNI_METHOD, OFFSET(signal_end_of_input_stream_id), 0 }, + ++ { "android/media/MediaCodec", "setParameters", "(Landroid/os/Bundle;)V", FF_JNI_METHOD, OFFSET(set_parameters_id), 0 }, ++ + { "android/media/MediaCodec$BufferInfo", NULL, NULL, FF_JNI_CLASS, OFFSET(mediainfo_class), 1 }, + + { "android/media/MediaCodec.BufferInfo", "", "()V", FF_JNI_METHOD, OFFSET(init_id), 1 }, +@@ -292,6 +298,24 @@ typedef struct FFAMediaCodecJni { + + static const FFAMediaCodec media_codec_jni; + ++struct JNIABundleFields ++{ ++ jclass bundle_class; ++ jmethodID init_id; ++ jmethodID put_int_id; ++}; ++ ++#define OFFSET(x) offsetof(struct JNIABundleFields, x) ++static const struct FFJniField jni_abundle_mapping[] = { ++ { "android/os/Bundle", NULL, NULL, FF_JNI_CLASS, OFFSET(bundle_class), 1 }, ++ ++ { "android/os/Bundle", "", "()V", FF_JNI_METHOD, OFFSET(init_id), 1 }, ++ { "android/os/Bundle", "putInt", "(Ljava/lang/String;I)V", FF_JNI_METHOD, OFFSET(put_int_id), 1 }, ++ ++ { NULL } ++}; ++#undef OFFSET ++ + #define JNI_GET_ENV_OR_RETURN(env, log_ctx, ret) do { \ + (env) = ff_jni_get_env(log_ctx); \ + if (!(env)) { \ +@@ -1761,6 +1785,69 @@ static int mediacodec_jni_signalEndOfInputStream(FFAMediaCodec *ctx) + return 0; + } + ++static int mediacodec_jni_setParameter(FFAMediaCodec *ctx, const char* name, int value) ++{ ++ JNIEnv *env = NULL; ++ struct JNIABundleFields jfields = { 0 }; ++ jobject object = NULL; ++ jstring key = NULL; ++ FFAMediaCodecJni *codec = (FFAMediaCodecJni *)ctx; ++ void *log_ctx = codec; ++ int ret = -1; ++ ++ JNI_GET_ENV_OR_RETURN(env, codec, AVERROR_EXTERNAL); ++ ++ if (ff_jni_init_jfields(env, &jfields, jni_abundle_mapping, 0, log_ctx) < 0) { ++ av_log(log_ctx, AV_LOG_ERROR, "Failed to init jfields\n"); ++ goto fail; ++ } ++ ++ object = (*env)->NewObject(env, jfields.bundle_class, jfields.init_id); ++ if (!object) { ++ av_log(log_ctx, AV_LOG_ERROR, "Failed to create bundle object\n"); ++ goto fail; ++ } ++ ++ key = ff_jni_utf_chars_to_jstring(env, name, log_ctx); ++ if (!key) { ++ av_log(log_ctx, AV_LOG_ERROR, "Failed to convert key to jstring\n"); ++ goto fail; ++ } ++ ++ (*env)->CallVoidMethod(env, object, jfields.put_int_id, key, value); ++ if (ff_jni_exception_check(env, 1, log_ctx) < 0) { ++ goto fail; ++ } ++ ++ if (!codec->jfields.set_parameters_id) { ++ av_log(log_ctx, AV_LOG_ERROR, "System doesn't support setParameters\n"); ++ goto fail; ++ } ++ ++ (*env)->CallVoidMethod(env, codec->object, codec->jfields.set_parameters_id, object); ++ if (ff_jni_exception_check(env, 1, log_ctx) < 0) { ++ goto fail; ++ } ++ ++ ret = 0; ++ ++fail: ++ if (key) { ++ (*env)->DeleteLocalRef(env, key); ++ } ++ if (object) { ++ (*env)->DeleteLocalRef(env, object); ++ } ++ ff_jni_reset_jfields(env, &jfields, jni_abundle_mapping, 0, log_ctx); ++ ++ return ret; ++} ++ ++static int mediacodec_jni_setDynamicBitrate(FFAMediaCodec *ctx, int bitrate) ++{ ++ return mediacodec_jni_setParameter(ctx, PARAMETER_KEY_VIDEO_BITRATE, bitrate); ++} ++ + static const FFAMediaFormat media_format_jni = { + .class = &amediaformat_class, + +@@ -1820,6 +1907,8 @@ static const FFAMediaCodec media_codec_jni = { + .getConfigureFlagEncode = mediacodec_jni_getConfigureFlagEncode, + .cleanOutputBuffers = mediacodec_jni_cleanOutputBuffers, + .signalEndOfInputStream = mediacodec_jni_signalEndOfInputStream, ++ ++ .setDynamicBitrate = mediacodec_jni_setDynamicBitrate, + }; + + typedef struct FFAMediaFormatNdk { +@@ -1893,6 +1982,8 @@ typedef struct FFAMediaCodecNdk { + // Available since API level 26. + media_status_t (*setInputSurface)(AMediaCodec*, ANativeWindow *); + media_status_t (*signalEndOfInputStream)(AMediaCodec *); ++ ++ media_status_t (*setParameters)(AMediaCodec *, const AMediaFormat *format); + } FFAMediaCodecNdk; + + static const FFAMediaFormat media_format_ndk; +@@ -2154,6 +2245,8 @@ static inline FFAMediaCodec *ndk_codec_create(int method, const char *arg) { + GET_SYMBOL(setInputSurface, 0) + GET_SYMBOL(signalEndOfInputStream, 0) + ++ GET_SYMBOL(setParameters, 0) ++ + #undef GET_SYMBOL + + switch (method) { +@@ -2428,6 +2521,12 @@ static int mediacodec_ndk_signalEndOfInputStream(FFAMediaCodec *ctx) + return 0; + } + ++static int mediacodec_ndk_setDynamicBitrate(FFAMediaCodec *ctx, int bitrate) ++{ ++ av_log(ctx, AV_LOG_ERROR, "ndk setDynamicBitrate unavailable\n"); ++ return -1; ++} ++ + static const FFAMediaFormat media_format_ndk = { + .class = &amediaformat_ndk_class, + +@@ -2489,6 +2588,8 @@ static const FFAMediaCodec media_codec_ndk = { + .getConfigureFlagEncode = mediacodec_ndk_getConfigureFlagEncode, + .cleanOutputBuffers = mediacodec_ndk_cleanOutputBuffers, + .signalEndOfInputStream = mediacodec_ndk_signalEndOfInputStream, ++ ++ .setDynamicBitrate = mediacodec_ndk_setDynamicBitrate, + }; + + FFAMediaFormat *ff_AMediaFormat_new(int ndk) +diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h +index 11a4260497..86c64556ad 100644 +--- a/libavcodec/mediacodec_wrapper.h ++++ b/libavcodec/mediacodec_wrapper.h +@@ -219,6 +219,8 @@ struct FFAMediaCodec { + + // For encoder with FFANativeWindow as input. + int (*signalEndOfInputStream)(FFAMediaCodec *); ++ ++ int (*setDynamicBitrate)(FFAMediaCodec *codec, int bitrate); + }; + + static inline char *ff_AMediaCodec_getName(FFAMediaCodec *codec) +@@ -343,6 +345,11 @@ static inline int ff_AMediaCodec_signalEndOfInputStream(FFAMediaCodec *codec) + return codec->signalEndOfInputStream(codec); + } + ++static inline int ff_AMediaCodec_setDynamicBitrate(FFAMediaCodec *codec, int bitrate) ++{ ++ return codec->setDynamicBitrate(codec, bitrate); ++} ++ + int ff_Build_SDK_INT(AVCodecContext *avctx); + + enum FFAMediaFormatColorRange { +diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c +index d3bf27cb7f..621529d686 100644 +--- a/libavcodec/mediacodecenc.c ++++ b/libavcodec/mediacodecenc.c +@@ -73,6 +73,8 @@ typedef struct MediaCodecEncContext { + int bitrate_mode; + int level; + int pts_as_dts; ++ ++ int last_bit_rate; + } MediaCodecEncContext; + + enum { +@@ -155,6 +157,8 @@ static av_cold int mediacodec_init(AVCodecContext *avctx) + int ret; + int gop; + ++ s->last_bit_rate = avctx->bit_rate; ++ + if (s->use_ndk_codec < 0) + s->use_ndk_codec = !av_jni_get_java_vm(avctx); + +@@ -515,12 +519,26 @@ static int mediacodec_send(AVCodecContext *avctx, + return 0; + } + ++static void update_config(AVCodecContext *avctx) ++{ ++ MediaCodecEncContext *s = avctx->priv_data; ++ if (avctx->bit_rate != s->last_bit_rate) { ++ s->last_bit_rate = avctx->bit_rate; ++ if (0 != ff_AMediaCodec_setDynamicBitrate(s->codec, avctx->bit_rate)) { ++ av_log(avctx, AV_LOG_ERROR, "Failed to set bitrate to %d\n", avctx->bit_rate); ++ } else { ++ av_log(avctx, AV_LOG_INFO, "Set bitrate to %d\n", avctx->bit_rate); ++ } ++ } ++} ++ + static int mediacodec_encode(AVCodecContext *avctx, AVPacket *pkt) + { + MediaCodecEncContext *s = avctx->priv_data; + int ret; + int got_packet = 0; + ++ update_config(avctx); + // Return on three case: + // 1. Serious error + // 2. Got a packet success +-- +2.43.0.windows.1 + diff --git a/res/vcpkg/ffmpeg/portfile.cmake b/res/vcpkg/ffmpeg/portfile.cmake index 3d4c10906dfa..d56475c059f8 100644 --- a/res/vcpkg/ffmpeg/portfile.cmake +++ b/res/vcpkg/ffmpeg/portfile.cmake @@ -13,6 +13,8 @@ vcpkg_from_github( patch/0001-avcodec-amfenc-add-query_timeout-option-for-h264-hev.patch patch/0002-libavcodec-amfenc-reconfig-when-bitrate-change.patch patch/0003-amf-colorspace.patch + patch/0004-videotoolbox-changing-bitrate.patch + patch/0005-mediacodec-changing-bitrate.patch ) if(SOURCE_PATH MATCHES " ") diff --git a/src/client.rs b/src/client.rs index 474c7fdfc0b8..4ff2c6b522b6 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1188,9 +1188,15 @@ impl VideoHandler { pub fn new(format: CodecFormat, _display: usize) -> Self { let luid = Self::get_adapter_luid(); log::info!("new video handler for display #{_display}, format: {format:?}, luid: {luid:?}"); + let rgba_format = + if cfg!(feature = "flutter") && (cfg!(windows) || cfg!(target_os = "linux")) { + ImageFormat::ABGR + } else { + ImageFormat::ARGB + }; VideoHandler { decoder: Decoder::new(format, luid), - rgb: ImageRgb::new(ImageFormat::ARGB, crate::get_dst_align_rgba()), + rgb: ImageRgb::new(rgba_format, crate::get_dst_align_rgba()), texture: Default::default(), recorder: Default::default(), record: false,