From 9039257db24ab3d03c8d0712f5ab5f1ee039eb1a Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 8 Jan 2024 05:43:48 -0800 Subject: [PATCH 1/3] Added Face Detector C Tasks API --- mediapipe/tasks/c/vision/face_detector/BUILD | 100 +++++++ .../c/vision/face_detector/face_detector.cc | 277 ++++++++++++++++++ .../c/vision/face_detector/face_detector.h | 138 +++++++++ .../face_detector/face_detector_test.cc | 272 +++++++++++++++++ .../face_landmarker/face_landmarker_test.cc | 8 +- 5 files changed, 791 insertions(+), 4 deletions(-) create mode 100644 mediapipe/tasks/c/vision/face_detector/BUILD create mode 100644 mediapipe/tasks/c/vision/face_detector/face_detector.cc create mode 100644 mediapipe/tasks/c/vision/face_detector/face_detector.h create mode 100644 mediapipe/tasks/c/vision/face_detector/face_detector_test.cc diff --git a/mediapipe/tasks/c/vision/face_detector/BUILD b/mediapipe/tasks/c/vision/face_detector/BUILD new file mode 100644 index 0000000000..b0b2ad5b8b --- /dev/null +++ b/mediapipe/tasks/c/vision/face_detector/BUILD @@ -0,0 +1,100 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "face_detector_lib", + srcs = ["face_detector.cc"], + hdrs = ["face_detector.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/tasks/c/components/containers:detection_result", + "//mediapipe/tasks/c/components/containers:detection_result_converter", + "//mediapipe/tasks/c/core:base_options", + "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/c/vision/core:common", + "//mediapipe/tasks/cc/vision/core:running_mode", + "//mediapipe/tasks/cc/vision/face_detector", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], + alwayslink = 1, +) + +cc_test( + name = "face_detector_test", + srcs = ["face_detector_test.cc"], + data = [ + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + linkstatic = 1, + deps = [ + ":face_detector_lib", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/c/components/containers:category", + "//mediapipe/tasks/c/components/containers:keypoint", + "//mediapipe/tasks/c/vision/core:common", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/face_detector:libface_detector.so +cc_binary( + name = "libface_detector.so", + linkopts = [ + "-Wl,-soname=libface_detector.so", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":face_detector_lib"], +) + +# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/face_detector:libface_detector.dylib +cc_binary( + name = "libface_detector.dylib", + linkopts = [ + "-Wl,-install_name,libface_detector.dylib", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":face_detector_lib"], +) diff --git a/mediapipe/tasks/c/vision/face_detector/face_detector.cc b/mediapipe/tasks/c/vision/face_detector/face_detector.cc new file mode 100644 index 0000000000..d76cb03f8c --- /dev/null +++ b/mediapipe/tasks/c/vision/face_detector/face_detector.cc @@ -0,0 +1,277 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/face_detector/face_detector.h" + +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/tasks/c/components/containers/detection_result_converter.h" +#include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/c/vision/core/common.h" +#include "mediapipe/tasks/cc/vision/core/running_mode.h" +#include "mediapipe/tasks/cc/vision/face_detector/face_detector.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace mediapipe::tasks::c::vision::face_detector { + +namespace { + +using ::mediapipe::tasks::c::components::containers::CppCloseDetectionResult; +using ::mediapipe::tasks::c::components::containers:: + CppConvertToDetectionResult; +using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; +using ::mediapipe::tasks::vision::CreateImageFromBuffer; +using ::mediapipe::tasks::vision::core::RunningMode; +using ::mediapipe::tasks::vision::face_detector::FaceDetector; +typedef ::mediapipe::tasks::vision::face_detector::FaceDetectorResult + CppFaceDetectorResult; + +int CppProcessError(absl::Status status, char** error_msg) { + if (error_msg) { + *error_msg = strdup(status.ToString().c_str()); + } + return status.raw_code(); +} + +} // namespace + +void CppConvertToFaceDetectorOptions( + const FaceDetectorOptions& in, + mediapipe::tasks::vision::face_detector::FaceDetectorOptions* out) { + out->min_detection_confidence = in.min_detection_confidence; + out->min_suppression_threshold = in.min_suppression_threshold; +} + +FaceDetector* CppFaceDetectorCreate(const FaceDetectorOptions& options, + char** error_msg) { + auto cpp_options = std::make_unique< + ::mediapipe::tasks::vision::face_detector::FaceDetectorOptions>(); + + CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); + CppConvertToFaceDetectorOptions(options, cpp_options.get()); + cpp_options->running_mode = static_cast(options.running_mode); + + // Enable callback for processing live stream data when the running mode is + // set to RunningMode::LIVE_STREAM. + if (cpp_options->running_mode == RunningMode::LIVE_STREAM) { + if (options.result_callback == nullptr) { + const absl::Status status = absl::InvalidArgumentError( + "Provided null pointer to callback function."); + ABSL_LOG(ERROR) << "Failed to create FaceDetector: " << status; + CppProcessError(status, error_msg); + return nullptr; + } + + FaceDetectorOptions::result_callback_fn result_callback = + options.result_callback; + cpp_options->result_callback = + [result_callback](absl::StatusOr cpp_result, + const Image& image, int64_t timestamp) { + char* error_msg = nullptr; + + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); + CppProcessError(cpp_result.status(), &error_msg); + result_callback({}, MpImage(), timestamp, error_msg); + free(error_msg); + return; + } + + // Result is valid for the lifetime of the callback function. + FaceDetectorResult result; + CppConvertToDetectionResult(*cpp_result, &result); + + const auto& image_frame = image.GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast<::ImageFormat>(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + result_callback(&result, mp_image, timestamp, + /* error_msg= */ nullptr); + + CppCloseDetectionResult(&result); + }; + } + + auto detector = FaceDetector::Create(std::move(cpp_options)); + if (!detector.ok()) { + ABSL_LOG(ERROR) << "Failed to create FaceDetector: " << detector.status(); + CppProcessError(detector.status(), error_msg); + return nullptr; + } + return detector->release(); +} + +int CppFaceDetectorDetect(void* detector, const MpImage& image, + FaceDetectorResult* result, char** error_msg) { + if (image.type == MpImage::GPU_BUFFER) { + const absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet."); + + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_detector = static_cast(detector); + auto cpp_result = cpp_detector->Detect(*img); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToDetectionResult(*cpp_result, result); + return 0; +} + +int CppFaceDetectorDetectForVideo(void* detector, const MpImage& image, + int64_t timestamp_ms, + FaceDetectorResult* result, + char** error_msg) { + if (image.type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_detector = static_cast(detector); + auto cpp_result = cpp_detector->DetectForVideo(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToDetectionResult(*cpp_result, result); + return 0; +} + +int CppFaceDetectorDetectAsync(void* detector, const MpImage& image, + int64_t timestamp_ms, char** error_msg) { + if (image.type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_detector = static_cast(detector); + auto cpp_result = cpp_detector->DetectAsync(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Data preparation for the landmark detection failed: " + << cpp_result; + return CppProcessError(cpp_result, error_msg); + } + return 0; +} + +void CppFaceDetectorCloseResult(FaceDetectorResult* result) { + CppCloseDetectionResult(result); +} + +int CppFaceDetectorClose(void* detector, char** error_msg) { + auto cpp_detector = static_cast(detector); + auto result = cpp_detector->Close(); + if (!result.ok()) { + ABSL_LOG(ERROR) << "Failed to close FaceDetector: " << result; + return CppProcessError(result, error_msg); + } + delete cpp_detector; + return 0; +} + +} // namespace mediapipe::tasks::c::vision::face_detector + +extern "C" { + +void* face_detector_create(struct FaceDetectorOptions* options, + char** error_msg) { + return mediapipe::tasks::c::vision::face_detector::CppFaceDetectorCreate( + *options, error_msg); +} + +int face_detector_detect_image(void* detector, const MpImage& image, + FaceDetectorResult* result, char** error_msg) { + return mediapipe::tasks::c::vision::face_detector::CppFaceDetectorDetect( + detector, image, result, error_msg); +} + +int face_detector_detect_for_video(void* detector, const MpImage& image, + int64_t timestamp_ms, + FaceDetectorResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::face_detector:: + CppFaceDetectorDetectForVideo(detector, image, timestamp_ms, result, + error_msg); +} + +int face_detector_detect_async(void* detector, const MpImage& image, + int64_t timestamp_ms, char** error_msg) { + return mediapipe::tasks::c::vision::face_detector::CppFaceDetectorDetectAsync( + detector, image, timestamp_ms, error_msg); +} + +void face_detector_close_result(FaceDetectorResult* result) { + mediapipe::tasks::c::vision::face_detector::CppFaceDetectorCloseResult( + result); +} + +int face_detector_close(void* detector, char** error_ms) { + return mediapipe::tasks::c::vision::face_detector::CppFaceDetectorClose( + detector, error_ms); +} + +} // extern "C" diff --git a/mediapipe/tasks/c/vision/face_detector/face_detector.h b/mediapipe/tasks/c/vision/face_detector/face_detector.h new file mode 100644 index 0000000000..d9d15fb3b9 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_detector/face_detector.h @@ -0,0 +1,138 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_FACE_DETECTOR_FACE_DETECTOR_H_ +#define MEDIAPIPE_TASKS_C_VISION_FACE_DETECTOR_FACE_DETECTOR_H_ + +#include "mediapipe/tasks/c/components/containers/detection_result.h" +#include "mediapipe/tasks/c/core/base_options.h" +#include "mediapipe/tasks/c/vision/core/common.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +typedef DetectionResult FaceDetectorResult; + +// The options for configuring a MediaPipe face detector task. +struct FaceDetectorOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // The running mode of the task. Default to the image mode. + // Face Detector has three running modes: + // 1) The image mode for detecting faces on single image inputs. + // 2) The video mode for detecting faces on the decoded frames of a video. + // 3) The live stream mode for detecting faces on the live stream of input + // data, such as from camera. In this mode, the "result_callback" below must + // be specified to receive the detection results asynchronously. + RunningMode running_mode; + + // The minimum confidence score for the face detection to be considered + // successful. + float min_detection_confidence = 0.5; + + // The minimum non-maximum-suppression threshold for face detection to be + // considered overlapped. + float min_suppression_threshold = 0.5; + + // The user-defined result callback for processing live stream data. + // The result callback should only be specified when the running mode is set + // to RunningMode::LIVE_STREAM. Arguments of the callback function include: + // the pointer to recognition result, the image that result was obtained + // on, the timestamp relevant to recognition results and pointer to error + // message in case of any failure. The validity of the passed arguments is + // true for the lifetime of the callback function. + // + // A caller is responsible for closing face detector result. + typedef void (*result_callback_fn)(const FaceDetectorResult* result, + const MpImage& image, int64_t timestamp_ms, + char* error_msg); + result_callback_fn result_callback; +}; + +// Creates an FaceDetector from the provided `options`. +// Returns a pointer to the face detector on success. +// If an error occurs, returns `nullptr` and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT void* face_detector_create(struct FaceDetectorOptions* options, + char** error_msg); + +// Performs face detection on the input `image`. Returns `0` on +// success. If an error occurs, returns an error code and sets the error +// parameter to an an error message (if `error_msg` is not `nullptr`). You must +// free the memory allocated for the error message. +MP_EXPORT int face_detector_detect_image(void* detector, const MpImage& image, + FaceDetectorResult* result, + char** error_msg); + +// Performs face detection on the provided video frame. +// Only use this method when the FaceDetector is created with the video +// running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide the video frame's timestamp (in milliseconds). The input timestamps +// must be monotonically increasing. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int face_detector_detect_for_video(void* detector, + const MpImage& image, + int64_t timestamp_ms, + FaceDetectorResult* result, + char** error_msg); + +// Sends live image data to face detection, and the results will be +// available via the `result_callback` provided in the FaceDetectorOptions. +// Only use this method when the FaceDetector is created with the live +// stream running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide a timestamp (in milliseconds) to indicate when the input image is +// sent to the face detector. The input timestamps must be monotonically +// increasing. +// The `result_callback` provides: +// - The recognition results as an FaceDetectorResult object. +// - The const reference to the corresponding input image that the face +// detector runs on. Note that the const reference to the image will no +// longer be valid when the callback returns. To access the image data +// outside of the callback, callers need to make a copy of the image. +// - The input timestamp in milliseconds. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int face_detector_detect_async(void* detector, const MpImage& image, + int64_t timestamp_ms, + char** error_msg); + +// Frees the memory allocated inside a FaceDetectorResult result. +// Does not free the result pointer itself. +MP_EXPORT void face_detector_close_result(FaceDetectorResult* result); + +// Frees face detector. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int face_detector_close(void* detector, char** error_msg); + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_FACE_DETECTOR_FACE_DETECTOR_H_ diff --git a/mediapipe/tasks/c/vision/face_detector/face_detector_test.cc b/mediapipe/tasks/c/vision/face_detector/face_detector_test.cc new file mode 100644 index 0000000000..03f28dfa28 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_detector/face_detector_test.cc @@ -0,0 +1,272 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/face_detector/face_detector.h" + +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/category.h" +#include "mediapipe/tasks/c/components/containers/keypoint.h" +#include "mediapipe/tasks/c/vision/core/common.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace { + +using ::mediapipe::file::JoinPath; +using ::mediapipe::tasks::vision::DecodeImageFromFile; +using testing::HasSubstr; + +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kModelName[] = "face_detection_short_range.tflite"; +constexpr char kImageFile[] = "portrait.jpg"; +constexpr int kPixelDiffTolerance = 5; +constexpr float kKeypointErrorThreshold = 0.02; +constexpr int kIterations = 100; + +std::string GetFullPath(absl::string_view file_name) { + return JoinPath("./", kTestDataDirectory, file_name); +} + +void AssertFaceDetectorResult(const FaceDetectorResult* result, + const int pixel_diff_tolerance, + const float keypoint_error_threshold) { + int MAX_KEYPOINTS = 2; + + NormalizedKeypoint keypoints[MAX_KEYPOINTS] = { + {0.4432f, 0.1792f, "foo", 0}, + {0.5609f, 0.1800f, "foo", 0}, + }; + + MPRect bounding_box = {283, 115, 349, 517}; + + // Create a single detection with expected results + Detection expected_detection = {/* categories= */ nullptr, + /* categories_count= */ 0, + /* bounding_box= */ bounding_box, + /* keypoints= */ keypoints, + /* keypoints_count= */ MAX_KEYPOINTS}; + + // Create DetectionResult with one Detection + DetectionResult expected_results = {/* detections= */ &expected_detection, + /* detections_count= */ 1}; + + EXPECT_EQ(result->detections_count, expected_results.detections_count); + for (int i = 0; i < result->detections_count; i++) { + const auto& actual_bbox = result->detections[i].bounding_box; + const auto& expected_bbox = expected_results.detections[i].bounding_box; + EXPECT_NEAR(actual_bbox.bottom, expected_bbox.bottom, pixel_diff_tolerance); + EXPECT_NEAR(actual_bbox.right, expected_bbox.right, pixel_diff_tolerance); + EXPECT_NEAR(actual_bbox.top, expected_bbox.top, pixel_diff_tolerance); + EXPECT_NEAR(actual_bbox.left, expected_bbox.left, pixel_diff_tolerance); + EXPECT_EQ(result->detections[i].keypoints_count, 6); + for (int j = 0; j < expected_results.detections[i].keypoints_count; j++) { + EXPECT_NEAR(result->detections[i].keypoints[j].x, + expected_results.detections[i].keypoints[j].x, + keypoint_error_threshold); + EXPECT_NEAR(result->detections[i].keypoints[j].y, + expected_results.detections[i].keypoints[j].y, + keypoint_error_threshold); + } + } +} + +TEST(FaceDetectorTest, ImageModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + FaceDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* min_detection_confidence= */ 0.5, + /* min_suppression_threshold= */ 0.5, + }; + + void* detector = face_detector_create(&options, /* error_msg */ nullptr); + EXPECT_NE(detector, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + FaceDetectorResult result; + face_detector_detect_image(detector, mp_image, &result, + /* error_msg */ nullptr); + AssertFaceDetectorResult(&result, kPixelDiffTolerance, + kKeypointErrorThreshold); + face_detector_close_result(&result); + face_detector_close(detector, /* error_msg */ nullptr); +} + +TEST(FaceDetectorTest, VideoModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + FaceDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::VIDEO, + /* min_detection_confidence= */ 0.5, + /* min_suppression_threshold= */ 0.5, + }; + + void* detector = face_detector_create(&options, + /* error_msg */ nullptr); + EXPECT_NE(detector, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + FaceDetectorResult result; + face_detector_detect_for_video(detector, mp_image, i, &result, + /* error_msg */ nullptr); + + AssertFaceDetectorResult(&result, kPixelDiffTolerance, + kKeypointErrorThreshold); + face_detector_close_result(&result); + } + face_detector_close(detector, /* error_msg */ nullptr); +} + +// A structure to support LiveStreamModeTest below. This structure holds a +// static method `Fn` for a callback function of C API. A `static` qualifier +// allows to take an address of the method to follow API style. Another static +// struct member is `last_timestamp` that is used to verify that current +// timestamp is greater than the previous one. +struct LiveStreamModeCallback { + static int64_t last_timestamp; + static void Fn(const FaceDetectorResult* detector_result, + const MpImage& image, int64_t timestamp, char* error_msg) { + ASSERT_NE(detector_result, nullptr); + ASSERT_EQ(error_msg, nullptr); + AssertFaceDetectorResult(detector_result, kPixelDiffTolerance, + kKeypointErrorThreshold); + EXPECT_GT(image.image_frame.width, 0); + EXPECT_GT(image.image_frame.height, 0); + EXPECT_GT(timestamp, last_timestamp); + ++last_timestamp; + } +}; +int64_t LiveStreamModeCallback::last_timestamp = -1; + +TEST(FaceDetectorTest, LiveStreamModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + + FaceDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::LIVE_STREAM, + /* min_detection_confidence= */ 0.5, + /* min_suppression_threshold= */ 0.5, + /* result_callback= */ LiveStreamModeCallback::Fn, + }; + + void* detector = face_detector_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(detector, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + EXPECT_GE(face_detector_detect_async(detector, mp_image, i, + /* error_msg */ nullptr), + 0); + } + face_detector_close(detector, /* error_msg */ nullptr); + + // Due to the flow limiter, the total of outputs might be smaller than the + // number of iterations. + EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations); + EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0); +} + +TEST(FaceDetectorTest, InvalidArgumentHandling) { + // It is an error to set neither the asset buffer nor the path. + FaceDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ nullptr}, + /* running_mode= */ RunningMode::IMAGE, + /* min_detection_confidence= */ 0.5, + /* min_suppression_threshold= */ 0.5, + }; + + char* error_msg; + void* detector = face_detector_create(&options, &error_msg); + EXPECT_EQ(detector, nullptr); + + EXPECT_THAT(error_msg, HasSubstr("INVALID_ARGUMENT")); + + free(error_msg); +} + +TEST(FaceDetectorTest, FailedRecognitionHandling) { + const std::string model_path = GetFullPath(kModelName); + FaceDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* min_detection_confidence= */ 0.5, + /* min_suppression_threshold= */ 0.5, + }; + + void* detector = face_detector_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(detector, nullptr); + + const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; + FaceDetectorResult result; + char* error_msg; + face_detector_detect_image(detector, mp_image, &result, &error_msg); + EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); + free(error_msg); + face_detector_close(detector, /* error_msg */ nullptr); +} + +} // namespace diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc index 77f069bc3d..7ee02fd766 100644 --- a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc @@ -49,7 +49,7 @@ std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); } -void AssertHandLandmarkerResult(const FaceLandmarkerResult* result, +void AssertFaceLandmarkerResult(const FaceLandmarkerResult* result, const float blendshapes_precision, const float landmark_precision, const float matrix_precison) { @@ -116,7 +116,7 @@ TEST(FaceLandmarkerTest, ImageModeTest) { FaceLandmarkerResult result; face_landmarker_detect_image(landmarker, mp_image, &result, /* error_msg */ nullptr); - AssertHandLandmarkerResult(&result, kBlendshapesPrecision, + AssertFaceLandmarkerResult(&result, kBlendshapesPrecision, kLandmarksPrecision, kFacialTransformationMatrixPrecision); face_landmarker_close_result(&result); @@ -158,7 +158,7 @@ TEST(FaceLandmarkerTest, VideoModeTest) { face_landmarker_detect_for_video(landmarker, mp_image, i, &result, /* error_msg */ nullptr); - AssertHandLandmarkerResult(&result, kBlendshapesPrecision, + AssertFaceLandmarkerResult(&result, kBlendshapesPrecision, kLandmarksPrecision, kFacialTransformationMatrixPrecision); face_landmarker_close_result(&result); @@ -177,7 +177,7 @@ struct LiveStreamModeCallback { const MpImage& image, int64_t timestamp, char* error_msg) { ASSERT_NE(landmarker_result, nullptr); ASSERT_EQ(error_msg, nullptr); - AssertHandLandmarkerResult(landmarker_result, kBlendshapesPrecision, + AssertFaceLandmarkerResult(landmarker_result, kBlendshapesPrecision, kLandmarksPrecision, kFacialTransformationMatrixPrecision); EXPECT_GT(image.image_frame.width, 0); From fababa4e036387ba43153b10feb80680404a87f9 Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 8 Jan 2024 05:44:44 -0800 Subject: [PATCH 2/3] Update Copyright --- mediapipe/tasks/c/vision/face_detector/face_detector.cc | 2 +- mediapipe/tasks/c/vision/face_detector/face_detector.h | 2 +- mediapipe/tasks/c/vision/face_detector/face_detector_test.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/c/vision/face_detector/face_detector.cc b/mediapipe/tasks/c/vision/face_detector/face_detector.cc index d76cb03f8c..4cef3e97be 100644 --- a/mediapipe/tasks/c/vision/face_detector/face_detector.cc +++ b/mediapipe/tasks/c/vision/face_detector/face_detector.cc @@ -1,4 +1,4 @@ -/* Copyright 2023 The MediaPipe Authors. +/* Copyright 2024 The MediaPipe Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/mediapipe/tasks/c/vision/face_detector/face_detector.h b/mediapipe/tasks/c/vision/face_detector/face_detector.h index d9d15fb3b9..7fc15e4f37 100644 --- a/mediapipe/tasks/c/vision/face_detector/face_detector.h +++ b/mediapipe/tasks/c/vision/face_detector/face_detector.h @@ -1,4 +1,4 @@ -/* Copyright 2023 The MediaPipe Authors. +/* Copyright 2024 The MediaPipe Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/mediapipe/tasks/c/vision/face_detector/face_detector_test.cc b/mediapipe/tasks/c/vision/face_detector/face_detector_test.cc index 03f28dfa28..4714533322 100644 --- a/mediapipe/tasks/c/vision/face_detector/face_detector_test.cc +++ b/mediapipe/tasks/c/vision/face_detector/face_detector_test.cc @@ -1,4 +1,4 @@ -/* Copyright 2023 The MediaPipe Authors. +/* Copyright 2024 The MediaPipe Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 1fce1b8daac93d860aa94faabfd753951cd4f919 Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 8 Jan 2024 05:45:10 -0800 Subject: [PATCH 3/3] Update Copyright --- mediapipe/tasks/c/vision/face_detector/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/c/vision/face_detector/BUILD b/mediapipe/tasks/c/vision/face_detector/BUILD index b0b2ad5b8b..f65f48cd00 100644 --- a/mediapipe/tasks/c/vision/face_detector/BUILD +++ b/mediapipe/tasks/c/vision/face_detector/BUILD @@ -1,4 +1,4 @@ -# Copyright 2023 The MediaPipe Authors. +# Copyright 2024 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.