diff --git a/mediapipe/tasks/ios/BUILD b/mediapipe/tasks/ios/BUILD index 787808b5df..806a52a729 100644 --- a/mediapipe/tasks/ios/BUILD +++ b/mediapipe/tasks/ios/BUILD @@ -70,6 +70,7 @@ TENSORFLOW_LITE_C_DEPS = [ CALCULATORS_AND_GRAPHS = [ "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/tasks/cc/audio/audio_classifier:audio_classifier_graph", "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", @@ -112,6 +113,14 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/core:sources/MPPBaseOptions.h", "//mediapipe/tasks/ios/core:sources/MPPTaskOptions.h", "//mediapipe/tasks/ios/core:sources/MPPTaskResult.h", + "//mediapipe/tasks/ios/audio/audio_classifier:sources/MPPAudioClassifier.h", + "//mediapipe/tasks/ios/audio/audio_classifier:sources/MPPAudioClassifierOptions.h", + "//mediapipe/tasks/ios/audio/audio_classifier:sources/MPPAudioClassifierResult.h", + "//mediapipe/tasks/ios/audio/core:sources/MPPAudioData.h", + "//mediapipe/tasks/ios/audio/core:sources/MPPAudioDataFormat.h", + "//mediapipe/tasks/ios/audio/core:sources/MPPAudioRecord.h", + "//mediapipe/tasks/ios/audio/core:sources/MPPAudioRunningMode.h", + "//mediapipe/tasks/ios/audio/core:sources/MPPFloatBuffer.h", "//mediapipe/tasks/ios/text/text_classifier:sources/MPPTextClassifier.h", "//mediapipe/tasks/ios/text/text_classifier:sources/MPPTextClassifierOptions.h", "//mediapipe/tasks/ios/text/text_classifier:sources/MPPTextClassifierResult.h", @@ -164,6 +173,44 @@ strip_api_include_path_prefix( ], ) +apple_static_xcframework( + name = "MediaPipeTasksAudio_framework", + # Avoid dependencies of ":MediaPipeTasksCommon_framework" and + # ":MediaPipeTaskGraphs_library in order to prevent duplicate symbols error + # when the frameworks are imported in iOS projects. + avoid_deps = MEDIAPIPE_TASKS_COMMON_DEPS + CALCULATORS_AND_GRAPHS, + bundle_name = "MediaPipeTasksAudio", + ios = { + "simulator": [ + "arm64", + "x86_64", + ], + "device": ["arm64"], + }, + minimum_os_versions = { + "ios": MPP_TASK_MINIMUM_OS_VERSION, + }, + public_hdrs = [ + ":MPPAudioClassifier.h", + ":MPPAudioClassifierOptions.h", + ":MPPAudioClassifierResult.h", + ":MPPAudioData.h", + ":MPPAudioDataFormat.h", + ":MPPAudioRecord.h", + ":MPPAudioRunningMode.h", + ":MPPBaseOptions.h", + ":MPPCategory.h", + ":MPPClassificationResult.h", + ":MPPCommon.h", + ":MPPFloatBuffer.h", + ":MPPTaskOptions.h", + ":MPPTaskResult.h", + ], + deps = [ + "//mediapipe/tasks/ios/audio/audio_classifier:MPPAudioClassifier", + ], +) + apple_static_xcframework( name = "MediaPipeTasksText_framework", # Avoid dependencies of ":MediaPipeTasksCommon_framework" and diff --git a/mediapipe/tasks/ios/MediaPipeTasksAudio.podspec.template b/mediapipe/tasks/ios/MediaPipeTasksAudio.podspec.template new file mode 100644 index 0000000000..11376cc2fc --- /dev/null +++ b/mediapipe/tasks/ios/MediaPipeTasksAudio.podspec.template @@ -0,0 +1,18 @@ +Pod::Spec.new do |s| + s.name = 'MediaPipeTasksAudio' + s.version = '${MPP_BUILD_VERSION}' + s.authors = 'Google Inc.' + s.license = { :type => 'Apache',:file => "LICENSE" } + s.homepage = 'https://github.com/google/mediapipe' + s.source = { :http => '${MPP_VISION_DOWNLOAD_URL}' } + s.summary = 'MediaPipe Task Library - Audio' + s.description = 'The audio APIs of the MediaPipe Task Library' + + s.ios.deployment_target = '12.0' + + s.module_name = 'MediaPipeTasksAudio' + s.static_framework = true + s.dependency 'MediaPipeTasksCommon', '${MPP_TASKS_COMMON_VERSION}' + s.library = 'c++' + s.vendored_frameworks = 'frameworks/MediaPipeTasksAudio.xcframework' +end diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m b/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m index 23dacbec31..6163846cb0 100644 --- a/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m @@ -40,16 +40,36 @@ - (BOOL)loadBuffer:(MPPFloatBuffer *)buffer } - (BOOL)loadAudioRecord:(MPPAudioRecord *)audioRecord error:(NSError **)error { - if (![audioRecord.audioDataFormat isEqual:self.format]) { - [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"The provided audio record has incompatible audio format"]; + if (![self isValidAudioRecordFormat:audioRecord.audioDataFormat error:error]) { return NO; } MPPFloatBuffer *audioRecordBuffer = [audioRecord readAtOffset:0 withLength:audioRecord.bufferLength error:error]; + return [self loadRingBufferWithAudioRecordBuffer:audioRecordBuffer error:error]; +} + +- (BOOL)isValidAudioRecordFormat:(MPPAudioDataFormat *)format error:(NSError **)error { + if (![format isEqual:self.format]) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"The provided audio record has incompatible audio format"]; + return NO; + } + + return YES; +} + +- (BOOL)loadRingBufferWithAudioRecordBuffer:(MPPFloatBuffer *)audioRecordBuffer + error:(NSError **)error { + // Returns `NO` without populating an error since the function that created `audioRecordBuffer` is + // expected to populate the error param of the caller (`loadAudioRecord`) which passed into this + // function. + // For ease of mocking the logic of `loadAudioRecord` in the tests. + if (!audioRecordBuffer) { + return NO; + } return [_ringBuffer loadFloatBuffer:audioRecordBuffer offset:0 length:audioRecordBuffer.length diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.h b/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.h index 2fd8087926..85102e7cfc 100644 --- a/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.h +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.h @@ -23,6 +23,38 @@ NS_ASSUME_NONNULL_BEGIN * A wrapper class to record the device's microphone continuously. Currently this class only * supports recording up to 2 channels. If the number of channels is 2, then the mono microphone * input is duplicated to provide dual channel data. + * + * Internally the class manages an instance of `AVAudioEngine` for tapping the microphone samples. + * + * Recording sets the following parameters on the `AVAudioSession.sharedInstance()`: + * + * 1. `category` = .playAndRecord + * 2. `mode` = .default + * 3. `categoryOptions` = [.overrideMutedMicrophoneInterruption] + * + * If you need more control, use the native `AVAudioEngine` directly for tapping the microphone. + * `AudioRecord` does not actively monitor audio interruptions, route changes, resetting of media + * services etc. They can be handled in the app using the notifications provided by `AVFAudio`. + * Refer to the official docs for the following: + * 1. Handling audio interruptions + * https://developer.apple.com/documentation/avfaudio/handling_audio_interruptions + * + * 2. Responding to audio route changes + * https://developer.apple.com/documentation/avfaudio/responding_to_audio_route_change + * + * 3. Responding to resetting of media services + * https://developer.apple.com/documentation/avfaudio/avaudiosessionmediaserviceswereresetnotification + * + * iOS may automatically stop the audio engine in some cases. A few examples of such events are a + * system interrupt, route change, etc. In such cases, `read(offset:length:)` returns + * `TasksErrorCode.audioRecordNotTappingMicError`. You can restart recording using + * `startRecording()`. + * + * Note: Avoid setting the `AVAudioSession.sharedInstance()` category, mode, options, channel + * configuration (enabling stereo) or other parameters while audio is being actively recorded using + * the audio record. Setting these parameters or changing the configuration of + * `AVAudioSession.sharedInstance()` while a recording is in progress will result in undefined + * behaviour. */ NS_SWIFT_NAME(AudioRecord) @interface MPPAudioRecord : NSObject @@ -63,20 +95,55 @@ NS_SWIFT_NAME(AudioRecord) * * Use `stop()` to stop recording audio sample from the microphone. * - * @return Boolean value indicating if audio recording started successfully. + * Internally this function activates the `AVAudioSession.sharedInstance()` with the following + * parameters before tapping the microphone using the `AVAudioEngine`: + * 1. `category` = .playAndRecord + * 2. `mode` = .default + * 3. `categoryOptions` = [.overrideMutedMicrophoneInterruption] + * + * If you need more control, use the native `AVAudioEngine` directly for tapping the microphone. + * `AudioRecord` does not actively monitor audio interruptions, route changes, resetting of media + * services etc. They can be handled in the app using the notifications provided by `AVFAudio`. + * Refer to the official docs for the following: + * 1. Handling audio interruptions + * https://developer.apple.com/documentation/avfaudio/handling_audio_interruptions + * + * 2. Responding to audio route changes + * https://developer.apple.com/documentation/avfaudio/responding_to_audio_route_change + * + * 3. Responding to resetting of media services + * https://developer.apple.com/documentation/avfaudio/avaudiosessionmediaserviceswereresetnotification + * + * @return Returns successfully if audio recording has started. */ - (BOOL)startRecordingWithError:(NSError **)error NS_SWIFT_NAME(startRecording()); /** * Stops recording audio from the microphone. All elements in the internal buffer of `AudioRecord` * will also be set to zero. + * + * Internally, this function deactivates the `AVAudioSession.sharedInstance()` after recording is + * stopped. If some audio resources are running in the app when the session is deactivated, this + * function throws an error. The session gets deactivated by the OS irrespective of the return + * status of this function. The error thrown is a warning to indicate the termination of any running + * audio resources. Refer to the official documentation for more details: + * https://developer.apple.com/documentation/avfaudio/avaudiosession/1616627-setactive + * + * @return Returns successfully if recording was stopped and the deactivation of + * `AVAudioSession.sharedInstance()` succeeded. Throws any error encountered when deactivating + * `AVAudioSession.sharedInstance()`. */ -- (void)stop; +- (BOOL)stopWithError:(NSError **)error; /** * Returns the `length` number of elements in the internal buffer of `AudioRecord` starting at * `offset`, i.e, `buffer[offset:offset+length]`. * + * iOS may automatically stop the audio engine in some cases. A few examples of such events are a + * system interrupt, route change, etc. In such cases, this function returns + * `TasksErrorCode.audioRecordNotTappingMicError`. You can restart recording using + * `startRecording()`. + * * @param offset Index in the buffer from which elements are to be read. * @param length Number of elements to be returned. * diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.m b/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.m index 82d46d192f..2dd2bfe084 100644 --- a/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.m +++ b/mediapipe/tasks/ios/audio/core/sources/MPPAudioRecord.m @@ -93,6 +93,16 @@ - (nullable instancetype)initWithAudioDataFormat:(MPPAudioDataFormat *)audioData } - (BOOL)startRecordingWithError:(NSError **)error { + // The audio engine's running state will be set to `NO` when a system interrupt happens and the + // user did not explicitly invoke `stop()`. This method allows restarting recording in such cases. + if (_audioEngine.isRunning) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeAudioRecordNotTappingMicError + description:@"Recording of microphone samples is already in progress. " + @"You can stop recording using `stopRecording()`."]; + return NO; + } + // TODO: This API is deprecated from iOS 17.0. Update to new APIs and restrict the following // code's use to versions below iOS 17.0. switch ([AVAudioSession sharedInstance].recordPermission) { @@ -115,17 +125,15 @@ - (BOOL)startRecordingWithError:(NSError **)error { } case AVAudioSessionRecordPermissionGranted: { - [self startTappingMicrophoneWithError:error]; - return YES; + return [self startTappingMicrophoneWithError:error]; } } return NO; } -- (void)stop { - [[_audioEngine inputNode] removeTapOnBus:0]; - [_audioEngine stop]; +- (BOOL)stopWithError:(NSError **)error { + [self stopAndResetAudioEngine]; // Using strong `self` (instance variable is available through strong self) is okay since the // block is shortlived and it'll release its strong reference to `self` when it finishes @@ -136,11 +144,38 @@ - (void)stop { dispatch_barrier_async(_convertLoadAndReadBufferQueue, ^{ [_floatRingBuffer clear]; }); + + // If any audio resources outside this audio record are currently running in the app (eg:, an + // audio player), an error maybe thrown when deactivating `AVAudioSession.sharedInstance()`. + // Irrespective of whether an error is returned, the session will be deactivated by iOS. Official + // docs that explain the reasons for an error: + // https://developer.apple.com/documentation/avfaudio/avaudiosession/1616627-setactive. + return [[AVAudioSession sharedInstance] + setActive:NO + withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation + error:error]; } - (nullable MPPFloatBuffer *)readAtOffset:(NSUInteger)offset withLength:(NSUInteger)length error:(NSError **)error { + if (!_audioEngine.isRunning) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeAudioRecordNotTappingMicError + description: + @"Recording of microphone samples is not in progress. You may not have started a " + @"recording or OS may have stopped the engine due to an interrupt, route change " + @"etc. You can start recording microphone samples using `startRecording()`."]; + return nil; + } + + return [self internalReadAtOffset:offset withLength:length error:error]; +} + +- (nullable MPPFloatBuffer *)internalReadAtOffset:(NSUInteger)offset + withLength:(NSUInteger)length + error:(NSError **)error { __block MPPFloatBuffer *bufferToReturn = nil; __block NSError *readError = nil; @@ -165,7 +200,31 @@ - (nullable MPPFloatBuffer *)readAtOffset:(NSUInteger)offset return bufferToReturn; } -- (void)startTappingMicrophoneWithError:(NSError **)error { +- (BOOL)startTappingMicrophoneWithError:(NSError **)error { + // Stopping and resetting the audio engine to handle the case where user maybe resuming the audio + // engine stopped by the OS due to an interrupt (eg:, a phone call) or a route change. In such + // cases, if a new tap is installed without removing an existing tap the app will crash. + [self stopAndResetAudioEngine]; + + // For tapping the microphone, `AVAudioSession`'s `category` must be set and it must be + // activated. This audio record is not allowed to tap the microphone with any custom option + // including + // `.allowBluetooth`. This disallows microphone route changes in most scenarios to ensure that + // the recording happens through the device microphone. If users need more control over the + // recording devices or OS interruptions, native `AVAudioEngine` can be used. `category` is set to + // `AVAudioSessionCategoryPlayAndRecord` to ensure that audio playback can be configured in the + // app while an audio record is running. + // TODO: Investigate safe starting of `AVAudioEngine` without any side effects to enable this + // class ot accept custom category, mode and options. + if (!([[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord + error:error] && + [[AVAudioSession sharedInstance] + setActive:YES + withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation + error:error])) { + return NO; + } + AVAudioNode *inputNode = [_audioEngine inputNode]; AVAudioFormat *format = [inputNode outputFormatForBus:0]; @@ -209,6 +268,15 @@ - (void)startTappingMicrophoneWithError:(NSError **)error { [_audioEngine prepare]; [_audioEngine startAndReturnError:error]; + + return YES; +} + +// To stop engine internally without deactivating `AVAudioSession.sharedInstance()`. +- (void)stopAndResetAudioEngine { + [[_audioEngine inputNode] removeTapOnBus:0]; + [_audioEngine stop]; + [_audioEngine reset]; } - (BOOL)loadAudioPCMBuffer:(AVAudioPCMBuffer *)pcmBuffer error:(NSError **)error { diff --git a/mediapipe/tasks/ios/audio/core/sources/MPPFloatBuffer.h b/mediapipe/tasks/ios/audio/core/sources/MPPFloatBuffer.h index c6cb46c12e..7f1318a06c 100644 --- a/mediapipe/tasks/ios/audio/core/sources/MPPFloatBuffer.h +++ b/mediapipe/tasks/ios/audio/core/sources/MPPFloatBuffer.h @@ -48,6 +48,10 @@ NS_ASSUME_NONNULL_BEGIN /** Clears the `FloatBuffer` by setting all elements to zero */ - (void)clear; +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/build_ios_framework.sh b/mediapipe/tasks/ios/build_ios_framework.sh index 9f7fddd733..9bfba1427e 100755 --- a/mediapipe/tasks/ios/build_ios_framework.sh +++ b/mediapipe/tasks/ios/build_ios_framework.sh @@ -16,7 +16,9 @@ # Set the following variables as appropriate. # * BAZEL: path to bazel. defaults to the first one available in PATH # * FRAMEWORK_NAME: name of the iOS framework to be built. Currently the -# * accepted values are MediaPipeTasksCommon, MediaPipeTasksText, MediaPipeTasksVision. +# * accepted values are MediaPipeTasksCommon, MediaPipeTasksText, +# * MediaPipeTasksVision, MediaPipeTasksAudio, MediaPipeTasksGenAIC, +# * MediaPipeTasksGenAI. # * MPP_BUILD_VERSION: to specify the release version. defaults to 0.0.1-dev # * IS_RELEASE_BUILD: set as true if this build should be a release build # * ARCHIVE_FRAMEWORK: set as true if the framework should be archived @@ -56,12 +58,14 @@ case $FRAMEWORK_NAME in ;; "MediaPipeTasksText") ;; + "MediaPipeTasksAudio") + ;; "MediaPipeTasksGenAIC") ;; "MediaPipeTasksGenAI") ;; *) - echo "Wrong framework name. The following framework names are allowed: MediaPipeTasksText, MediaPipeTasksVision, MediaPipeTasksCommon, MediaPipeTasksGenAI, MediaPipeTasksGenAIC" + echo "Wrong framework name. The following framework names are allowed: MediaPipeTasksText, MediaPipeTasksVision, MediaPipeTasksAudio, MediaPipeTasksCommon, MediaPipeTasksGenAI, MediaPipeTasksGenAIC" exit 1 ;; esac diff --git a/mediapipe/tasks/ios/common/sources/MPPCommon.h b/mediapipe/tasks/ios/common/sources/MPPCommon.h index 08ef26c66f..fe29a93f13 100644 --- a/mediapipe/tasks/ios/common/sources/MPPCommon.h +++ b/mediapipe/tasks/ios/common/sources/MPPCommon.h @@ -108,14 +108,20 @@ typedef NS_ENUM(NSUInteger, MPPTasksErrorCode) { */ MPPTasksErrorCodeAudioRecordPermissionUndeterminedError = 18, - /** Indicates that `AudioRecord` is waiting for new mic input. */ + /** Indicates that `AudioRecord` is waiting for new microphone input. */ MPPTasksErrorCodeAudioRecordWaitingForNewMicInputError = 19, + /** + * Indicates that `AudioRecord` is not tapping the microphone. Operations permitted only while the + * microphone is being actively tapped can return this error. + */ + MPPTasksErrorCodeAudioRecordNotTappingMicError = 20, + /** The first error code in MPPTasksErrorCode (for internal use only). */ MPPTasksErrorCodeFirst = MPPTasksErrorCodeCancelledError, /** The last error code in MPPTasksErrorCode (for internal use only). */ - MPPTasksErrorCodeLast = MPPTasksErrorCodeAudioRecordWaitingForNewMicInputError, + MPPTasksErrorCodeLast = MPPTasksErrorCodeAudioRecordNotTappingMicError, } NS_SWIFT_NAME(TasksErrorCode); diff --git a/mediapipe/tasks/ios/test/audio/audio_classifier/BUILD b/mediapipe/tasks/ios/test/audio/audio_classifier/BUILD index 7352e04f51..dab76b836f 100644 --- a/mediapipe/tasks/ios/test/audio/audio_classifier/BUILD +++ b/mediapipe/tasks/ios/test/audio/audio_classifier/BUILD @@ -42,7 +42,6 @@ objc_library( ], deps = [ "//mediapipe/tasks/ios/audio/audio_classifier:MPPAudioClassifier", - "//mediapipe/tasks/ios/audio/core:MPPAudioData", "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//mediapipe/tasks/ios/test/audio/core/utils:AVAudioFileTestUtils", diff --git a/mediapipe/tasks/ios/test/audio/audio_classifier/MPPAudioClassifierTests.mm b/mediapipe/tasks/ios/test/audio/audio_classifier/MPPAudioClassifierTests.mm index acf500135a..cff5a5bc54 100644 --- a/mediapipe/tasks/ios/test/audio/audio_classifier/MPPAudioClassifierTests.mm +++ b/mediapipe/tasks/ios/test/audio/audio_classifier/MPPAudioClassifierTests.mm @@ -442,11 +442,57 @@ - (void)testClassifyWithAudioStreamModeAndOutOfOrderTimestampsFails { - (void)testClassifyWithAudioStreamModeSucceeds { [self classifyUsingYamnetAsyncAudioFileWithInfo:kSpeech16KHzMonoFileInfo - info:&_16kHZAudioStreamSucceedsTestDict]; + info:&_16kHZAudioStreamSucceedsTestDict]; [self classifyUsingYamnetAsyncAudioFileWithInfo:kSpeech48KHzMonoFileInfo - info:&_48kHZAudioStreamSucceedsTestDict]; + info:&_48kHZAudioStreamSucceedsTestDict]; } +#pragma mark Audio Record Tests + +- (void)testCreateAudioRecordSucceeds { + const NSUInteger channelCount = 1; + const NSUInteger bufferLength = channelCount * kYamnetSampleCount; + + NSError *error; + MPPAudioRecord *audioRecord = + [MPPAudioClassifier createAudioRecordWithChannelCount:channelCount + sampleRate:kYamnetSampleRate + bufferLength:kYamnetSampleCount * channelCount + error:&error]; + + XCTAssertNotNil(audioRecord); + XCTAssertNil(error); + XCTAssertEqual(audioRecord.audioDataFormat.channelCount, channelCount); + XCTAssertEqual(audioRecord.audioDataFormat.sampleRate, kYamnetSampleRate); + XCTAssertEqual(audioRecord.bufferLength, bufferLength); +} + +// Test for error propogation from audio record creation. +- (void)testCreateAudioRecordWithInvalidChannelCountFails { + const NSUInteger channelCount = 3; + + NSError *error; + MPPAudioRecord *audioRecord = + [MPPAudioClassifier createAudioRecordWithChannelCount:channelCount + sampleRate:kYamnetSampleRate + bufferLength:kYamnetSampleCount * channelCount + error:&error]; + XCTAssertNil(audioRecord); + + NSError *expectedError = [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The channel count provided does not match the " + @"supported channel count. Only channels counts " + @"in the range [1 : 2] are supported" + }]; + + AssertEqualErrors(error, expectedError); +} + +#pragma mark MPPAudioClassifierStreamDelegate + - (void)audioClassifier:(MPPAudioClassifier *)audioClassifier didFinishClassificationWithResult:(MPPAudioClassifierResult *)result timestampInMilliseconds:(NSInteger)timestampInMilliseconds @@ -484,8 +530,7 @@ - (void)audioClassifier:(MPPAudioClassifier *)audioClassifier // info is strong here since address of global variables will be passed to this function. By default // `NSDictionary **` will be `NSDictionary * __autoreleasing *. - (void)classifyUsingYamnetAsyncAudioFileWithInfo:(MPPFileInfo *)audioFileInfo - info:(NSDictionary *__strong *) - info { + info:(NSDictionary *__strong *)info { MPPAudioClassifier *audioClassifier = [self audioClassifierInStreamModeWithModelFileInfo:kYamnetModelFileInfo]; diff --git a/mediapipe/tasks/ios/test/audio/core/MPPAudioDataTests.mm b/mediapipe/tasks/ios/test/audio/core/MPPAudioDataTests.mm index 562730feea..9b90ce1516 100644 --- a/mediapipe/tasks/ios/test/audio/core/MPPAudioDataTests.mm +++ b/mediapipe/tasks/ios/test/audio/core/MPPAudioDataTests.mm @@ -70,6 +70,36 @@ + (AVAudioPCMBuffer *)bufferFromInputBuffer:(AVAudioPCMBuffer *)pcmBuffer error:(NSError **)error; - (BOOL)loadAudioPCMBuffer:(AVAudioPCMBuffer *)pcmBuffer error:(NSError **)error; + +- (nullable MPPFloatBuffer *)internalReadAtOffset:(NSUInteger)offset + withLength:(NSUInteger)length + error:(NSError **)error; +@end + +@interface MPPAudioData () +- (BOOL)loadRingBufferWithAudioRecordBuffer:audioRecordBuffer error:(NSError **)error; +- (BOOL)isValidAudioRecordFormat:(MPPAudioDataFormat *)format error:(NSError **)error; +@end + +@interface MPPAudioData (Tests) +- (BOOL)mockLoadAudioRecord:(MPPAudioRecord *)audioRecord error:(NSError **)error; +@end + +@implementation MPPAudioData (Tests) +// Mocks the logic of `loadAudioRecord` for tests to avoid audio engine running state checks. +- (BOOL)mockLoadAudioRecord:(MPPAudioRecord *)audioRecord error:(NSError **)error { + if (![self isValidAudioRecordFormat:audioRecord.audioDataFormat error:error]) { + return NO; + } + + // Invoking `internalReadAtOffset` instead of `readAtOffset` to avoid audio engine running state + // checks. + MPPFloatBuffer *audioRecordBuffer = [audioRecord internalReadAtOffset:0 + withLength:audioRecord.bufferLength + error:error]; + + return [self loadRingBufferWithAudioRecordBuffer:audioRecordBuffer error:error]; +} @end @interface MPPAudioDataTests : XCTestCase @@ -392,11 +422,11 @@ + (void)assertSuccessOfLoadAudioData:(MPPAudioData *)audioData fromAudioRecord:(MPPAudioRecord *)audioRecord { MPPFloatBuffer *previousStateOfAudioData = audioData.buffer; - XCTAssertTrue([audioData loadAudioRecord:audioRecord error:nil]); + XCTAssertTrue([audioData mockLoadAudioRecord:audioRecord error:nil]); - MPPFloatBuffer *audioRecordBuffer = [audioRecord readAtOffset:0 - withLength:audioRecord.bufferLength - error:nil]; + MPPFloatBuffer *audioRecordBuffer = [audioRecord internalReadAtOffset:0 + withLength:audioRecord.bufferLength + error:nil]; [MPPAudioDataTests assertDataOfFloatBuffer:audioData.buffer containsInOrderSamplesFromPreviousStateOfFloatBuffer:previousStateOfAudioData diff --git a/mediapipe/tasks/ios/test/audio/core/MPPAudioRecordTests.mm b/mediapipe/tasks/ios/test/audio/core/MPPAudioRecordTests.mm index 7c44a1a089..aa316a19a4 100644 --- a/mediapipe/tasks/ios/test/audio/core/MPPAudioRecordTests.mm +++ b/mediapipe/tasks/ios/test/audio/core/MPPAudioRecordTests.mm @@ -62,6 +62,10 @@ + (AVAudioPCMBuffer *)bufferFromInputBuffer:(AVAudioPCMBuffer *)pcmBuffer error:(NSError **)error; - (BOOL)loadAudioPCMBuffer:(AVAudioPCMBuffer *)pcmBuffer error:(NSError **)error; + +- (nullable MPPFloatBuffer *)internalReadAtOffset:(NSUInteger)offset + withLength:(NSUInteger)length + error:(NSError **)error; @end @implementation MPPAudioRecordTests @@ -239,7 +243,7 @@ - (void)testReadAudioRecordAtOffsetFailsWithIndexOutOfBounds { const NSUInteger offset = 4; const NSUInteger length = 100; NSError *error; - [audioRecord readAtOffset:offset withLength:length error:&error]; + [audioRecord internalReadAtOffset:offset withLength:length error:&error]; NSError *expectedError = [NSError errorWithDomain:kExpectedErrorDomain @@ -322,9 +326,8 @@ + (AVAudioPCMBuffer *)bufferFromAudioFileWithInfo:(MPPFileInfo *)fileInfo + (void)assertSuccessOfLoadAudioRecord:(MPPAudioRecord *)audioRecord withPCMBuffer:(AVAudioPCMBuffer *)bufferInAudioRecordFormat { - MPPFloatBuffer *previousAudioRecordBuffer = [audioRecord readAtOffset:0 - withLength:audioRecord.bufferLength - error:nil]; + MPPFloatBuffer *previousAudioRecordBuffer = + [audioRecord internalReadAtOffset:0 withLength:audioRecord.bufferLength error:nil]; XCTAssertTrue([audioRecord loadAudioPCMBuffer:bufferInAudioRecordFormat error:nil]); @@ -360,7 +363,9 @@ + (MPPFloatBuffer *)readFullLengthBufferOfAudioRecord:(MPPAudioRecord *)audioRec + (MPPFloatBuffer *)readAudioRecord:(MPPAudioRecord *)audioRecord atOffset:(NSUInteger)offset length:(NSUInteger)length { - MPPFloatBuffer *audioRecordBuffer = [audioRecord readAtOffset:offset withLength:length error:nil]; + MPPFloatBuffer *audioRecordBuffer = [audioRecord internalReadAtOffset:offset + withLength:length + error:nil]; XCTAssertNotNil(audioRecordBuffer); XCTAssertEqual(audioRecordBuffer.length, length); return audioRecordBuffer;