diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 466321a..37e7cd7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,6 +35,18 @@ To run the example app on iOS: yarn example ios ``` +To enable CoreML in example app, you need to convert the model first: + +- Remove `whisper.cpp/models/ggml-base.en-encoder.mlmodelc` if it is empty (It's generated by `yarn bootstrap`) +- Please read [Core ML support](../README.md#core-ml-support) section of README and [Core ML support](https://github.com/ggerganov/whisper.cpp#core-ml-support) section of whisper.cpp. + +Then modify `App.js` to use local bundle assets: + +```patch +- const USE_DOWNLOAD_MODEL = true ++ const USE_DOWNLOAD_MODEL = false +``` + Make sure your code passes TypeScript and ESLint. Run the following to verify: ```sh diff --git a/README.md b/README.md index a00a881..e7cbd5c 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,30 @@ In Android, you may need to request the microphone permission by [`PermissionAnd The documentation is not ready yet, please see the comments of [index](./src/index.ts) file for more details at the moment. +## Core ML support + +__*Platform: iOS 15.0+, tvOS 15.0+*__ + +To use Core ML on iOS, you will need to have the Core ML model files. + +The `.mlmodelc` model files is load depend on the ggml model file path. For example, if your ggml model path is `ggml-base.en.bin`, the Core ML model path will be `ggml-base.en-encoder.mlmodelc`. Please note that the ggml model is still needed as decoder or encoder fallback. + +Currently there is no official way to get the Core ML models by URL, you will need to convert the ggml model to Core ML model folder by yourself. Please see [Core ML Support](https://github.com/ggerganov/whisper.cpp#core-ml-support) of whisper.cpp for more details. + +During the `.mlmodelc` is a directory, you will need to download 5 files: + +```json5 +[ + 'model.mil', + 'metadata.json', + 'coremldata.bin', + 'weights/weights.bin', + 'analysis/coremldata.bin', +] +``` + +Or just add them to your app's bundle resourcesas, like the example app does, but this would increase the app size significantly. + ## Run with example The example app is using [react-native-fs](https://github.com/itinance/react-native-fs) to download the model file and audio file. diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 0000000..5771990 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1,2 @@ +*.mlmodelc +**/assets/*.bin diff --git a/example/ios/Podfile b/example/ios/Podfile index 89ced2a..26e95e4 100644 --- a/example/ios/Podfile +++ b/example/ios/Podfile @@ -22,6 +22,9 @@ if linkage != nil end target 'RNWhisperExample' do + # Tip: You can use RNWHISPER_DISABLE_COREML = '1' to disable CoreML support. + ENV['RNWHISPER_DISABLE_COREML'] = '0' + config = use_native_modules! # Flags change depending on the env values. diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 14fd9e4..f475e6f 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -2,14 +2,14 @@ PODS: - boost (1.76.0) - CocoaAsyncSocket (7.6.5) - DoubleConversion (1.1.6) - - FBLazyVector (0.71.7) - - FBReactNativeSpec (0.71.7): + - FBLazyVector (0.71.8) + - FBReactNativeSpec (0.71.8): - RCT-Folly (= 2021.07.22.00) - - RCTRequired (= 0.71.7) - - RCTTypeSafety (= 0.71.7) - - React-Core (= 0.71.7) - - React-jsi (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) + - RCTRequired (= 0.71.8) + - RCTTypeSafety (= 0.71.8) + - React-Core (= 0.71.8) + - React-jsi (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) - Flipper (0.125.0): - Flipper-Folly (~> 2.6) - Flipper-RSocket (~> 1.4) @@ -73,9 +73,9 @@ PODS: - FlipperKit/FlipperKitNetworkPlugin - fmt (6.2.1) - glog (0.3.5) - - hermes-engine (0.71.6): - - hermes-engine/Pre-built (= 0.71.6) - - hermes-engine/Pre-built (0.71.6) + - hermes-engine (0.71.8): + - hermes-engine/Pre-built (= 0.71.8) + - hermes-engine/Pre-built (0.71.8) - libevent (2.1.12) - OpenSSL-Universal (1.1.1100) - RCT-Folly (2021.07.22.00): @@ -95,26 +95,26 @@ PODS: - fmt (~> 6.2.1) - glog - libevent - - RCTRequired (0.71.7) - - RCTTypeSafety (0.71.7): - - FBLazyVector (= 0.71.7) - - RCTRequired (= 0.71.7) - - React-Core (= 0.71.7) - - React (0.71.7): - - React-Core (= 0.71.7) - - React-Core/DevSupport (= 0.71.7) - - React-Core/RCTWebSocket (= 0.71.7) - - React-RCTActionSheet (= 0.71.7) - - React-RCTAnimation (= 0.71.7) - - React-RCTBlob (= 0.71.7) - - React-RCTImage (= 0.71.7) - - React-RCTLinking (= 0.71.7) - - React-RCTNetwork (= 0.71.7) - - React-RCTSettings (= 0.71.7) - - React-RCTText (= 0.71.7) - - React-RCTVibration (= 0.71.7) - - React-callinvoker (0.71.7) - - React-Codegen (0.71.7): + - RCTRequired (0.71.8) + - RCTTypeSafety (0.71.8): + - FBLazyVector (= 0.71.8) + - RCTRequired (= 0.71.8) + - React-Core (= 0.71.8) + - React (0.71.8): + - React-Core (= 0.71.8) + - React-Core/DevSupport (= 0.71.8) + - React-Core/RCTWebSocket (= 0.71.8) + - React-RCTActionSheet (= 0.71.8) + - React-RCTAnimation (= 0.71.8) + - React-RCTBlob (= 0.71.8) + - React-RCTImage (= 0.71.8) + - React-RCTLinking (= 0.71.8) + - React-RCTNetwork (= 0.71.8) + - React-RCTSettings (= 0.71.8) + - React-RCTText (= 0.71.8) + - React-RCTVibration (= 0.71.8) + - React-callinvoker (0.71.8) + - React-Codegen (0.71.8): - FBReactNativeSpec - hermes-engine - RCT-Folly @@ -125,294 +125,294 @@ PODS: - React-jsiexecutor - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - React-Core (0.71.7): + - React-Core (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-Core/Default (= 0.71.7) - - React-cxxreact (= 0.71.7) + - React-Core/Default (= 0.71.8) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/CoreModulesHeaders (0.71.7): + - React-Core/CoreModulesHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/Default (0.71.7): + - React-Core/Default (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/DevSupport (0.71.7): + - React-Core/DevSupport (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-Core/Default (= 0.71.7) - - React-Core/RCTWebSocket (= 0.71.7) - - React-cxxreact (= 0.71.7) + - React-Core/Default (= 0.71.8) + - React-Core/RCTWebSocket (= 0.71.8) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-jsinspector (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-jsinspector (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTActionSheetHeaders (0.71.7): + - React-Core/RCTActionSheetHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTAnimationHeaders (0.71.7): + - React-Core/RCTAnimationHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTBlobHeaders (0.71.7): + - React-Core/RCTBlobHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTImageHeaders (0.71.7): + - React-Core/RCTImageHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTLinkingHeaders (0.71.7): + - React-Core/RCTLinkingHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTNetworkHeaders (0.71.7): + - React-Core/RCTNetworkHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTSettingsHeaders (0.71.7): + - React-Core/RCTSettingsHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTTextHeaders (0.71.7): + - React-Core/RCTTextHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTVibrationHeaders (0.71.7): + - React-Core/RCTVibrationHeaders (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - React-Core/Default - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-Core/RCTWebSocket (0.71.7): + - React-Core/RCTWebSocket (0.71.8): - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-Core/Default (= 0.71.7) - - React-cxxreact (= 0.71.7) + - React-Core/Default (= 0.71.8) + - React-cxxreact (= 0.71.8) - React-hermes - - React-jsi (= 0.71.7) - - React-jsiexecutor (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-jsi (= 0.71.8) + - React-jsiexecutor (= 0.71.8) + - React-perflogger (= 0.71.8) - Yoga - - React-CoreModules (0.71.7): + - React-CoreModules (0.71.8): - RCT-Folly (= 2021.07.22.00) - - RCTTypeSafety (= 0.71.7) - - React-Codegen (= 0.71.7) - - React-Core/CoreModulesHeaders (= 0.71.7) - - React-jsi (= 0.71.7) + - RCTTypeSafety (= 0.71.8) + - React-Codegen (= 0.71.8) + - React-Core/CoreModulesHeaders (= 0.71.8) + - React-jsi (= 0.71.8) - React-RCTBlob - - React-RCTImage (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-cxxreact (0.71.7): + - React-RCTImage (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-cxxreact (0.71.8): - boost (= 1.76.0) - DoubleConversion - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-callinvoker (= 0.71.7) - - React-jsi (= 0.71.7) - - React-jsinspector (= 0.71.7) - - React-logger (= 0.71.7) - - React-perflogger (= 0.71.7) - - React-runtimeexecutor (= 0.71.7) - - React-hermes (0.71.7): + - React-callinvoker (= 0.71.8) + - React-jsi (= 0.71.8) + - React-jsinspector (= 0.71.8) + - React-logger (= 0.71.8) + - React-perflogger (= 0.71.8) + - React-runtimeexecutor (= 0.71.8) + - React-hermes (0.71.8): - DoubleConversion - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - RCT-Folly/Futures (= 2021.07.22.00) - - React-cxxreact (= 0.71.7) + - React-cxxreact (= 0.71.8) - React-jsi - - React-jsiexecutor (= 0.71.7) - - React-jsinspector (= 0.71.7) - - React-perflogger (= 0.71.7) - - React-jsi (0.71.7): + - React-jsiexecutor (= 0.71.8) + - React-jsinspector (= 0.71.8) + - React-perflogger (= 0.71.8) + - React-jsi (0.71.8): - boost (= 1.76.0) - DoubleConversion - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-jsiexecutor (0.71.7): + - React-jsiexecutor (0.71.8): - DoubleConversion - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-cxxreact (= 0.71.7) - - React-jsi (= 0.71.7) - - React-perflogger (= 0.71.7) - - React-jsinspector (0.71.7) - - React-logger (0.71.7): + - React-cxxreact (= 0.71.8) + - React-jsi (= 0.71.8) + - React-perflogger (= 0.71.8) + - React-jsinspector (0.71.8) + - React-logger (0.71.8): - glog - - React-perflogger (0.71.7) - - React-RCTActionSheet (0.71.7): - - React-Core/RCTActionSheetHeaders (= 0.71.7) - - React-RCTAnimation (0.71.7): + - React-perflogger (0.71.8) + - React-RCTActionSheet (0.71.8): + - React-Core/RCTActionSheetHeaders (= 0.71.8) + - React-RCTAnimation (0.71.8): - RCT-Folly (= 2021.07.22.00) - - RCTTypeSafety (= 0.71.7) - - React-Codegen (= 0.71.7) - - React-Core/RCTAnimationHeaders (= 0.71.7) - - React-jsi (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-RCTAppDelegate (0.71.7): + - RCTTypeSafety (= 0.71.8) + - React-Codegen (= 0.71.8) + - React-Core/RCTAnimationHeaders (= 0.71.8) + - React-jsi (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-RCTAppDelegate (0.71.8): - RCT-Folly - RCTRequired - RCTTypeSafety - React-Core - ReactCommon/turbomodule/core - - React-RCTBlob (0.71.7): + - React-RCTBlob (0.71.8): - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-Codegen (= 0.71.7) - - React-Core/RCTBlobHeaders (= 0.71.7) - - React-Core/RCTWebSocket (= 0.71.7) - - React-jsi (= 0.71.7) - - React-RCTNetwork (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-RCTImage (0.71.7): + - React-Codegen (= 0.71.8) + - React-Core/RCTBlobHeaders (= 0.71.8) + - React-Core/RCTWebSocket (= 0.71.8) + - React-jsi (= 0.71.8) + - React-RCTNetwork (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-RCTImage (0.71.8): - RCT-Folly (= 2021.07.22.00) - - RCTTypeSafety (= 0.71.7) - - React-Codegen (= 0.71.7) - - React-Core/RCTImageHeaders (= 0.71.7) - - React-jsi (= 0.71.7) - - React-RCTNetwork (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-RCTLinking (0.71.7): - - React-Codegen (= 0.71.7) - - React-Core/RCTLinkingHeaders (= 0.71.7) - - React-jsi (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-RCTNetwork (0.71.7): + - RCTTypeSafety (= 0.71.8) + - React-Codegen (= 0.71.8) + - React-Core/RCTImageHeaders (= 0.71.8) + - React-jsi (= 0.71.8) + - React-RCTNetwork (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-RCTLinking (0.71.8): + - React-Codegen (= 0.71.8) + - React-Core/RCTLinkingHeaders (= 0.71.8) + - React-jsi (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-RCTNetwork (0.71.8): - RCT-Folly (= 2021.07.22.00) - - RCTTypeSafety (= 0.71.7) - - React-Codegen (= 0.71.7) - - React-Core/RCTNetworkHeaders (= 0.71.7) - - React-jsi (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-RCTSettings (0.71.7): + - RCTTypeSafety (= 0.71.8) + - React-Codegen (= 0.71.8) + - React-Core/RCTNetworkHeaders (= 0.71.8) + - React-jsi (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-RCTSettings (0.71.8): - RCT-Folly (= 2021.07.22.00) - - RCTTypeSafety (= 0.71.7) - - React-Codegen (= 0.71.7) - - React-Core/RCTSettingsHeaders (= 0.71.7) - - React-jsi (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-RCTText (0.71.7): - - React-Core/RCTTextHeaders (= 0.71.7) - - React-RCTVibration (0.71.7): + - RCTTypeSafety (= 0.71.8) + - React-Codegen (= 0.71.8) + - React-Core/RCTSettingsHeaders (= 0.71.8) + - React-jsi (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-RCTText (0.71.8): + - React-Core/RCTTextHeaders (= 0.71.8) + - React-RCTVibration (0.71.8): - RCT-Folly (= 2021.07.22.00) - - React-Codegen (= 0.71.7) - - React-Core/RCTVibrationHeaders (= 0.71.7) - - React-jsi (= 0.71.7) - - ReactCommon/turbomodule/core (= 0.71.7) - - React-runtimeexecutor (0.71.7): - - React-jsi (= 0.71.7) - - ReactCommon/turbomodule/bridging (0.71.7): + - React-Codegen (= 0.71.8) + - React-Core/RCTVibrationHeaders (= 0.71.8) + - React-jsi (= 0.71.8) + - ReactCommon/turbomodule/core (= 0.71.8) + - React-runtimeexecutor (0.71.8): + - React-jsi (= 0.71.8) + - ReactCommon/turbomodule/bridging (0.71.8): - DoubleConversion - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-callinvoker (= 0.71.7) - - React-Core (= 0.71.7) - - React-cxxreact (= 0.71.7) - - React-jsi (= 0.71.7) - - React-logger (= 0.71.7) - - React-perflogger (= 0.71.7) - - ReactCommon/turbomodule/core (0.71.7): + - React-callinvoker (= 0.71.8) + - React-Core (= 0.71.8) + - React-cxxreact (= 0.71.8) + - React-jsi (= 0.71.8) + - React-logger (= 0.71.8) + - React-perflogger (= 0.71.8) + - ReactCommon/turbomodule/core (0.71.8): - DoubleConversion - glog - hermes-engine - RCT-Folly (= 2021.07.22.00) - - React-callinvoker (= 0.71.7) - - React-Core (= 0.71.7) - - React-cxxreact (= 0.71.7) - - React-jsi (= 0.71.7) - - React-logger (= 0.71.7) - - React-perflogger (= 0.71.7) + - React-callinvoker (= 0.71.8) + - React-Core (= 0.71.8) + - React-cxxreact (= 0.71.8) + - React-jsi (= 0.71.8) + - React-logger (= 0.71.8) + - React-perflogger (= 0.71.8) - RNFS (2.20.0): - React-Core - SocketRocket (0.6.0) @@ -581,8 +581,8 @@ SPEC CHECKSUMS: boost: 57d2868c099736d80fcd648bf211b4431e51a558 CocoaAsyncSocket: 065fd1e645c7abab64f7a6a2007a48038fdc6a99 DoubleConversion: 5189b271737e1565bdce30deb4a08d647e3f5f54 - FBLazyVector: a89a0525bc7ca174675045c2b492b5280d5a2470 - FBReactNativeSpec: 7714e6bc1e9ea23df6c4cb42f0b2fd9c6a3a559c + FBLazyVector: f637f31eacba90d4fdeff3fa41608b8f361c173b + FBReactNativeSpec: 0d9a4f4de7ab614c49e98c00aedfd3bfbda33d59 Flipper: 26fc4b7382499f1281eb8cb921e5c3ad6de91fe0 Flipper-Boost-iOSX: fd1e2b8cbef7e662a122412d7ac5f5bea715403c Flipper-DoubleConversion: 2dc99b02f658daf147069aad9dbd29d8feb06d30 @@ -594,42 +594,42 @@ SPEC CHECKSUMS: FlipperKit: cbdee19bdd4e7f05472a66ce290f1b729ba3cb86 fmt: ff9d55029c625d3757ed641535fd4a75fedc7ce9 glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b - hermes-engine: b434cea529ad0152c56c7cb6486b0c4c0b23b5de + hermes-engine: 47986d26692ae75ee7a17ab049caee8864f855de libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913 OpenSSL-Universal: ebc357f1e6bc71fa463ccb2fe676756aff50e88c RCT-Folly: 424b8c9a7a0b9ab2886ffe9c3b041ef628fd4fb1 - RCTRequired: 5a4a30ac20c86eeadd6844a9328f78d4168cf9b2 - RCTTypeSafety: 279fc5861a89f0f37db3a585f27f971485b4b734 - React: 88307a9be3bd0e71a6822271cf28b84a587fb97f - React-callinvoker: 35fb980c454104ebe82f0afb9826830089248e08 - React-Codegen: a8dbde3b7476d5c19437d2adb9e8ea1b426b9595 - React-Core: 385cb6fa78762c6409ff39faeb0dd9ad664b6e84 - React-CoreModules: c2b7db313b04d9b71954ffd55d0c2e46bc40e9fb - React-cxxreact: 845fefb889132e5d004ff818f7a599e32c52e7d6 - React-hermes: 86135f35e1dd2dfccfb97afe96d0c06f6a3970c4 - React-jsi: 39c116aa6c3d6f3d9874eff6998a670b47882a28 - React-jsiexecutor: eaa5f71eb8f6861cf0e57f1a0f52aeb020d9e18e - React-jsinspector: 9885f6f94d231b95a739ef7bb50536fb87ce7539 - React-logger: 3f8ebad1be1bf3299d1ab6d7f971802d7395c7ef - React-perflogger: 2d505bbe298e3b7bacdd9e542b15535be07220f6 - React-RCTActionSheet: 0e96e4560bd733c9b37efbf68f5b1a47615892fb - React-RCTAnimation: fd138e26f120371c87e406745a27535e2c8a04ef - React-RCTAppDelegate: 4a9fd1230a98dc3d4382f8a934dc9f50834d8335 - React-RCTBlob: 38a7185f06a0ce8153a023e63b406a28d67b955d - React-RCTImage: 92b0966e7c1cadda889e961c474397ad5180e194 - React-RCTLinking: b80f8d0c6e94c54294b0048def51f57eaa9a27af - React-RCTNetwork: 491b0c65ac22edbd6695d12d084b4943103b009b - React-RCTSettings: 97af3e8abe0023349ec015910df3bda1a0380117 - React-RCTText: 33c85753bd714d527d2ae538dc56ec24c6783d84 - React-RCTVibration: 08f132cad9896458776f37c112e71d60aef1c6ae - React-runtimeexecutor: c5c89f8f543842dd864b63ded1b0bbb9c9445328 - ReactCommon: dbfbe2f7f3c5ce4ce44f43f2fd0d5950d1eb67c5 + RCTRequired: 8af6a32dfc2b65ec82193c2dee6e1011ff22ac2a + RCTTypeSafety: bee9dd161c175896c680d47ef1d9eaacf2b587f4 + React: d850475db9ba8006a8b875d79e1e0d6ac8a0f8b6 + React-callinvoker: 6a0c75475ddc17c9ed54e4ff0478074a18fd7ab5 + React-Codegen: 786571642e87add634e7f4d299c85314ec6cc158 + React-Core: 1adfab153f59e4f56e09b97a153089f466d7b8aa + React-CoreModules: 958d236715415d4ccdd5fa35c516cf0356637393 + React-cxxreact: 2e7a6283807ce8755c3d501735acd400bec3b5cd + React-hermes: 8102c3112ba32207c3052619be8cfae14bf99d84 + React-jsi: dd29264f041a587e91f994e4be97e86c127742b2 + React-jsiexecutor: 747911ab5921641b4ed7e4900065896597142125 + React-jsinspector: c712f9e3bb9ba4122d6b82b4f906448b8a281580 + React-logger: 342f358b8decfbf8f272367f4eacf4b6154061be + React-perflogger: d21f182895de9d1b077f8a3cd00011095c8c9100 + React-RCTActionSheet: 0151f83ef92d2a7139bba7dfdbc8066632a6d47b + React-RCTAnimation: 5ec9c0705bb2297549c120fe6473aa3e4a01e215 + React-RCTAppDelegate: 9895fd1b6d1176d88c4b10ddc169b2e1300c91f0 + React-RCTBlob: f3634eb45b6e7480037655e1ca93d1136ac984dd + React-RCTImage: 3c12cb32dec49549ae62ed6cba4018db43841ffc + React-RCTLinking: 310e930ee335ef25481b4a173d9edb64b77895f9 + React-RCTNetwork: b6837841fe88303b0c04c1e3c01992b30f1f5498 + React-RCTSettings: 600d91fe25fa7c16b0ff891304082440f2904b89 + React-RCTText: a0a19f749088280c6def5397ed6211b811e7eef3 + React-RCTVibration: 43ffd976a25f6057a7cf95ea3648ba4e00287f89 + React-runtimeexecutor: 7c51ae9d4b3e9608a2366e39ccaa606aa551b9ed + ReactCommon: 85c98ab0a509e70bf5ee5d9715cf68dbf495b84c RNFS: 4ac0f0ea233904cb798630b3c077808c06931688 SocketRocket: fccef3f9c5cedea1353a9ef6ada904fde10d6608 - whisper-rn: 860b5438336a0cead4ac179269db6b0d6c94484b - Yoga: d56980c8914db0b51692f55533409e844b66133c + whisper-rn: 9ce387a4756541f9549f5d4c2f7ee84bf78c420e + Yoga: 065f0b74dba4832d6e328238de46eb72c5de9556 YogaKit: f782866e155069a2cca2517aafea43200b01fd5a -PODFILE CHECKSUM: 7374ab149e8c2afbc0b2e12093c546837269e6e9 +PODFILE CHECKSUM: 74c803fe52ead33303df8797ef6aec85cc2771da COCOAPODS: 1.11.3 diff --git a/example/ios/RNWhisperExample.xcodeproj/project.pbxproj b/example/ios/RNWhisperExample.xcodeproj/project.pbxproj index 886d83a..6018978 100644 --- a/example/ios/RNWhisperExample.xcodeproj/project.pbxproj +++ b/example/ios/RNWhisperExample.xcodeproj/project.pbxproj @@ -13,6 +13,8 @@ 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; }; 13B07FC11A68108700A75B9A /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; }; 7699B88040F8A987B510C191 /* libPods-RNWhisperExample-RNWhisperExampleTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 19F6CBCC0A4E27FBF8BF4A61 /* libPods-RNWhisperExample-RNWhisperExampleTests.a */; }; + 7FE3421E2A0B66B10015A058 /* ggml-base.en-encoder.mlmodelc in Resources */ = {isa = PBXBuildFile; fileRef = 7FE3421D2A0B66B10015A058 /* ggml-base.en-encoder.mlmodelc */; }; + 7FE342202A0B672A0015A058 /* ggml-base.en.bin in Resources */ = {isa = PBXBuildFile; fileRef = 7FE3421F2A0B672A0015A058 /* ggml-base.en.bin */; }; 81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; }; /* End PBXBuildFile section */ @@ -41,6 +43,8 @@ 5709B34CF0A7D63546082F79 /* Pods-RNWhisperExample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RNWhisperExample.release.xcconfig"; path = "Target Support Files/Pods-RNWhisperExample/Pods-RNWhisperExample.release.xcconfig"; sourceTree = ""; }; 5B7EB9410499542E8C5724F5 /* Pods-RNWhisperExample-RNWhisperExampleTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RNWhisperExample-RNWhisperExampleTests.debug.xcconfig"; path = "Target Support Files/Pods-RNWhisperExample-RNWhisperExampleTests/Pods-RNWhisperExample-RNWhisperExampleTests.debug.xcconfig"; sourceTree = ""; }; 5DCACB8F33CDC322A6C60F78 /* libPods-RNWhisperExample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-RNWhisperExample.a"; sourceTree = BUILT_PRODUCTS_DIR; }; + 7FE3421D2A0B66B10015A058 /* ggml-base.en-encoder.mlmodelc */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "ggml-base.en-encoder.mlmodelc"; path = "../../whisper.cpp/models/ggml-base.en-encoder.mlmodelc"; sourceTree = ""; }; + 7FE3421F2A0B672A0015A058 /* ggml-base.en.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = "ggml-base.en.bin"; path = "../../whisper.cpp/models/ggml-base.en.bin"; sourceTree = ""; }; 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = RNWhisperExample/LaunchScreen.storyboard; sourceTree = ""; }; 89C6BE57DB24E9ADA2F236DE /* Pods-RNWhisperExample-RNWhisperExampleTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RNWhisperExample-RNWhisperExampleTests.release.xcconfig"; path = "Target Support Files/Pods-RNWhisperExample-RNWhisperExampleTests/Pods-RNWhisperExample-RNWhisperExampleTests.release.xcconfig"; sourceTree = ""; }; ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; }; @@ -86,6 +90,8 @@ 13B07FAE1A68108700A75B9A /* RNWhisperExample */ = { isa = PBXGroup; children = ( + 7FE3421D2A0B66B10015A058 /* ggml-base.en-encoder.mlmodelc */, + 7FE3421F2A0B672A0015A058 /* ggml-base.en.bin */, 13B07FAF1A68108700A75B9A /* AppDelegate.h */, 13B07FB01A68108700A75B9A /* AppDelegate.mm */, 13B07FB51A68108700A75B9A /* Images.xcassets */, @@ -242,8 +248,10 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + 7FE3421E2A0B66B10015A058 /* ggml-base.en-encoder.mlmodelc in Resources */, 81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */, 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */, + 7FE342202A0B672A0015A058 /* ggml-base.en.bin in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/example/src/App.js b/example/src/App.js index 875fb4a..f80bf0d 100644 --- a/example/src/App.js +++ b/example/src/App.js @@ -45,20 +45,6 @@ const styles = StyleSheet.create({ logText: { fontSize: 12, color: '#333' }, }) -const mode = process.env.NODE_ENV === 'development' ? 'debug' : 'release' - -const modelURL = - 'https://huggingface.co/datasets/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin' -const sampleURL = - 'https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav' - -const fileDir = `${RNFS.DocumentDirectoryPath}/whisper` - -console.log('[App] fileDir', fileDir) - -const modelFilePath = `${fileDir}/base.en` -const sampleFilePath = `${fileDir}/jfk.wav` - function toTimestamp(t, comma = false) { let msec = t * 10 const hr = Math.floor(msec / (1000 * 60 * 60)) @@ -80,9 +66,53 @@ function toTimestamp(t, comma = false) { return timestamp } +const mode = process.env.NODE_ENV === 'development' ? 'debug' : 'release' + +const modelURL = + 'https://huggingface.co/datasets/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin' +const sampleURL = + 'https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav' + +const fileDir = `${RNFS.DocumentDirectoryPath}/whisper` + +console.log('[App] fileDir', fileDir) + +const modelFilePath = `${fileDir}/ggml-base.en.bin` +const sampleFilePath = `${fileDir}/jfk.wav` + +const createDir = async (log) => { + if (!(await RNFS.exists(fileDir))) { + log('Create dir', fileDir) + await RNFS.mkdir(fileDir) + } +} + const filterPath = (path) => path.replace(RNFS.DocumentDirectoryPath, '') +const downloadModel = async (log, progress) => { + await createDir(log) + if (await RNFS.exists(modelFilePath)) { + log('Model already exists:') + log(filterPath(modelFilePath)) + } else { + log('Start Download Model to:') + log(filterPath(modelFilePath)) + await RNFS.downloadFile({ + fromUrl: modelURL, + toFile: modelFilePath, + progressInterval: 1000, + begin: () => {}, + progress, + }).promise + log('Downloaded model file:') + log(filterPath(modelFilePath)) + } +} + +// Set to false to use the model from the bundle resources +const USE_DOWNLOAD_MODEL = true + export default function App() { const [whisperContext, setWhisperContext] = useState(null) const [logs, setLogs] = useState([]) @@ -93,13 +123,6 @@ export default function App() { setLogs((prev) => [...prev, messages.join(' ')]) }, []) - const createDir = useCallback(async () => { - if (!(await RNFS.exists(fileDir))) { - log('Create dir', fileDir) - await RNFS.mkdir(fileDir) - } - }, [log]) - const progress = useCallback( ({ contentLength, bytesWritten }) => { const written = bytesWritten >= 0 ? bytesWritten : 0 @@ -121,32 +144,23 @@ export default function App() { setWhisperContext(null) log('Released previous context') } - await createDir() - if (await RNFS.exists(modelFilePath)) { - log('Model already exists:') - log(filterPath(modelFilePath)) + let options + if (USE_DOWNLOAD_MODEL) { + await downloadModel(log, progress) + options = { + filePath: modelFilePath, + isBundleAsset: false, + } } else { - log('Start Download Model to:') - log(filterPath(modelFilePath)) - await RNFS.downloadFile({ - fromUrl: modelURL, - toFile: modelFilePath, - progressInterval: 1000, - begin: () => {}, - progress, - }).promise - log('Downloaded model file:') - log(filterPath(modelFilePath)) + options = { + // Use the bundle resource (Need add model to Xcode project / Android assets) + filePath: 'ggml-base.en.bin', + isBundleAsset: true, + } } log('Initialize context...') const startTime = Date.now() - const ctx = await initWhisper({ - // Use downloaded modelFilePath - filePath: modelFilePath, - // Or you can choose to use the asset (Need to add to Xcode project / Android assets), for example: - // filePath: 'ggml-base.en-q8_0.bin', - // isBundleAsset: true - }) + const ctx = await initWhisper(options) const endTime = Date.now() log('Loaded model, ID:', ctx.id) log('Loaded model in', endTime - startTime, `ms in ${mode} mode`) @@ -159,12 +173,9 @@ export default function App() { style={styles.button} disabled={!!stopTranscribe?.stop} onPress={async () => { - if (!whisperContext) { - log('No context') - return - } - await createDir() + if (!whisperContext) return log('No context') + await createDir(log) if (await RNFS.exists(sampleFilePath)) { log('Sample file already exists:') log(filterPath(sampleFilePath)) @@ -218,10 +229,7 @@ export default function App() { stopTranscribe?.stop ? styles.buttonClear : null, ]} onPress={async () => { - if (!whisperContext) { - log('No context') - return - } + if (!whisperContext) return log('No context') if (stopTranscribe?.stop) { stopTranscribe?.stop() setStopTranscribe(null) @@ -308,8 +316,7 @@ export default function App() { style={[styles.button, styles.buttonClear]} title="Clear Download files" onPress={async () => { - await RNFS.unlink(modelFilePath).catch(() => {}) - await RNFS.unlink(sampleFilePath).catch(() => {}) + await RNFS.unlink(fileDir).catch(() => {}) log('Deleted files') }} > diff --git a/ios/RNWhisper.mm b/ios/RNWhisper.mm index 1c188e1..c4be403 100644 --- a/ios/RNWhisper.mm +++ b/ios/RNWhisper.mm @@ -9,6 +9,18 @@ @implementation RNWhisper RCT_EXPORT_MODULE() +- (NSDictionary *)constantsToExport +{ + return @{ +#if WHISPER_USE_COREML + @"WHISPER_USE_COREML": @YES, +#endif +#if WHISPER_COREML_ALLOW_FALLBACK + @"WHISPER_COREML_ALLOW_FALLBACK": @YES, +#endif + }; +} + RCT_REMAP_METHOD(initContext, withPath:(NSString *)modelPath withBundleResource:(BOOL)isBundleAsset diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 0c82f4f..04edd2f 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -7,7 +7,31 @@ cp ./whisper.cpp/ggml.h ./cpp/ggml.h cp ./whisper.cpp/ggml.c ./cpp/ggml.c cp ./whisper.cpp/whisper.h ./cpp/whisper.h cp ./whisper.cpp/whisper.cpp ./cpp/whisper.cpp -# TODO: -# cp -R ./whisper.cpp/coreml/ ./cpp/coreml/ +cp -R ./whisper.cpp/coreml/ ./cpp/coreml/ + +# Apply patch (ref: https://github.com/ggerganov/whisper.cpp/pull/910) +patch -p0 -d ./cpp/coreml < ./scripts/whisper-endocder.mm.patch yarn example + +# Download model for example +cd whisper.cpp/models + +# If CI env is `true`, use dummy model +if [ "$CI" = "true" ]; then + cp for-tests-ggml-base.en.bin ggml-base.en.bin + echo "CI: Copied for-tests-ggml-base.en.bin to ggml-base.en.bin" +else + ./download-ggml-model.sh base.en +fi + +# Copy to Android example +cp ggml-base.en.bin ../../example/android/app/src/main/assets +echo "Copied ggml-base.en.bin to example/android/app/src/main/assets" + +# Check whisper.cpp/models/ggml-base.en-encoder.mlmodelc exist +if [ ! -d ./ggml-base.en-encoder.mlmodelc ]; then + mkdir ggml-base.en-encoder.mlmodelc + echo "Created a dummy ggml-base.en-encoder.mlmodelc for testing." + echo "Please follow https://github.com/ggerganov/whisper.cpp#core-ml-support for convert a real model." +fi diff --git a/scripts/whisper-endocder.mm.patch b/scripts/whisper-endocder.mm.patch new file mode 100644 index 0000000..8f37c11 --- /dev/null +++ b/scripts/whisper-endocder.mm.patch @@ -0,0 +1,25 @@ +--- whisper-encoder.mm 2023-05-13 12:32:22 ++++ whisper-encoder.new.mm 2023-05-13 12:33:42 +@@ -1,6 +1,10 @@ +-#import "coreml/whisper-encoder.h" +-#import "coreml/whisper-encoder-impl.h" ++#if !__has_feature(objc_arc) ++#error This file must be compiled with automatic reference counting enabled (-fobjc-arc) ++#endif + ++#import "whisper-encoder.h" ++#import "whisper-encoder-impl.h" ++ + #import + + #include +@@ -52,9 +56,6 @@ + whisper_encoder_implOutput * outCoreML = [(__bridge id) ctx->data predictionFromLogmel_data:inMultiArray error:nil]; + + memcpy(out, outCoreML.output.dataPointer, outCoreML.output.count * sizeof(float)); +- +- [outCoreML release]; +- [inMultiArray release]; + } + + #if __cplusplus diff --git a/src/index.ts b/src/index.ts index b353c90..9ed6d85 100644 --- a/src/index.ts +++ b/src/index.ts @@ -264,4 +264,10 @@ export async function initWhisper( export async function releaseAllWhisper(): Promise { return RNWhisper.releaseAllContexts() -} \ No newline at end of file +} + +/** Is use CoreML models on iOS */ +export const isUseCoreML: boolean = !!RNWhisper.WHISPER_USE_COREML + +/** Is allow fallback to CPU if load CoreML model failed */ +export const isCoreMLAllowFallback: boolean = !!RNWhisper.WHISPER_COREML_ALLOW_FALLBACK diff --git a/whisper-rn.podspec b/whisper-rn.podspec index 618676a..f2d75a2 100644 --- a/whisper-rn.podspec +++ b/whisper-rn.podspec @@ -1,10 +1,16 @@ require "json" package = JSON.parse(File.read(File.join(__dir__, "package.json"))) +base_ld_flags = "-framework Accelerate" base_compiler_flags = "-DGGML_USE_ACCELERATE -Wno-shorten-64-to-32" folly_compiler_flags = "-DFOLLY_NO_CONFIG -DFOLLY_MOBILE=1 -DFOLLY_USE_LIBCPP=1 -Wno-comma" base_optimizer_flags = "-O3 -DNDEBUG" +if ENV['RNWHISPER_DISABLE_COREML'] != '1' then + base_ld_flags += " -framework CoreML" + base_compiler_flags += " -DWHISPER_USE_COREML -DWHISPER_COREML_ALLOW_FALLBACK" +end + Pod::Spec.new do |s| s.name = "whisper-rn" s.version = package["version"] @@ -16,13 +22,13 @@ Pod::Spec.new do |s| s.platforms = { :ios => "11.0", :tvos => "11.0" } s.source = { :git => "https://github.com/mybigday/whisper.rn.git", :tag => "#{s.version}" } - s.source_files = "ios/**/*.{h,m,mm}", "cpp/**/*.{h,cpp,c}" + s.source_files = "ios/**/*.{h,m,mm}", "cpp/**/*.{h,cpp,c,m,mm}" s.dependency "React-Core" s.compiler_flags = base_compiler_flags s.pod_target_xcconfig = { - "OTHER_LDFLAGS" => "-framework Accelerate", + "OTHER_LDFLAGS" => base_ld_flags, "OTHER_CFLAGS[config=Release]" => base_optimizer_flags, "OTHER_CPLUSPLUSFLAGS[config=Release]" => base_optimizer_flags }