diff --git a/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.h b/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.h index 7b6b2fb9ff..04fef204e1 100644 --- a/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.h +++ b/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.h @@ -104,7 +104,7 @@ NS_ASSUME_NONNULL_BEGIN /// /// @param error On failure, error is filled with the failure information. /// @retval `YES` is the assets are purged otherwise `NO`. -- (BOOL)purge:(NSError* __autoreleasing*)error; +- (BOOL)purgeAndReturnError:(NSError* __autoreleasing*)error; /// The estimated size of the assets store. The returned value might not correct if the asset /// directory is tampered externally. diff --git a/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.mm b/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.mm index 6fe37925d2..73e9cc0f33 100644 --- a/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.mm +++ b/backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.mm @@ -740,7 +740,7 @@ - (BOOL)_purge:(NSError * __autoreleasing *)error { return static_cast(status); } -- (BOOL)purge:(NSError * __autoreleasing *)error { +- (BOOL)purgeAndReturnError:(NSError * __autoreleasing *)error { __block BOOL result = 0; dispatch_sync(self.syncQueue, ^{ result = [self _purge:error]; diff --git a/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.h b/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.h index 394cff4f89..abc5ef517b 100644 --- a/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.h +++ b/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.h @@ -103,8 +103,11 @@ __attribute__((objc_subclassing_restricted)) /// @retval `YES` if the model was pre-warmed otherwise `NO`. - (BOOL)prewarmModelWithHandle:(ModelHandle*)handle error:(NSError* __autoreleasing*)error; -/// The `ETCoreMLAssetManager` instance used to manage models cache. -@property (strong, readonly, nonatomic) ETCoreMLAssetManager* assetManager; +/// Purges model cache. +/// +/// @param error On failure, error is filled with the failure information. +/// @retval `YES` if the cache is purged otherwise `NO`. +- (BOOL)purgeModelsCacheAndReturnError:(NSError* __autoreleasing*)error; @end diff --git a/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm b/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm index c51de9d1e1..e7846256e6 100644 --- a/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm +++ b/backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm @@ -352,6 +352,7 @@ @interface ETCoreMLModelManager () { } @property (nonatomic, readonly, strong) NSFileManager *fileManager; +@property (strong, readonly, nonatomic) ETCoreMLAssetManager* assetManager; @property (nonatomic, readonly, strong) NSMutableDictionary> *handleToExecutorMap; @property (nonatomic, readonly, strong) NSMapTable *modelIdentifierToLoadingQueueMap; @property (nonatomic, readonly, strong) NSMutableDictionary *modelIdentifierToPrewarmedAssetMap; @@ -832,4 +833,8 @@ - (BOOL)unloadModelWithHandle:(ModelHandle *)handle { return result; } +- (BOOL)purgeModelsCacheAndReturnError:(NSError *__autoreleasing *)error { + return [self.assetManager purgeAndReturnError:error]; +} + @end diff --git a/backends/apple/coreml/runtime/delegate/backend_delegate.mm b/backends/apple/coreml/runtime/delegate/backend_delegate.mm index b31cc3f754..f6eb7a83fd 100644 --- a/backends/apple/coreml/runtime/delegate/backend_delegate.mm +++ b/backends/apple/coreml/runtime/delegate/backend_delegate.mm @@ -10,7 +10,6 @@ #import #import #import -#import #import #import #import @@ -88,6 +87,153 @@ MLComputeUnits get_compute_units(const Buffer& buffer) { } } //namespace +@interface ETCoreMLModelManagerDelegate : NSObject + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +- (instancetype)initWithConfig:(BackendDelegate::Config)config NS_DESIGNATED_INITIALIZER; + +- (BOOL)loadAndReturnError:(NSError * _Nullable __autoreleasing *)error; + +- (void)loadAsynchronously; + +- (ModelHandle*)loadModelFromAOTData:(NSData*)data + configuration:(MLModelConfiguration*)configuration + error:(NSError* __autoreleasing*)error; + +- (BOOL)executeModelWithHandle:(ModelHandle*)handle + argsVec:(const std::vector&)argsVec + loggingOptions:(const executorchcoreml::ModelLoggingOptions&)loggingOptions + eventLogger:(const executorchcoreml::ModelEventLogger* _Nullable)eventLogger + error:(NSError* __autoreleasing*)error; + +- (BOOL)unloadModelWithHandle:(ModelHandle*)handle; + +- (BOOL)purgeModelsCacheAndReturnError:(NSError * _Nullable __autoreleasing *)error; + +@property (assign, readonly, nonatomic) BackendDelegate::Config config; +@property (strong, readonly, nonatomic) dispatch_queue_t syncQueue; +@property (strong, nonatomic, nullable) ETCoreMLModelManager *impl; +@property (assign, readonly, nonatomic) BOOL isAvailable; + +@end + +@implementation ETCoreMLModelManagerDelegate + +- (instancetype)initWithConfig:(BackendDelegate::Config)config { + self = [super init]; + if (self) { + _config = std::move(config); + _syncQueue = dispatch_queue_create("com.executorchcoreml.modelmanagerdelegate.sync", DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL); + } + + return self; +} + +- (BOOL)_loadAndReturnError:(NSError * _Nullable __autoreleasing *)error { + if (self.impl != nil) { + return YES; + } + + ETCoreMLAssetManager *assetManager = create_asset_manager(ETCoreMLStrings.assetsDirectoryPath, + ETCoreMLStrings.trashDirectoryPath, + ETCoreMLStrings.databaseDirectoryPath, + ETCoreMLStrings.databaseName, + self.config.max_models_cache_size, + error); + if (!assetManager) { + return NO; + } + + ETCoreMLModelManager *modelManager = [[ETCoreMLModelManager alloc] initWithAssetManager:assetManager]; + if (!modelManager) { + return NO; + } + + self.impl = modelManager; + + if (self.config.should_prewarm_asset) { + [modelManager prewarmRecentlyUsedAssetsWithMaxCount:1]; + } + + return YES; +} + +- (BOOL)loadAndReturnError:(NSError * _Nullable __autoreleasing *)error { + __block NSError *localError = nil; + __block BOOL result = NO; + dispatch_sync(self.syncQueue, ^{ + result = [self _loadAndReturnError:&localError]; + }); + + if (error) { + *error = localError; + } + + return result; +} + +- (void)loadAsynchronously { + dispatch_async(self.syncQueue, ^{ + (void)[self _loadAndReturnError:nil]; + }); +} + +- (ModelHandle*)loadModelFromAOTData:(NSData*)data + configuration:(MLModelConfiguration*)configuration + error:(NSError* __autoreleasing*)error { + if (![self loadAndReturnError:error]) { + return nil; + } + + return [self.impl loadModelFromAOTData:data + configuration:configuration + error:error]; +} + +- (BOOL)executeModelWithHandle:(ModelHandle*)handle + argsVec:(const std::vector&)argsVec + loggingOptions:(const executorchcoreml::ModelLoggingOptions&)loggingOptions + eventLogger:(const executorchcoreml::ModelEventLogger* _Nullable)eventLogger + error:(NSError* __autoreleasing*)error { + assert(self.impl != nil && "Impl must not be nil"); + return [self.impl executeModelWithHandle:handle + argsVec:argsVec + loggingOptions:loggingOptions + eventLogger:eventLogger + error:error]; +} + +- (nullable ETCoreMLModel*)modelWithHandle:(ModelHandle*)handle { + assert(self.impl != nil && "Impl must not be nil"); + return [self.impl modelWithHandle:handle]; +} + +- (BOOL)unloadModelWithHandle:(ModelHandle*)handle { + assert(self.impl != nil && "Impl must not be nil"); + return [self.impl unloadModelWithHandle:handle]; +} + +- (BOOL)purgeModelsCacheAndReturnError:(NSError * _Nullable __autoreleasing *)error { + if (![self loadAndReturnError:error]) { + return NO; + } + + return [self.impl purgeModelsCacheAndReturnError:error];; +} + +- (BOOL)isAvailable { + if (![self loadAndReturnError:nil]) { + return NO; + } + + return YES; +} + +@end + namespace executorchcoreml { std::string BackendDelegate::ErrorCategory::message(int code) const { @@ -114,20 +260,9 @@ MLComputeUnits get_compute_units(const Buffer& buffer) { class BackendDelegateImpl: public BackendDelegate { public: explicit BackendDelegateImpl(const Config& config) noexcept - :BackendDelegate(), config_(config) { - NSError *localError = nil; - ETCoreMLAssetManager *asset_manager = create_asset_manager(ETCoreMLStrings.assetsDirectoryPath, - ETCoreMLStrings.trashDirectoryPath, - ETCoreMLStrings.databaseDirectoryPath, - ETCoreMLStrings.databaseName, - config.max_models_cache_size, - &localError); - - model_manager_ = (asset_manager != nil) ? [[ETCoreMLModelManager alloc] initWithAssetManager:asset_manager] : nil; - if (model_manager_ != nil && config_.should_prewarm_asset) { - [model_manager_ prewarmRecentlyUsedAssetsWithMaxCount:1]; - } - available_.store(model_manager_ != nil, std::memory_order_seq_cst); + :BackendDelegate(), model_manager_([[ETCoreMLModelManagerDelegate alloc] initWithConfig:config]) + { + [model_manager_ loadAsynchronously]; } BackendDelegateImpl(BackendDelegateImpl const&) = delete; @@ -142,11 +277,6 @@ explicit BackendDelegateImpl(const Config& config) noexcept ModelHandle *modelHandle = [model_manager_ loadModelFromAOTData:data configuration:configuration error:&localError]; - if (modelHandle && config_.should_prewarm_model) { - NSError *localError = nil; - [model_manager_ prewarmModelWithHandle:modelHandle error:&localError]; - } - return modelHandle; } @@ -158,7 +288,7 @@ bool execute(Handle* handle, NSError *error = nil; if (![model_manager_ executeModelWithHandle:handle argsVec:args - loggingOptions:logging_options + loggingOptions:logging_options eventLogger:event_logger error:&error]) { ec = static_cast(error.code); @@ -173,7 +303,7 @@ bool is_valid_handle(Handle* handle) const noexcept override { } bool is_available() const noexcept override { - return available_.load(std::memory_order_acquire); + return static_cast(model_manager_.isAvailable); } std::pair get_num_arguments(Handle* handle) const noexcept override { @@ -187,12 +317,11 @@ void destroy(Handle* handle) const noexcept override { bool purge_models_cache() const noexcept override { NSError *localError = nil; - bool result = static_cast([model_manager_.assetManager purge:&localError]); + bool result = static_cast([model_manager_ purgeModelsCacheAndReturnError:&localError]); return result; } - ETCoreMLModelManager *model_manager_; - std::atomic available_; + ETCoreMLModelManagerDelegate *model_manager_; Config config_; }; diff --git a/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm b/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm index 71143e096a..ab596575a2 100644 --- a/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm +++ b/backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm @@ -11,6 +11,7 @@ #import #import #import +#import #import #import #import diff --git a/backends/apple/coreml/runtime/test/BackendDelegateTests.mm b/backends/apple/coreml/runtime/test/BackendDelegateTests.mm index 6f0e3cff31..8222349650 100644 --- a/backends/apple/coreml/runtime/test/BackendDelegateTests.mm +++ b/backends/apple/coreml/runtime/test/BackendDelegateTests.mm @@ -15,6 +15,7 @@ #import #import #import +#import using namespace executorchcoreml; @@ -58,6 +59,10 @@ + (nullable NSURL *)bundledResourceWithName:(NSString *)name extension:(NSString return [bundle URLForResource:name withExtension:extension]; } ++ (void)setUp { + torch::executor::runtime_init(); +} + - (void)setUp { @autoreleasepool { _delegate = BackendDelegate::make(BackendDelegate::Config()); @@ -151,9 +156,6 @@ - (void)testAddModelExecution { int y = 50; // add_coreml_all does the following operations. int z = x + y; - z = z + x; - z = z + x; - z = z + z; NSArray *inputs = [ETCoreMLTestUtils inputsForModel:model repeatedValues:@[@(x), @(y)] error:&localError]; XCTAssertNotNil(inputs); diff --git a/backends/apple/coreml/runtime/test/CoreMLBackendDelegateTests.mm b/backends/apple/coreml/runtime/test/CoreMLBackendDelegateTests.mm index 94b862d842..feb8000703 100644 --- a/backends/apple/coreml/runtime/test/CoreMLBackendDelegateTests.mm +++ b/backends/apple/coreml/runtime/test/CoreMLBackendDelegateTests.mm @@ -6,14 +6,12 @@ // Please refer to the license found in the LICENSE file in the root directory of the source tree. #import - -#import - #import #import #import #import #import +#import static constexpr size_t kRuntimeMemorySize = 50 * 1024U * 1024U; // 50 MB @@ -128,7 +126,7 @@ @interface CoreMLBackendDelegateTests : XCTestCase @implementation CoreMLBackendDelegateTests + (void)setUp { - runtime_init(); + torch::executor::runtime_init(); } + (nullable NSURL *)bundledResourceWithName:(NSString *)name extension:(NSString *)extension { diff --git a/backends/apple/coreml/runtime/test/ETCoreMLAssetManagerTests.mm b/backends/apple/coreml/runtime/test/ETCoreMLAssetManagerTests.mm index aa16f47dfd..f466899559 100644 --- a/backends/apple/coreml/runtime/test/ETCoreMLAssetManagerTests.mm +++ b/backends/apple/coreml/runtime/test/ETCoreMLAssetManagerTests.mm @@ -6,12 +6,11 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import - +#import "ETCoreMLTestUtils.h" #import #import - -#import "ETCoreMLTestUtils.h" +#import +#import @interface ETCoreMLAssetManagerTests : XCTestCase @@ -23,6 +22,10 @@ @interface ETCoreMLAssetManagerTests : XCTestCase @implementation ETCoreMLAssetManagerTests ++ (void)setUp { + torch::executor::runtime_init(); +} + - (void)setUp { @autoreleasepool { NSError *localError = nil; @@ -145,7 +148,7 @@ - (void)testPurge { // Close the asset so that it could be deleted. [asset close]; } - XCTAssertTrue([self.assetManager purge:&localError]); + XCTAssertTrue([self.assetManager purgeAndReturnError:&localError]); } @end diff --git a/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm b/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm index 75411f2bd4..95c84ab674 100644 --- a/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm +++ b/backends/apple/coreml/runtime/test/ETCoreMLModelDebuggerTests.mm @@ -6,14 +6,14 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import +#import "ETCoreMLTestUtils.h" #import -#import #import -#import +#import +#import +#import #import - -#import "ETCoreMLTestUtils.h" +#import namespace { using namespace executorchcoreml::modelstructure; @@ -57,6 +57,10 @@ @interface ETCoreMLModelDebuggerTests : XCTestCase @implementation ETCoreMLModelDebuggerTests ++ (void)setUp { + torch::executor::runtime_init(); +} + + (nullable NSURL *)bundledResourceWithName:(NSString *)name extension:(NSString *)extension { NSBundle *bundle = [NSBundle bundleForClass:ETCoreMLModelDebuggerTests.class]; return [bundle URLForResource:name withExtension:extension]; @@ -100,9 +104,7 @@ - (void)testAddProgramDebugging { NotifyFn notify = [](NSDictionary *debuggingResult, NSDictionary *pathToSymbolNameMap) { // There are 3 add ops, we verify that we get the outputs for the ops. - XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten_add_tensor_2_cast_fp16")]); XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten_add_tensor_cast_fp16")]); - XCTAssertNotNil(debuggingResult[make_path_with_output_name("aten_add_tensor_1_cast_fp16")]); }; [self debugModelWithName:@"add_coreml_all" diff --git a/backends/apple/coreml/runtime/test/ETCoreMLModelManagerTests.mm b/backends/apple/coreml/runtime/test/ETCoreMLModelManagerTests.mm index 8ad712497e..5fceb9ac75 100644 --- a/backends/apple/coreml/runtime/test/ETCoreMLModelManagerTests.mm +++ b/backends/apple/coreml/runtime/test/ETCoreMLModelManagerTests.mm @@ -6,15 +6,14 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import - - #import "ETCoreMLTestUtils.h" #import #import #import #import #import +#import +#import #import @interface ETCoreMLModelManagerTests : XCTestCase @@ -33,6 +32,7 @@ + (nullable NSURL *)bundledResourceWithName:(NSString *)name extension:(NSString } - (void)setUp { + torch::executor::runtime_init(); @autoreleasepool { NSError *localError = nil; self.fileManager = [[NSFileManager alloc] init]; @@ -103,11 +103,8 @@ - (void)testAddModelExecution { ETCoreMLModel *model = [self.modelManager modelWithHandle:handle]; int x = 20; int y = 50; - // add_coreml_all does the following operations. + // add_coreml_all does the following operation. int z = x + y; - z = z + x; - z = z + x; - z = z + z; NSArray *inputs = [ETCoreMLTestUtils inputsForModel:model repeatedValues:@[@(x), @(y)] error:&localError]; XCTAssertNotNil(inputs); diff --git a/backends/apple/coreml/runtime/test/ETCoreMLModelProfilerTests.mm b/backends/apple/coreml/runtime/test/ETCoreMLModelProfilerTests.mm index 0d6c54bdc2..c243ab5159 100644 --- a/backends/apple/coreml/runtime/test/ETCoreMLModelProfilerTests.mm +++ b/backends/apple/coreml/runtime/test/ETCoreMLModelProfilerTests.mm @@ -6,15 +6,15 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import +#import "ETCoreMLTestUtils.h" #import #import -#import #import -#import +#import +#import +#import #import - -#import "ETCoreMLTestUtils.h" +#import namespace { using namespace executorchcoreml::modelstructure; @@ -58,6 +58,10 @@ @interface ETCoreMLModelProfilerTests : XCTestCase @implementation ETCoreMLModelProfilerTests ++ (void)setUp { + torch::executor::runtime_init(); +} + + (nullable NSURL *)bundledResourceWithName:(NSString *)name extension:(NSString *)extension { NSBundle *bundle = [NSBundle bundleForClass:ETCoreMLModelProfilerTests.class]; return [bundle URLForResource:name withExtension:extension]; @@ -102,10 +106,8 @@ - (void)testAddProgramProfiling { if (@available(macOS 14.4, iOS 17.4, watchOS 10.4, tvOS 17.4, *)) { NotifyFn notify = [](NSDictionary *profilingResult, NSDictionary *__unused pathToSymbolNameMap) { - // There are 3 add ops, we verify that the profiling info exists for the ops. - XCTAssertNotNil(profilingResult[make_path_with_output_name("aten_add_tensor_2_cast_fp16")]); + // There is 1 add op, we verify that the profiling info exists for the ops. XCTAssertNotNil(profilingResult[make_path_with_output_name("aten_add_tensor_cast_fp16")]); - XCTAssertNotNil(profilingResult[make_path_with_output_name("aten_add_tensor_1_cast_fp16")]); }; [self profileModelWithName:@"add_coreml_all" diff --git a/backends/apple/coreml/runtime/test/ETCoreMLModelStructurePathTests.mm b/backends/apple/coreml/runtime/test/ETCoreMLModelStructurePathTests.mm index 7b8b90966f..1afaad2b7d 100644 --- a/backends/apple/coreml/runtime/test/ETCoreMLModelStructurePathTests.mm +++ b/backends/apple/coreml/runtime/test/ETCoreMLModelStructurePathTests.mm @@ -6,8 +6,8 @@ // // Please refer to the license found in the LICENSE file in the root directory of the source tree. -#import #import +#import #import namespace {