diff --git a/source/adapters/level_zero/context.cpp b/source/adapters/level_zero/context.cpp index 296e3e98d5..a989cea36d 100644 --- a/source/adapters/level_zero/context.cpp +++ b/source/adapters/level_zero/context.cpp @@ -560,9 +560,12 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool( ur_event_handle_t ur_context_handle_t_::getEventFromContextCache( bool HostVisible, bool WithProfiling, ur_device_handle_t Device, - bool CounterBasedEventEnabled) { + bool CounterBasedEventEnabled, bool UsingImmCmdList) { std::scoped_lock Lock(EventCacheMutex); auto Cache = getEventCache(HostVisible, WithProfiling, Device); + if (CounterBasedEventEnabled) { + Cache = getCounterBasedEventCache(WithProfiling, UsingImmCmdList, Device); + } if (Cache->empty()) return nullptr; @@ -585,9 +588,16 @@ void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) { Device = Event->UrQueue->Device; } - auto Cache = getEventCache(Event->isHostVisible(), - Event->isProfilingEnabled(), Device); - Cache->emplace_back(Event); + if (Event->CounterBasedEventsEnabled) { + auto Cache = getCounterBasedEventCache( + Event->isProfilingEnabled(), + !(Event->UrQueue) || (Event->UrQueue)->UsingImmCmdLists, Device); + Cache->emplace_back(Event); + } else { + auto Cache = getEventCache(Event->isHostVisible(), + Event->isProfilingEnabled(), Device); + Cache->emplace_back(Event); + } } ur_result_t diff --git a/source/adapters/level_zero/context.hpp b/source/adapters/level_zero/context.hpp index e7c0d784a0..a5866f891f 100644 --- a/source/adapters/level_zero/context.hpp +++ b/source/adapters/level_zero/context.hpp @@ -39,6 +39,13 @@ struct ur_context_handle_t_ : _ur_object { : ZeContext{ZeContext}, Devices{Devs, Devs + NumDevices}, NumDevices{NumDevices} { OwnNativeHandle = OwnZeContext; + for (const auto &Device : Devices) { + for (int i = 0; i < EventCacheTypeCount; i++) { + EventCaches.emplace_back(); + EventCachesDeviceMap[i].insert( + std::make_pair(Device, EventCaches.size() - 1)); + } + } } ur_context_handle_t_(ze_context_handle_t ZeContext) : ZeContext{ZeContext} {} @@ -150,9 +157,10 @@ struct ur_context_handle_t_ : _ur_object { // head. // // Cache of event pools to which host-visible events are added to. - std::vector> ZeEventPoolCache{12}; + std::vector> ZeEventPoolCache{ + ZeEventPoolCacheTypeCount * 2}; std::vector> - ZeEventPoolCacheDeviceMap{12}; + ZeEventPoolCacheDeviceMap{ZeEventPoolCacheTypeCount * 2}; // This map will be used to determine if a pool is full or not // by storing number of empty slots available in the pool. @@ -174,9 +182,9 @@ struct ur_context_handle_t_ : _ur_object { // Caches for events. using EventCache = std::vector>; - EventCache EventCaches{4}; + EventCache EventCaches{EventCacheTypeCount}; std::vector> - EventCachesDeviceMap{4}; + EventCachesDeviceMap{EventCacheTypeCount}; // Initialize the PI context. ur_result_t initialize(); @@ -214,25 +222,39 @@ struct ur_context_handle_t_ : _ur_object { ur_event_handle_t getEventFromContextCache(bool HostVisible, bool WithProfiling, ur_device_handle_t Device, - bool CounterBasedEventEnabled); + bool CounterBasedEventEnabled, + bool UsingImmCmdList); // Add ur_event_handle_t to cache. void addEventToContextCache(ur_event_handle_t); - enum EventPoolCacheType { + enum ZeEventPoolCacheType { HostVisibleCacheType, HostInvisibleCacheType, HostVisibleCounterBasedRegularCacheType, HostInvisibleCounterBasedRegularCacheType, HostVisibleCounterBasedImmediateCacheType, - HostInvisibleCounterBasedImmediateCacheType + HostInvisibleCounterBasedImmediateCacheType, + ZeEventPoolCacheTypeCount + }; + + enum EventCacheType { + HostVisibleProfilingCacheType, + HostVisibleRegularCacheType, + HostInvisibleProfilingCacheType, + HostInvisibleRegularCacheType, + CounterBasedImmediateCacheType, + CounterBasedRegularCacheType, + CounterBasedImmediateProfilingCacheType, + CounterBasedRegularProfilingCacheType, + EventCacheTypeCount }; std::list * getZeEventPoolCache(bool HostVisible, bool WithProfiling, bool CounterBasedEventEnabled, bool UsingImmediateCmdList, ze_device_handle_t ZeDevice) { - EventPoolCacheType CacheType; + ZeEventPoolCacheType CacheType; calculateCacheIndex(HostVisible, CounterBasedEventEnabled, UsingImmediateCmdList, CacheType); @@ -255,7 +277,7 @@ struct ur_context_handle_t_ : _ur_object { ur_result_t calculateCacheIndex(bool HostVisible, bool CounterBasedEventEnabled, bool UsingImmediateCmdList, - EventPoolCacheType &CacheType) { + ZeEventPoolCacheType &CacheType) { if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) { CacheType = HostVisibleCounterBasedRegularCacheType; } else if (CounterBasedEventEnabled && !HostVisible && @@ -319,28 +341,51 @@ struct ur_context_handle_t_ : _ur_object { if (HostVisible) { if (Device) { auto EventCachesMap = - WithProfiling ? &EventCachesDeviceMap[0] : &EventCachesDeviceMap[1]; - if (EventCachesMap->find(Device) == EventCachesMap->end()) { - EventCaches.emplace_back(); - EventCachesMap->insert( - std::make_pair(Device, EventCaches.size() - 1)); - } + WithProfiling ? &EventCachesDeviceMap[HostVisibleProfilingCacheType] + : &EventCachesDeviceMap[HostVisibleRegularCacheType]; + return &EventCaches[(*EventCachesMap)[Device]]; + } else { + return WithProfiling ? &EventCaches[HostVisibleProfilingCacheType] + : &EventCaches[HostVisibleRegularCacheType]; + } + } else { + if (Device) { + auto EventCachesMap = + WithProfiling + ? &EventCachesDeviceMap[HostInvisibleProfilingCacheType] + : &EventCachesDeviceMap[HostInvisibleRegularCacheType]; + return &EventCaches[(*EventCachesMap)[Device]]; + } else { + return WithProfiling ? &EventCaches[HostInvisibleProfilingCacheType] + : &EventCaches[HostInvisibleRegularCacheType]; + } + } + }; + auto getCounterBasedEventCache(bool WithProfiling, bool UsingImmediateCmdList, + ur_device_handle_t Device) { + if (UsingImmediateCmdList) { + if (Device) { + auto EventCachesMap = + WithProfiling + ? &EventCachesDeviceMap[CounterBasedImmediateProfilingCacheType] + : &EventCachesDeviceMap[CounterBasedImmediateCacheType]; return &EventCaches[(*EventCachesMap)[Device]]; } else { - return WithProfiling ? &EventCaches[0] : &EventCaches[1]; + return WithProfiling + ? &EventCaches[CounterBasedImmediateProfilingCacheType] + : &EventCaches[CounterBasedImmediateCacheType]; } } else { if (Device) { auto EventCachesMap = - WithProfiling ? &EventCachesDeviceMap[2] : &EventCachesDeviceMap[3]; - if (EventCachesMap->find(Device) == EventCachesMap->end()) { - EventCaches.emplace_back(); - EventCachesMap->insert( - std::make_pair(Device, EventCaches.size() - 1)); - } + WithProfiling + ? &EventCachesDeviceMap[CounterBasedRegularProfilingCacheType] + : &EventCachesDeviceMap[CounterBasedRegularCacheType]; return &EventCaches[(*EventCachesMap)[Device]]; } else { - return WithProfiling ? &EventCaches[2] : &EventCaches[3]; + return WithProfiling + ? &EventCaches[CounterBasedRegularProfilingCacheType] + : &EventCaches[CounterBasedRegularCacheType]; } } } diff --git a/source/adapters/level_zero/event.cpp b/source/adapters/level_zero/event.cpp index 2bd3011b4b..43e728ed33 100644 --- a/source/adapters/level_zero/event.cpp +++ b/source/adapters/level_zero/event.cpp @@ -1272,7 +1272,8 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue, } if (auto CachedEvent = Context->getEventFromContextCache( - HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled)) { + HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled, + UsingImmediateCommandlists)) { *RetEvent = CachedEvent; return UR_RESULT_SUCCESS; } diff --git a/source/adapters/level_zero/queue.cpp b/source/adapters/level_zero/queue.cpp index 978547df10..b498992bdd 100644 --- a/source/adapters/level_zero/queue.cpp +++ b/source/adapters/level_zero/queue.cpp @@ -1187,7 +1187,7 @@ ur_queue_handle_t_::ur_queue_handle_t_( return std::atoi(UrRet) != 0; }(); this->CounterBasedEventsEnabled = - UsingImmCmdLists && isInOrderQueue() && Device->useDriverInOrderLists() && + isInOrderQueue() && Device->useDriverInOrderLists() && useDriverCounterBasedEvents && Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound; }