Merge pull request #1698 from winstonzhang-intel/counter-based-2

[L0] Phase 2 of Counter-Based Event Implementation

source/adapters/level_zero/context.cpp

@@ -560,9 +560,12 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
 
 ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
     bool HostVisible, bool WithProfiling, ur_device_handle_t Device,
-    bool CounterBasedEventEnabled) {
+    bool CounterBasedEventEnabled, bool UsingImmCmdList) {
   std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
   auto Cache = getEventCache(HostVisible, WithProfiling, Device);
+  if (CounterBasedEventEnabled) {
+    Cache = getCounterBasedEventCache(WithProfiling, UsingImmCmdList, Device);
+  }
   if (Cache->empty())
     return nullptr;
 
@@ -585,9 +588,16 @@ void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) {
     Device = Event->UrQueue->Device;
   }
 
-  auto Cache = getEventCache(Event->isHostVisible(),
-                             Event->isProfilingEnabled(), Device);
-  Cache->emplace_back(Event);
+  if (Event->CounterBasedEventsEnabled) {
+    auto Cache = getCounterBasedEventCache(
+        Event->isProfilingEnabled(),
+        !(Event->UrQueue) || (Event->UrQueue)->UsingImmCmdLists, Device);
+    Cache->emplace_back(Event);
+  } else {
+    auto Cache = getEventCache(Event->isHostVisible(),
+                               Event->isProfilingEnabled(), Device);
+    Cache->emplace_back(Event);
+  }
 }
 
 ur_result_t

source/adapters/level_zero/context.hpp

@@ -39,6 +39,13 @@ struct ur_context_handle_t_ : _ur_object {
       : ZeContext{ZeContext}, Devices{Devs, Devs + NumDevices},
         NumDevices{NumDevices} {
     OwnNativeHandle = OwnZeContext;
+    for (const auto &Device : Devices) {
+      for (int i = 0; i < EventCacheTypeCount; i++) {
+        EventCaches.emplace_back();
+        EventCachesDeviceMap[i].insert(
+            std::make_pair(Device, EventCaches.size() - 1));
+      }
+    }
   }
 
   ur_context_handle_t_(ze_context_handle_t ZeContext) : ZeContext{ZeContext} {}
@@ -147,9 +154,10 @@ struct ur_context_handle_t_ : _ur_object {
   // head.
   //
   // Cache of event pools to which host-visible events are added to.
-  std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{12};
+  std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{
+      ZeEventPoolCacheTypeCount * 2};
   std::vector<std::unordered_map<ze_device_handle_t, size_t>>
-      ZeEventPoolCacheDeviceMap{12};
+      ZeEventPoolCacheDeviceMap{ZeEventPoolCacheTypeCount * 2};
 
   // This map will be used to determine if a pool is full or not
   // by storing number of empty slots available in the pool.
@@ -171,9 +179,9 @@ struct ur_context_handle_t_ : _ur_object {
 
   // Caches for events.
   using EventCache = std::vector<std::list<ur_event_handle_t>>;
-  EventCache EventCaches{4};
+  EventCache EventCaches{EventCacheTypeCount};
   std::vector<std::unordered_map<ur_device_handle_t, size_t>>
-      EventCachesDeviceMap{4};
+      EventCachesDeviceMap{EventCacheTypeCount};
 
   // Initialize the PI context.
   ur_result_t initialize();
@@ -211,25 +219,39 @@ struct ur_context_handle_t_ : _ur_object {
   ur_event_handle_t getEventFromContextCache(bool HostVisible,
                                              bool WithProfiling,
                                              ur_device_handle_t Device,
-                                             bool CounterBasedEventEnabled);
+                                             bool CounterBasedEventEnabled,
+                                             bool UsingImmCmdList);
 
   // Add ur_event_handle_t to cache.
   void addEventToContextCache(ur_event_handle_t);
 
-  enum EventPoolCacheType {
+  enum ZeEventPoolCacheType {
     HostVisibleCacheType,
     HostInvisibleCacheType,
     HostVisibleCounterBasedRegularCacheType,
     HostInvisibleCounterBasedRegularCacheType,
     HostVisibleCounterBasedImmediateCacheType,
-    HostInvisibleCounterBasedImmediateCacheType
+    HostInvisibleCounterBasedImmediateCacheType,
+    ZeEventPoolCacheTypeCount
+  };
+
+  enum EventCacheType {
+    HostVisibleProfilingCacheType,
+    HostVisibleRegularCacheType,
+    HostInvisibleProfilingCacheType,
+    HostInvisibleRegularCacheType,
+    CounterBasedImmediateCacheType,
+    CounterBasedRegularCacheType,
+    CounterBasedImmediateProfilingCacheType,
+    CounterBasedRegularProfilingCacheType,
+    EventCacheTypeCount
   };
 
   std::list<ze_event_pool_handle_t> *
   getZeEventPoolCache(bool HostVisible, bool WithProfiling,
                       bool CounterBasedEventEnabled, bool UsingImmediateCmdList,
                       ze_device_handle_t ZeDevice) {
-    EventPoolCacheType CacheType;
+    ZeEventPoolCacheType CacheType;
 
     calculateCacheIndex(HostVisible, CounterBasedEventEnabled,
                         UsingImmediateCmdList, CacheType);
@@ -252,7 +274,7 @@ struct ur_context_handle_t_ : _ur_object {
   ur_result_t calculateCacheIndex(bool HostVisible,
                                   bool CounterBasedEventEnabled,
                                   bool UsingImmediateCmdList,
-                                  EventPoolCacheType &CacheType) {
+                                  ZeEventPoolCacheType &CacheType) {
     if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
       CacheType = HostVisibleCounterBasedRegularCacheType;
     } else if (CounterBasedEventEnabled && !HostVisible &&
@@ -316,34 +338,57 @@ struct ur_context_handle_t_ : _ur_object {
     if (HostVisible) {
       if (Device) {
         auto EventCachesMap =
-            WithProfiling ? &EventCachesDeviceMap[0] : &EventCachesDeviceMap[1];
-        if (EventCachesMap->find(Device) == EventCachesMap->end()) {
-          EventCaches.emplace_back();
-          EventCachesMap->insert(
-              std::make_pair(Device, EventCaches.size() - 1));
-        }
+            WithProfiling ? &EventCachesDeviceMap[HostVisibleProfilingCacheType]
+                          : &EventCachesDeviceMap[HostVisibleRegularCacheType];
+        return &EventCaches[(*EventCachesMap)[Device]];
+      } else {
+        return WithProfiling ? &EventCaches[HostVisibleProfilingCacheType]
+                             : &EventCaches[HostVisibleRegularCacheType];
+      }
+    } else {
+      if (Device) {
+        auto EventCachesMap =
+            WithProfiling
+                ? &EventCachesDeviceMap[HostInvisibleProfilingCacheType]
+                : &EventCachesDeviceMap[HostInvisibleRegularCacheType];
+        return &EventCaches[(*EventCachesMap)[Device]];
+      } else {
+        return WithProfiling ? &EventCaches[HostInvisibleProfilingCacheType]
+                             : &EventCaches[HostInvisibleRegularCacheType];
+      }
+    }
+  };
+  auto getCounterBasedEventCache(bool WithProfiling, bool UsingImmediateCmdList,
+                                 ur_device_handle_t Device) {
+    if (UsingImmediateCmdList) {
+      if (Device) {
+        auto EventCachesMap =
+            WithProfiling
+                ? &EventCachesDeviceMap[CounterBasedImmediateProfilingCacheType]
+                : &EventCachesDeviceMap[CounterBasedImmediateCacheType];
         return &EventCaches[(*EventCachesMap)[Device]];
       } else {
-        return WithProfiling ? &EventCaches[0] : &EventCaches[1];
+        return WithProfiling
+                   ? &EventCaches[CounterBasedImmediateProfilingCacheType]
+                   : &EventCaches[CounterBasedImmediateCacheType];
       }
     } else {
       if (Device) {
         auto EventCachesMap =
-            WithProfiling ? &EventCachesDeviceMap[2] : &EventCachesDeviceMap[3];
-        if (EventCachesMap->find(Device) == EventCachesMap->end()) {
-          EventCaches.emplace_back();
-          EventCachesMap->insert(
-              std::make_pair(Device, EventCaches.size() - 1));
-        }
+            WithProfiling
+                ? &EventCachesDeviceMap[CounterBasedRegularProfilingCacheType]
+                : &EventCachesDeviceMap[CounterBasedRegularCacheType];
         return &EventCaches[(*EventCachesMap)[Device]];
       } else {
-        return WithProfiling ? &EventCaches[2] : &EventCaches[3];
+        return WithProfiling
+                   ? &EventCaches[CounterBasedRegularProfilingCacheType]
+                   : &EventCaches[CounterBasedRegularCacheType];
       }
     }
   }
 };
 
-// Helper function to release the context, a caller must lock the platform-level
-// mutex guarding the container with contexts because the context can be removed
-// from the list of tracked contexts.
+// Helper function to release the context, a caller must lock the
+// platform-level mutex guarding the container with contexts because the
+// context can be removed from the list of tracked contexts.
 ur_result_t ContextReleaseHelper(ur_context_handle_t Context);

source/adapters/level_zero/event.cpp

@@ -835,7 +835,6 @@ urEventRelease(ur_event_handle_t Event ///< [in] handle of the event object
 ) {
   Event->RefCountExternal--;
   UR_CALL(urEventReleaseInternal(Event));
-
   return UR_RESULT_SUCCESS;
 }
 
@@ -1257,7 +1256,8 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   }
 
   if (auto CachedEvent = Context->getEventFromContextCache(
-          HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled)) {
+          HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled,
+          UsingImmediateCommandlists)) {
     *RetEvent = CachedEvent;
     return UR_RESULT_SUCCESS;
   }

source/adapters/level_zero/queue.cpp

@@ -1187,7 +1187,7 @@ ur_queue_handle_t_::ur_queue_handle_t_(
     return std::atoi(UrRet) != 0;
   }();
   this->CounterBasedEventsEnabled =
-      UsingImmCmdLists && isInOrderQueue() && Device->useDriverInOrderLists() &&
+      isInOrderQueue() && Device->useDriverInOrderLists() &&
       useDriverCounterBasedEvents &&
       Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound;
 }