[Wisp] Changeset for wispTenant.

Summary:
- ThreadLocal Preempt
Support coroutine preempt base on thread-local
handshake rather than global safepoint.
- Fix lost unpark in lazySet
dragonwell-project/dragonwell8_jdk#63
- Add container info in jstack output
https://github.com/alibaba/dragonwell8_hotspot/pull/96/files
- Add support for thread ids in root container
dragonwell-project/dragonwell8_jdk#62

Test Plan: wisp rcm tests

Reviewed-by: yulei

Issue:
dragonwell-project#144

src/hotspot/share/classfile/javaClasses.cpp

@@ -1760,6 +1760,7 @@ int java_lang_Thread::_stackSize_offset;
 int java_lang_Thread::_tid_offset;
 int java_lang_Thread::_thread_status_offset;
 int java_lang_Thread::_park_blocker_offset;
+int java_lang_Thread::_resourceContainer_offset;
 
 #define THREAD_FIELDS_DO(macro) \
   macro(_name_offset,          k, vmSymbols::name_name(), string_signature, false); \
@@ -1774,7 +1775,8 @@ int java_lang_Thread::_park_blocker_offset;
   macro(_stackSize_offset,     k, "stackSize", long_signature, false); \
   macro(_tid_offset,           k, "tid", long_signature, false); \
   macro(_thread_status_offset, k, "threadStatus", int_signature, false); \
-  macro(_park_blocker_offset,  k, "parkBlocker", object_signature, false)
+  macro(_park_blocker_offset,  k, "parkBlocker", object_signature, false); \
+  macro(_resourceContainer_offset,    k, "resourceContainer", vmSymbols::resourcecontainer_signature, false); \
 
 void java_lang_Thread::compute_offsets() {
   assert(_group_offset == 0, "offsets should be initialized only once");
@@ -1822,6 +1824,10 @@ oop java_lang_Thread::name(oop java_thread) {
 }
 
 
+oop java_lang_Thread::resourceContainer(oop java_thread) {
+  return java_thread->obj_field(_resourceContainer_offset);
+}
+
 void java_lang_Thread::set_name(oop java_thread, oop name) {
   java_thread->obj_field_put(_name_offset, name);
 }
@@ -4765,6 +4771,29 @@ bool com_alibaba_wisp_engine_WispCarrier::in_critical(oop obj) {
   return obj->bool_field(_isInCritical_offset);
 }
 
+
+#define RCM_FIELDS_DO(macro) \
+  macro(_id_offset,      k,   "id",     long_signature,  false);
+
+int com_alibaba_rcm_internal_AbstractResourceContainer::_id_offset = 0;
+
+long com_alibaba_rcm_internal_AbstractResourceContainer::get_id(oop obj) {
+  return obj->long_field(_id_offset);
+}
+
+
+void com_alibaba_rcm_internal_AbstractResourceContainer::compute_offsets() {
+  InstanceKlass *k = vmClasses::com_alibaba_rcm_internal_AbstractResourceContainer_klass();
+  assert(k != NULL, "AbstractResourceContainer is null");
+  RCM_FIELDS_DO(FIELD_COMPUTE_OFFSET);
+}
+
+#if INCLUDE_CDS
+void com_alibaba_rcm_internal_AbstractResourceContainer::serialize_offsets(SerializeClosure* f) {
+  RCM_FIELDS_DO(FIELD_SERIALIZE_OFFSET);
+}
+#endif
+
 int com_alibaba_wisp_engine_WispTask::_jvmParkStatus_offset = 0;
 int com_alibaba_wisp_engine_WispTask::_jdkParkStatus_offset = 0;
 int com_alibaba_wisp_engine_WispTask::_id_offset = 0;
@@ -4775,7 +4804,8 @@ int com_alibaba_wisp_engine_WispTask::_stealCount_offset = 0;
 int com_alibaba_wisp_engine_WispTask::_stealFailureCount_offset = 0;
 int com_alibaba_wisp_engine_WispTask::_preemptCount_offset = 0;
 int com_alibaba_wisp_engine_WispTask::_shutdownPending_offset = 0;
-
+int com_alibaba_wisp_engine_WispTask::_controlGroup_offset = 0;
+int com_alibaba_wisp_engine_WispTask::_ttr_offset = 0;
 
 #define WISPTASK_FIELDS_DO(macro) \
   macro(_jvmParkStatus_offset,      ik, vmSymbols::jvmParkStatus_name(),     int_signature,  false); \
@@ -4786,7 +4816,10 @@ int com_alibaba_wisp_engine_WispTask::_shutdownPending_offset = 0;
   macro(_activeCount_offset,        ik, vmSymbols::activeCount_name(),       int_signature,  false); \
   macro(_stealCount_offset,         ik, vmSymbols::stealCount_name(),        int_signature,  false); \
   macro(_stealFailureCount_offset,  ik, vmSymbols::stealFailureCount_name(), int_signature,  false); \
-  macro(_preemptCount_offset,       ik, vmSymbols::preemptCount_name(),      int_signature,  false)
+  macro(_preemptCount_offset,       ik, vmSymbols::preemptCount_name(),      int_signature,  false); \
+  macro(_shutdownPending_offset,    ik, vmSymbols::shutdownPending_name(),   bool_signature, false); \
+  macro(_controlGroup_offset,       ik, vmSymbols::controlGroup_name(),      vmSymbols::controlGroup_signature, false); \
+  macro(_ttr_offset,                ik, vmSymbols::ttr_name(),               long_signature, false);
 
 void com_alibaba_wisp_engine_WispTask::compute_offsets() {
   InstanceKlass* ik = vmClasses::com_alibaba_wisp_engine_WispTask_klass();
@@ -4800,6 +4833,62 @@ void com_alibaba_wisp_engine_WispTask::serialize_offsets(SerializeClosure* f) {
 }
 #endif
 
+oop com_alibaba_wisp_engine_WispTask::get_controlGroup(oop obj) {
+  return obj->obj_field(_controlGroup_offset);
+}
+
+long com_alibaba_wisp_engine_WispTask::get_ttr(oop obj) {
+  return obj->long_field(_ttr_offset);
+}
+
+#define WISPCG_FIELDS_DO(macro) \
+  macro(_cpuLimit_offset,      k, vmSymbols::cpuLimit_name(),     vmSymbols::controlGroup_Limit_signature,  false);
+
+int com_alibaba_wisp_engine_WispControlGroup::_cpuLimit_offset = 0;
+
+#if INCLUDE_CDS
+void com_alibaba_wisp_engine_WispControlGroup::serialize_offsets(SerializeClosure* f) {
+  WISPCG_FIELDS_DO(FIELD_SERIALIZE_OFFSET);
+}
+#endif
+
+void com_alibaba_wisp_engine_WispControlGroup::compute_offsets() {
+  InstanceKlass *k = vmClasses::com_alibaba_wisp_engine_WispControlGroup_klass();
+  assert(k != NULL, "WispControlGroup_klass is null");
+  WISPCG_FIELDS_DO(FIELD_COMPUTE_OFFSET);
+}
+
+oop com_alibaba_wisp_engine_WispControlGroup::get_cpuLimit(oop obj) {
+  return obj->obj_field(_cpuLimit_offset);
+}
+
+#define WISPCG_CPULIMIT_FIELDS_DO(macro) \
+  macro(_cfsPeriod_offset,      k, vmSymbols::cfsPeriod_name(),     long_signature,  false); \
+  macro(_cfsQuota_offset,       k, vmSymbols::cfsQuota_name(),      long_signature,  false);
+
+int com_alibaba_wisp_engine_WispControlGroup_CpuLimit::_cfsPeriod_offset = 0;
+int com_alibaba_wisp_engine_WispControlGroup_CpuLimit::_cfsQuota_offset = 0;
+
+void com_alibaba_wisp_engine_WispControlGroup_CpuLimit::compute_offsets() {
+  InstanceKlass *k = vmClasses::com_alibaba_wisp_engine_WispControlGroup_CpuLimit_klass();
+  assert(k != NULL, "WispControlGroup$CpuLimit_klass is null");
+  WISPCG_CPULIMIT_FIELDS_DO(FIELD_COMPUTE_OFFSET);
+}
+
+#if INCLUDE_CDS
+void com_alibaba_wisp_engine_WispControlGroup_CpuLimit::serialize_offsets(SerializeClosure* f) {
+  WISPCG_CPULIMIT_FIELDS_DO(FIELD_SERIALIZE_OFFSET);
+}
+#endif
+
+long com_alibaba_wisp_engine_WispControlGroup_CpuLimit::get_cfsPeriod(oop obj) {
+  return obj->long_field(_cfsPeriod_offset);
+}
+
+long com_alibaba_wisp_engine_WispControlGroup_CpuLimit::get_cfsQuota(oop obj) {
+  return obj->long_field(_cfsQuota_offset);
+}
+
 void com_alibaba_wisp_engine_WispTask::set_jvmParkStatus(oop obj, jint status) {
   obj->int_field_put(_jvmParkStatus_offset, status);
 }

src/hotspot/share/classfile/javaClasses.hpp

@@ -85,6 +85,9 @@ class RecordComponent;
   f(java_dyn_CoroutineBase) \
   f(com_alibaba_wisp_engine_WispCarrier) \
   f(com_alibaba_wisp_engine_WispTask) \
+  f(com_alibaba_wisp_engine_WispControlGroup) \
+  f(com_alibaba_wisp_engine_WispControlGroup_CpuLimit) \
+  f(com_alibaba_rcm_internal_AbstractResourceContainer) \
   //end
 
 #define BASIC_JAVA_CLASSES_DO(f) \
@@ -418,6 +421,7 @@ class java_lang_Thread : AllStatic {
   static int _tid_offset;
   static int _thread_status_offset;
   static int _park_blocker_offset;
+  static int _resourceContainer_offset;
 
   static void compute_offsets();
 
@@ -433,6 +437,7 @@ class java_lang_Thread : AllStatic {
   static void set_interrupted(oop java_thread, bool val);
   // Name
   static oop name(oop java_thread);
+  static oop resourceContainer(oop java_thread);
   static void set_name(oop java_thread, oop name);
   // Priority
   static ThreadPriority priority(oop java_thread);
@@ -1837,6 +1842,8 @@ class com_alibaba_wisp_engine_WispTask: AllStatic {
   static int _stealCount_offset;
   static int _stealFailureCount_offset;
   static int _preemptCount_offset;
+  static int _controlGroup_offset;
+  static int _ttr_offset;
   static int _shutdownPending_offset;
 public:
   static void set_jvmParkStatus(oop obj, jint status);
@@ -1851,12 +1858,44 @@ class com_alibaba_wisp_engine_WispTask: AllStatic {
   static int  get_stealFailureCount(oop obj);
   static int  get_preemptCount(oop obj);
   static void set_preemptCount(oop obj, jint count);
+  static oop  get_controlGroup(oop obj);
+  static oop  get_cpuLimit(oop obj);
+  static long get_ttr(oop obj);
   static bool get_shutdownPending(oop obj);
 
   static void compute_offsets();
   static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN;
 };
 
+class com_alibaba_wisp_engine_WispControlGroup: AllStatic {
+private:
+  static int _cpuLimit_offset;
+public:
+  static oop  get_cpuLimit(oop obj);
+  static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN;
+  static void compute_offsets();
+};
+
+class com_alibaba_wisp_engine_WispControlGroup_CpuLimit: AllStatic {
+private:
+  static int _cfsPeriod_offset;
+  static int _cfsQuota_offset;
+public:
+  static long get_cfsPeriod(oop obj);
+  static long get_cfsQuota(oop obj);
+  static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN;
+  static void compute_offsets();
+};
+
+class com_alibaba_rcm_internal_AbstractResourceContainer: AllStatic {
+private:
+  static int _id_offset;
+public:
+  static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN;
+  static long get_id(oop obj);
+  static void compute_offsets();
+};
+
 // Interface to hard-coded offset checking
 
 class JavaClasses : AllStatic {

src/hotspot/share/classfile/vmClassMacros.hpp

@@ -174,6 +174,12 @@
   do_klass(com_alibaba_wisp_engine_WispTask_klass,      com_alibaba_wisp_engine_WispTask                      ) \
   do_klass(com_alibaba_wisp_engine_WispTask_CacheableCoroutine_klass,                                           \
                                                         com_alibaba_wisp_engine_WispTask_CacheableCoroutine   ) \
+  do_klass(com_alibaba_wisp_engine_WispControlGroup_klass,                                                      \
+                                                        com_alibaba_wisp_engine_WispControlGroup              ) \
+  do_klass(com_alibaba_wisp_engine_WispControlGroup_CpuLimit_klass,                                             \
+                                                        com_alibaba_wisp_engine_WispControlGroup_CpuLimit     ) \
+  do_klass(com_alibaba_rcm_internal_AbstractResourceContainer_klass,                                            \
+                                                        com_alibaba_rcm_internal_AbstractResourceContainer    ) \
   do_klass(com_alibaba_wisp_engine_WispEngine_klass,    com_alibaba_wisp_engine_WispEngine                    ) \
   do_klass(com_alibaba_wisp_engine_WispCarrier_klass,   com_alibaba_wisp_engine_WispCarrier                   ) \
   do_klass(com_alibaba_wisp_engine_WispEventPump_klass, com_alibaba_wisp_engine_WispEventPump                 ) \

src/hotspot/share/classfile/vmSymbols.hpp

@@ -597,6 +597,9 @@
   template(char_StringBuffer_signature,               "(C)Ljava/lang/StringBuffer;")                              \
   template(int_String_signature,                      "(I)Ljava/lang/String;")                                    \
   template(boolean_boolean_int_signature,             "(ZZ)I")                                                    \
+  template(resourcecontainer_signature,               "Lcom/alibaba/rcm/internal/AbstractResourceContainer;")     \
+  template(controlGroup_signature,                    "Lcom/alibaba/wisp/engine/WispControlGroup;")               \
+  template(controlGroup_Limit_signature,              "Lcom/alibaba/wisp/engine/WispControlGroup$CpuLimit;")      \
   template(big_integer_shift_worker_signature,        "([I[IIII)V")                                               \
   template(reflect_method_signature,                  "Ljava/lang/reflect/Method;")                               \
   template(getStackTrace_signature,                    "()[Ljava/lang/StackTraceElement;")                        \
@@ -694,13 +697,21 @@
   template(java_dyn_CoroutineExitException,            "java/dyn/CoroutineExitException")                         \
   template(com_alibaba_wisp_engine_WispTask,           "com/alibaba/wisp/engine/WispTask")                        \
   template(com_alibaba_wisp_engine_WispTask_CacheableCoroutine, "com/alibaba/wisp/engine/WispTask$CacheableCoroutine") \
+  template(com_alibaba_wisp_engine_WispControlGroup,   "com/alibaba/wisp/engine/WispControlGroup")                \
+  template(com_alibaba_wisp_engine_WispControlGroup_CpuLimit, "com/alibaba/wisp/engine/WispControlGroup$CpuLimit") \
+  template(com_alibaba_rcm_internal_AbstractResourceContainer, "com/alibaba/rcm/internal/AbstractResourceContainer") \
   template(com_alibaba_wisp_engine_WispEngine,         "com/alibaba/wisp/engine/WispEngine")                      \
   template(com_alibaba_wisp_engine_WispCarrier,        "com/alibaba/wisp/engine/WispCarrier")                     \
   template(com_alibaba_wisp_engine_WispEventPump,      "com/alibaba/wisp/engine/WispEventPump")                   \
   template(isInCritical_name,                          "isInCritical")                                            \
   template(jdkParkStatus_name,                         "jdkParkStatus")                                           \
   template(shutdownPending_name,                       "shutdownPending")                                         \
   template(jvmParkStatus_name,                         "jvmParkStatus")                                           \
+  template(ttr_name,                                   "ttr")                                                     \
+  template(controlGroup_name,                          "controlGroup")                                            \
+  template(cpuLimit_name,                              "cpuLimit")                                                \
+  template(cfsPeriod_name,                             "cfsPeriod")                                               \
+  template(cfsQuota_name,                              "cfsQuota")                                                \
   template(id_name,                                    "id")                                                      \
   template(threadWrapper_name,                         "threadWrapper")                                           \
   template(activeCount_name,                           "activeCount")                                             \

src/hotspot/share/include/jvm.h

@@ -1059,7 +1059,7 @@ JNIEXPORT jint JNICALL
 JVM_GetProxyUnpark(JNIEnv* env, jclass clz, jintArray res);
 
 JNIEXPORT void JNICALL
-JVM_MarkPreempted(JNIEnv* env, jclass clz, jobject thread, jboolean force);
+JVM_MarkPreempted(JNIEnv* env, jclass clz, jobject thread);
 
 /*************************************************************************
  PART 3: I/O and Network Support

src/hotspot/share/prims/jvm.cpp

@@ -3897,7 +3897,7 @@ JVM_ENTRY(jint, JVM_GetProxyUnpark(JNIEnv* env, jclass klass, jintArray res))
   return WispThread::get_proxy_unpark(res);
 JVM_END
 
-JVM_ENTRY(void, JVM_MarkPreempted(JNIEnv* env, jclass klass, jobject threadObj, jboolean force))
+JVM_ENTRY(void, JVM_MarkPreempted(JNIEnv* env, jclass klass, jobject threadObj))
   assert(EnableCoroutine, "Coroutine is disabled");
   JavaThread* thr = NULL;
   {
@@ -3911,11 +3911,9 @@ JVM_ENTRY(void, JVM_MarkPreempted(JNIEnv* env, jclass klass, jobject threadObj,
     }
     thr->set_wisp_preempted(true);
   }
-  // fire an empty safepoint to let the thread go check flag
-  if (force) {
-    VM_ForceSafepoint force_safepoint_op;
-    VMThread::execute(&force_safepoint_op);
-  }
+  // fire an thread-local handshake to let the thread go check flag
+  CoroutinePreemptClosure cps;
+  Handshake::execute(&cps, thr);
 JVM_END
 
 // Returns an array of java.lang.String objects containing the input arguments to the VM.

src/hotspot/share/runtime/coroutine.cpp

@@ -512,21 +512,44 @@ void Coroutine::print_stack_header_on(outputStream* st) {
       assert(_wisp_thread->threadObj() == thread_obj, "WispThread thread object is not updated from WispTask");
     }
 #endif
+    long cgId  = 0;
     char buf[128] = "<cached>";
     if (thread_obj != NULL) {
       oop name = java_lang_Thread::name(thread_obj);
       if (name != NULL) {
         java_lang_String::as_utf8_string(name, buf, sizeof(buf));
       }
+      oop container = java_lang_Thread::resourceContainer(thread_obj);
+      if (container != NULL) {
+        cgId = com_alibaba_rcm_internal_AbstractResourceContainer::get_id(container);
+      }
+    }
+    // calculate wisp container's cfs_period and cfs_quota
+    long cfsPeriod = 0;
+    long cfsQuota  = 0;
+    // step one: get WispControlGroup
+    oop wispControlGroup = com_alibaba_wisp_engine_WispTask::get_controlGroup(_wisp_task);
+    if (wispControlGroup != NULL) {
+      // step two: get CpuLimit
+      oop cpuLimit = com_alibaba_wisp_engine_WispControlGroup::get_cpuLimit(wispControlGroup);
+      guarantee(cpuLimit != NULL, "cpuLimit in WispControlGroup should not be null");
+      // step three: get cfsPeriod and cfsQuota
+      cfsPeriod = com_alibaba_wisp_engine_WispControlGroup_CpuLimit::get_cfsPeriod(cpuLimit);
+      cfsQuota  = com_alibaba_wisp_engine_WispControlGroup_CpuLimit::get_cfsQuota(cpuLimit);
     }
-    st->print(" \"%s\" #%d active=%d steal=%d steal_fail=%d preempt=%d park=%d/%d", buf,
+
+
+    st->print(" \"%s\" #%d active=%d steal=%d steal_fail=%d preempt=%d park=%d/%d containerId=%ld cg=%ld/%ld ttr=%ld", buf,
         com_alibaba_wisp_engine_WispTask::get_id(_wisp_task),
         com_alibaba_wisp_engine_WispTask::get_activeCount(_wisp_task),
         com_alibaba_wisp_engine_WispTask::get_stealCount(_wisp_task),
         com_alibaba_wisp_engine_WispTask::get_stealFailureCount(_wisp_task),
         com_alibaba_wisp_engine_WispTask::get_preemptCount(_wisp_task),
         com_alibaba_wisp_engine_WispTask::get_jvmParkStatus(_wisp_task),
-        com_alibaba_wisp_engine_WispTask::get_jdkParkStatus(_wisp_task));
+        com_alibaba_wisp_engine_WispTask::get_jdkParkStatus(_wisp_task),
+        cgId,
+        cfsQuota / 1000, cfsPeriod / 1000,
+        com_alibaba_wisp_engine_WispTask::get_ttr(_wisp_task));
   }
   if (_wisp_thread && PrintThreadCoroutineInfo) {
     st->print(" monitor_park_stage=%s", WispThread::print_blocking_status(_wisp_thread->_unpark_status));

src/hotspot/share/runtime/coroutine.hpp

@@ -695,4 +695,13 @@ class WispClinitCounterMark : public StackObj {
   JavaThread*   _thread;
 };
 
+class CoroutinePreemptClosure : public HandshakeClosure{
+public:
+  CoroutinePreemptClosure():HandshakeClosure("CoroutinePreempt") {}
+
+  void do_thread(Thread* thread) {
+  // do nothing
+  }
+};
+
 #endif // SHARE_VM_RUNTIME_COROUTINE_HPP