Merge pull request #21308 from fengxue-IS/jep491-0.51-base

(v0.51.0) Support JEP491 in VM code path

Author: babsingh

jcl/src/java.base/share/classes/jdk/internal/vm/Continuation.java

@@ -38,6 +38,10 @@ public class Continuation {
 	private long vmRef; /* J9VMContinuation */
 	protected Thread vthread; /* Parent VirtualThread */
 
+/*[IF JAVA_SPEC_VERSION >= 24]*/
+	private Object blocker;
+/*[ENDIF] JAVA_SPEC_VERSION >= 24 */
+
 	/* The live thread's scopedValueCache is always kept in J9VMThread->scopedValueCache
 	 * whereas the unmounted thread's scopedValueCache is stored in this field. This
 	 * field is modified in ContinuationHelpers.hpp::swapFieldsWithContinuation. This
@@ -71,7 +75,13 @@ public enum Pinned {
 		/** Holding monitor(s) */
 		MONITOR(2),
 		/** In critical section */
+/*[IF JAVA_SPEC_VERSION >= 24]*/
+		CRITICAL_SECTION(3),
+		/** Exception */
+		EXCEPTION(4);
+/*[ELSE] JAVA_SPEC_VERSION >= 24 */
 		CRITICAL_SECTION(3);
+/*[ENDIF] JAVA_SPEC_VERSION >= 24 */
 
 		private final int errorCode;
 
@@ -250,6 +260,10 @@ private boolean yield0() {
 				reason = Pinned.MONITOR;
 			} else if (rcPinned == Pinned.NATIVE.errorCode()) {
 				reason = Pinned.NATIVE;
+/*[IF JAVA_SPEC_VERSION >= 24]*/
+			} else if (rcPinned == Pinned.EXCEPTION.errorCode()) {
+				reason = Pinned.EXCEPTION;
+/*[ENDIF] JAVA_SPEC_VERSION >= 24 */
 			} else {
 				throw new AssertionError("Unknown pinned error code: " + rcPinned);
 			}

runtime/j9vm/javanextvmi.cpp

@@ -35,6 +35,9 @@
 #include "VMHelpers.hpp"
 #include "ContinuationHelpers.hpp"
 #endif /* JAVA_SPEC_VERSION >= 19 */
+#if JAVA_SPEC_VERSION >= 24
+#include "j9protos.h"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 
 extern "C" {
 
@@ -260,89 +263,27 @@ JVM_IsPreviewEnabled(void)
 	return isPreviewEnabled;
 }
 
-static void
-enterVThreadTransitionCritical(J9VMThread *currentThread, jobject thread)
-{
-	J9JavaVM *vm = currentThread->javaVM;
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
-	MM_ObjectAccessBarrierAPI objectAccessBarrier = MM_ObjectAccessBarrierAPI(currentThread);
-	j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
-
-retry:
-	if (!VM_VMHelpers::isThreadSuspended(currentThread, threadObj)) {
-		while (!objectAccessBarrier.inlineMixedObjectCompareAndSwapU64(currentThread, threadObj, vm->virtualThreadInspectorCountOffset, 0, ~(U_64)0)) {
-			/* Thread is being inspected or unmounted, wait. */
-			vmFuncs->internalReleaseVMAccess(currentThread);
-			VM_AtomicSupport::yieldCPU();
-			/* After wait, the thread may suspend here. */
-			vmFuncs->internalAcquireVMAccess(currentThread);
-			threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
-		}
-
-		/* Now we have locked access to virtualThreadInspectorCount, check if the vthread is suspended.
-		 * If suspended, release the access and spin-wait until the vthread is resumed.
-		 * If not suspended, link the current J9VMThread with the virtual thread object.
-		 */
-		if (!VM_VMHelpers::isThreadSuspended(currentThread, threadObj)
-		&& objectAccessBarrier.inlineMixedObjectCompareAndSwapU64(currentThread, threadObj, vm->internalSuspendStateOffset, J9_VIRTUALTHREAD_INTERNAL_STATE_NONE, (U_64)currentThread)
-		) {
-			return;
-		}
-		J9OBJECT_I64_STORE(currentThread, threadObj, vm->virtualThreadInspectorCountOffset, 0);
-	}
-	vmFuncs->internalReleaseVMAccess(currentThread);
-	/* Spin is used instead of the halt flag as we cannot guarantee suspend flag is still set now.
-	 *
-	 * TODO: Dynamically increase the sleep time to a bounded maximum.
-	 */
-	f_threadSleep(10);
-	/* After wait, the thread may suspend here. */
-	vmFuncs->internalAcquireVMAccess(currentThread);
-	threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
-	goto retry;
-}
-
-static void
-exitVThreadTransitionCritical(J9VMThread *currentThread, jobject thread)
-{
-	J9JavaVM *vm = currentThread->javaVM;
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
-	j9object_t vthread = J9_JNI_UNWRAP_REFERENCE(thread);
-	MM_ObjectAccessBarrierAPI objectAccessBarrier = MM_ObjectAccessBarrierAPI(currentThread);
-
-	/* Remove J9VMThread address from internalSuspendedState field, as the thread state is no longer in a transition. */
-	while (!objectAccessBarrier.inlineMixedObjectCompareAndSwapU64(currentThread, vthread, vm->internalSuspendStateOffset, (U_64)currentThread, J9_VIRTUALTHREAD_INTERNAL_STATE_NONE)) {
-		/* Wait if the suspend flag is set. */
-		vmFuncs->internalReleaseVMAccess(currentThread);
-		VM_AtomicSupport::yieldCPU();
-		/* After wait, the thread may suspend here. */
-		vmFuncs->internalAcquireVMAccess(currentThread);
-		vthread = J9_JNI_UNWRAP_REFERENCE(thread);
-	}
-
-	/* Update to virtualThreadInspectorCount must be after clearing isSuspendedInternal field to retain sync ordering. */
-	Assert_SC_true(-1 == J9OBJECT_I64_LOAD(currentThread, vthread, vm->virtualThreadInspectorCountOffset));
-	J9OBJECT_I64_STORE(currentThread, vthread, vm->virtualThreadInspectorCountOffset, 0);
-}
-
 static void
 setContinuationStateToLastUnmount(J9VMThread *currentThread, jobject thread)
 {
-	enterVThreadTransitionCritical(currentThread, thread);
+	J9InternalVMFunctions const * const vmFuncs = currentThread->javaVM->internalVMFunctions;
+	vmFuncs->enterVThreadTransitionCritical(currentThread, thread);
+
 	/* Re-fetch reference as enterVThreadTransitionCritical may release VMAccess. */
 	j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
 	j9object_t continuationObj = J9VMJAVALANGVIRTUALTHREAD_CONT(currentThread, threadObj);
 	ContinuationState volatile *continuationStatePtr = VM_ContinuationHelpers::getContinuationStateAddress(currentThread, continuationObj);
 	/* Used in JVMTI to not suspend the virtual thread once it enters the last unmount phase. */
 	VM_ContinuationHelpers::setLastUnmount(continuationStatePtr);
-	exitVThreadTransitionCritical(currentThread, thread);
+	vmFuncs->exitVThreadTransitionCritical(currentThread, thread);
 }
 
 /* Caller must have VMAccess. */
 static void
 virtualThreadMountBegin(JNIEnv *env, jobject thread)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
+	J9InternalVMFunctions const * const vmFuncs = currentThread->javaVM->internalVMFunctions;
 
 	j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
 	Assert_SC_true(IS_JAVA_LANG_VIRTUALTHREAD(currentThread, threadObj));
@@ -361,7 +302,7 @@ virtualThreadMountBegin(JNIEnv *env, jobject thread)
 				continuation);
 	}
 
-	enterVThreadTransitionCritical(currentThread, thread);
+	vmFuncs->enterVThreadTransitionCritical(currentThread, thread);
 
 	VM_VMHelpers::virtualThreadHideFrames(currentThread, JNI_TRUE);
 }
@@ -372,6 +313,7 @@ virtualThreadMountEnd(JNIEnv *env, jobject thread)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
 	J9JavaVM *vm = currentThread->javaVM;
+	J9InternalVMFunctions const * const vmFuncs = vm->internalVMFunctions;
 	j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
 
 	Assert_SC_true(IS_JAVA_LANG_VIRTUALTHREAD(currentThread, threadObj));
@@ -391,7 +333,7 @@ virtualThreadMountEnd(JNIEnv *env, jobject thread)
 	VM_VMHelpers::virtualThreadHideFrames(currentThread, JNI_FALSE);
 
 	/* Allow thread to be inspected again. */
-	exitVThreadTransitionCritical(currentThread, thread);
+	vmFuncs->exitVThreadTransitionCritical(currentThread, thread);
 
 	TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_MOUNT(vm->hookInterface, currentThread);
 }
@@ -402,6 +344,7 @@ virtualThreadUnmountBegin(JNIEnv *env, jobject thread)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
 	J9JavaVM *vm = currentThread->javaVM;
+	J9InternalVMFunctions const * const vmFuncs = vm->internalVMFunctions;
 
 	j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
 
@@ -421,16 +364,15 @@ virtualThreadUnmountBegin(JNIEnv *env, jobject thread)
 
 	TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_UNMOUNT(vm->hookInterface, currentThread);
 
-	enterVThreadTransitionCritical(currentThread, thread);
+	vmFuncs->enterVThreadTransitionCritical(currentThread, thread);
 
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
 	j9object_t carrierThreadObject = currentThread->carrierThreadObject;
 	/* Virtual thread is being umounted. If its carrier thread is suspended, spin until
 	 * the carrier thread is resumed. The carrier thread should not be mounted until it
 	 * is resumed.
 	 */
 	while (VM_VMHelpers::isThreadSuspended(currentThread, carrierThreadObject)) {
-		exitVThreadTransitionCritical(currentThread, thread);
+		vmFuncs->exitVThreadTransitionCritical(currentThread, thread);
 		vmFuncs->internalReleaseVMAccess(currentThread);
 		/* Spin is used instead of the halt flag; otherwise, the virtual thread will
 		 * show as suspended.
@@ -439,7 +381,7 @@ virtualThreadUnmountBegin(JNIEnv *env, jobject thread)
 		 */
 		f_threadSleep(10);
 		vmFuncs->internalAcquireVMAccess(currentThread);
-		enterVThreadTransitionCritical(currentThread, thread);
+		vmFuncs->enterVThreadTransitionCritical(currentThread, thread);
 		carrierThreadObject = currentThread->carrierThreadObject;
 	}
 
@@ -452,7 +394,7 @@ virtualThreadUnmountEnd(JNIEnv *env, jobject thread)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
 	J9JavaVM *vm = currentThread->javaVM;
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
+	J9InternalVMFunctions const * const vmFuncs = vm->internalVMFunctions;
 
 	j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread);
 	j9object_t continuationObj = J9VMJAVALANGVIRTUALTHREAD_CONT(currentThread, threadObj);
@@ -478,7 +420,7 @@ virtualThreadUnmountEnd(JNIEnv *env, jobject thread)
 	VM_VMHelpers::virtualThreadHideFrames(currentThread, JNI_FALSE);
 
 	/* Allow thread to be inspected again. */
-	exitVThreadTransitionCritical(currentThread, thread);
+	vmFuncs->exitVThreadTransitionCritical(currentThread, thread);
 }
 #endif /* JAVA_SPEC_VERSION >= 19 */
 
@@ -534,14 +476,14 @@ JVM_VirtualThreadHideFrames(
 #if JAVA_SPEC_VERSION < 23
 		Assert_SC_true(vThreadObj == J9_JNI_UNWRAP_REFERENCE(vthread));
 #endif /* JAVA_SPEC_VERSION < 23 */
-		enterVThreadTransitionCritical(currentThread, (jobject)&currentThread->threadObject);
+		vmFuncs->enterVThreadTransitionCritical(currentThread, (jobject)&currentThread->threadObject);
 	}
 
 	VM_VMHelpers::virtualThreadHideFrames(currentThread, hide);
 
 	if (!hide) {
 		Assert_SC_true(hiddenFrames);
-		exitVThreadTransitionCritical(currentThread, (jobject)&currentThread->threadObject);
+		vmFuncs->exitVThreadTransitionCritical(currentThread, (jobject)&currentThread->threadObject);
 	}
 
 	vmFuncs->internalExitVMToJNI(currentThread);
@@ -567,7 +509,7 @@ JVM_VirtualThreadMount(JNIEnv *env, jobject vthread, jboolean hide)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
 	J9JavaVM *vm = currentThread->javaVM;
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
+	J9InternalVMFunctions const * const vmFuncs = vm->internalVMFunctions;
 
 	Trc_SC_VirtualThreadMount_Entry(currentThread, vthread, hide);
 
@@ -589,7 +531,7 @@ JVM_VirtualThreadUnmount(JNIEnv *env, jobject vthread, jboolean hide)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
 	J9JavaVM *vm = currentThread->javaVM;
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
+	J9InternalVMFunctions const * const vmFuncs = vm->internalVMFunctions;
 
 	Trc_SC_VirtualThreadUnmount_Entry(currentThread, vthread, hide);
 
@@ -617,7 +559,7 @@ JVM_VirtualThreadStart(JNIEnv *env, jobject vthread)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
 	J9JavaVM *vm = currentThread->javaVM;
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
+	J9InternalVMFunctions const * const vmFuncs = vm->internalVMFunctions;
 
 	Trc_SC_VirtualThreadStart_Entry(currentThread, vthread);
 
@@ -636,7 +578,7 @@ JVM_VirtualThreadEnd(JNIEnv *env, jobject vthread)
 {
 	J9VMThread *currentThread = (J9VMThread *)env;
 	J9JavaVM *vm = currentThread->javaVM;
-	J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions;
+	J9InternalVMFunctions const * const vmFuncs = vm->internalVMFunctions;
 
 	Trc_SC_VirtualThreadEnd_Entry(currentThread, vthread);
 
@@ -720,17 +662,19 @@ JVM_IsStaticallyLinked(void)
 }
 
 JNIEXPORT void JNICALL
-JVM_VirtualThreadPinnedEvent(JNIEnv* env, jclass clazz, jstring op)
+JVM_VirtualThreadPinnedEvent(JNIEnv *env, jclass clazz, jstring op)
 {
 	// TODO: emit JFR Event
 	return;
 }
 
 JNIEXPORT jobject JNICALL
-JVM_TakeVirtualThreadListToUnblock(JNIEnv* env, jclass ignored)
+JVM_TakeVirtualThreadListToUnblock(JNIEnv *env, jclass ignored)
 {
-	// TODO: return the unblocked list
-	return NULL;
+	J9VMThread *currentThread = (J9VMThread *)env;
+	J9JavaVM *vm = currentThread->javaVM;
+
+	return vm->internalVMFunctions->takeVirtualThreadListToUnblock(currentThread, vm);
 }
 #endif /* JAVA_SPEC_VERSION >= 24 */
 

runtime/oti/ContinuationHelpers.hpp

@@ -30,6 +30,9 @@
 #define J9VM_CONTINUATION_PINNED_REASON_NATIVE 1
 #define J9VM_CONTINUATION_PINNED_REASON_MONITOR 2
 #define J9VM_CONTINUATION_PINNED_REASON_CRITICAL_SECTION 3
+#if JAVA_SPEC_VERSION >= 24
+#define J9VM_CONTINUATION_PINNED_REASON_EXCEPTION 4
+#endif /* JAVA_SPEC_VERSION >= 24 */
 
 class VM_ContinuationHelpers {
 	/*
@@ -88,6 +91,15 @@ class VM_ContinuationHelpers {
 		j9object_t scopedValueCache = J9VMJDKINTERNALVMCONTINUATION_SCOPEDVALUECACHE(vmThread, continuationObject);
 		J9VMJDKINTERNALVMCONTINUATION_SET_SCOPEDVALUECACHE(vmThread, continuationObject, vmThread->scopedValueCache);
 		vmThread->scopedValueCache = scopedValueCache;
+
+#if JAVA_SPEC_VERSION >= 24
+		if (J9_ARE_ANY_BITS_SET(vmThread->javaVM->extendedRuntimeFlags3, J9_EXTENDED_RUNTIME3_YIELD_PINNED_CONTINUATION)) {
+			SWAP_MEMBER(ownedMonitorCount, UDATA, vmThread, continuation);
+			SWAP_MEMBER(monitorEnterRecordPool, J9Pool*, vmThread, continuation);
+			SWAP_MEMBER(monitorEnterRecords, J9MonitorEnterRecord*, vmThread, continuation);
+			SWAP_MEMBER(jniMonitorEnterRecords, J9MonitorEnterRecord*, vmThread, continuation);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
 	}
 
 	static VMINLINE ContinuationState volatile *
@@ -261,6 +273,24 @@ class VM_ContinuationHelpers {
 
 		return threadObject;
 	}
+
+#if JAVA_SPEC_VERSION >= 24
+	/**
+	 * Check if the threadObject mounted on a J9VMThread is a virtual thread and can be yielded.
+	 *
+	 * @param[in] vmThread the J9VMThread
+	 *
+	 * @return true if the virtual thread is yieldable, otherwise false
+	 */
+	static VMINLINE bool
+	isYieldableVirtualThread(J9VMThread *vmThread)
+	{
+		return (J9_ARE_ANY_BITS_SET(vmThread->javaVM->extendedRuntimeFlags3, J9_EXTENDED_RUNTIME3_YIELD_PINNED_CONTINUATION)
+				&& IS_JAVA_LANG_VIRTUALTHREAD(vmThread, vmThread->threadObject)
+				&& (0 == vmThread->continuationPinCount)
+				&& (0 == vmThread->callOutCount));
+	}
+#endif /* JAVA_SPEC_VERSION >= 24 */
 };
 
 #endif /* CONTINUATIONHELPERS_HPP_ */

runtime/oti/j9consts.h

@@ -371,6 +371,7 @@ extern "C" {
 
 /* constants for J9JavaVM.extendedRuntimeFlags3 */
 #define J9_EXTENDED_RUNTIME3_START_FLIGHT_RECORDING 0x1
+#define J9_EXTENDED_RUNTIME3_YIELD_PINNED_CONTINUATION 0x2
 
 #define J9_OBJECT_HEADER_AGE_DEFAULT 0xA /* OBJECT_HEADER_AGE_DEFAULT */
 #define J9_OBJECT_HEADER_SHAPE_MASK 0xE /* OBJECT_HEADER_SHAPE_MASK */