8315884: New Object to ObjectMonitor mapping

Co-authored-by: Erik Österlund <[email protected]>
Co-authored-by: Stefan Karlsson <[email protected]>
Co-authored-by: Coleen Phillimore <[email protected]>
Reviewed-by: rkennke, coleenp, dcubed

src/hotspot/cpu/aarch64/aarch64.ad

@@ -15793,7 +15793,7 @@ instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp
   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
 
   ins_encode %{
-    __ fast_lock_lightweight($object$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
+    __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
   %}
 
   ins_pipe(pipe_serial);
@@ -15809,7 +15809,7 @@ instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNo
   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
 
   ins_encode %{
-    __ fast_unlock_lightweight($object$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
+    __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
   %}
 
   ins_pipe(pipe_serial);

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -81,7 +81,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
   }
 
   if (LockingMode == LM_LIGHTWEIGHT) {
-    lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
+    lightweight_lock(disp_hdr, obj, hdr, temp, rscratch2, slow_case);
   } else if (LockingMode == LM_LEGACY) {
     Label done;
     // Load object header

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -224,10 +224,10 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
   bind(no_count);
 }
 
-void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register t1,
                                               Register t2, Register t3) {
   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
-  assert_different_registers(obj, t1, t2, t3);
+  assert_different_registers(obj, box, t1, t2, t3);
 
   // Handle inflated monitor.
   Label inflated;
@@ -236,6 +236,11 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
   // Finish fast lock unsuccessfully. MUST branch to with flag == NE
   Label slow_path;
 
+  if (UseObjectMonitorTable) {
+    // Clear cache in case fast locking succeeds.
+    str(zr, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
+  }
+
   if (DiagnoseSyncOnValueBasedClasses != 0) {
     load_klass(t1, obj);
     ldrw(t1, Address(t1, Klass::access_flags_offset()));
@@ -244,14 +249,14 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
   }
 
   const Register t1_mark = t1;
+  const Register t3_t = t3;
 
   { // Lightweight locking
 
     // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
     Label push;
 
     const Register t2_top = t2;
-    const Register t3_t = t3;
 
     // Check if lock-stack is full.
     ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
@@ -289,26 +294,71 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
   { // Handle inflated monitor.
     bind(inflated);
 
-    // mark contains the tagged ObjectMonitor*.
-    const Register t1_tagged_monitor = t1_mark;
-    const uintptr_t monitor_tag = markWord::monitor_value;
+    const Register t1_monitor = t1;
+
+    if (!UseObjectMonitorTable) {
+      assert(t1_monitor == t1_mark, "should be the same here");
+    } else {
+      Label monitor_found;
+
+      // Load cache address
+      lea(t3_t, Address(rthread, JavaThread::om_cache_oops_offset()));
+
+      const int num_unrolled = 2;
+      for (int i = 0; i < num_unrolled; i++) {
+        ldr(t1, Address(t3_t));
+        cmp(obj, t1);
+        br(Assembler::EQ, monitor_found);
+        increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
+      }
+
+      Label loop;
+
+      // Search for obj in cache.
+      bind(loop);
+
+      // Check for match.
+      ldr(t1, Address(t3_t));
+      cmp(obj, t1);
+      br(Assembler::EQ, monitor_found);
+
+      // Search until null encountered, guaranteed _null_sentinel at end.
+      increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
+      cbnz(t1, loop);
+      // Cache Miss, NE set from cmp above, cbnz does not set flags
+      b(slow_path);
+
+      bind(monitor_found);
+      ldr(t1_monitor, Address(t3_t, OMCache::oop_to_monitor_difference()));
+    }
+
     const Register t2_owner_addr = t2;
     const Register t3_owner = t3;
+    const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
+    const Address owner_address(t1_monitor, ObjectMonitor::owner_offset() - monitor_tag);
+    const Address recursions_address(t1_monitor, ObjectMonitor::recursions_offset() - monitor_tag);
+
+    Label monitor_locked;
 
     // Compute owner address.
-    lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
+    lea(t2_owner_addr, owner_address);
 
     // CAS owner (null => current thread).
     cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
             /*release*/ false, /*weak*/ false, t3_owner);
-    br(Assembler::EQ, locked);
+    br(Assembler::EQ, monitor_locked);
 
     // Check if recursive.
     cmp(t3_owner, rthread);
     br(Assembler::NE, slow_path);
 
     // Recursive.
-    increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
+    increment(recursions_address, 1);
+
+    bind(monitor_locked);
+    if (UseObjectMonitorTable) {
+      str(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
+    }
   }
 
   bind(locked);
@@ -331,13 +381,13 @@ void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
   // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
 }
 
-void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
-                                                Register t3) {
+void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Register t1,
+                                                Register t2, Register t3) {
   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
-  assert_different_registers(obj, t1, t2, t3);
+  assert_different_registers(obj, box, t1, t2, t3);
 
   // Handle inflated monitor.
-  Label inflated, inflated_load_monitor;
+  Label inflated, inflated_load_mark;
   // Finish fast unlock successfully. MUST branch to with flag == EQ
   Label unlocked;
   // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
@@ -349,13 +399,15 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Regis
 
   { // Lightweight unlock
 
+    Label push_and_slow_path;
+
     // Check if obj is top of lock-stack.
     ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
     subw(t2_top, t2_top, oopSize);
     ldr(t3_t, Address(rthread, t2_top));
     cmp(obj, t3_t);
     // Top of lock stack was not obj. Must be monitor.
-    br(Assembler::NE, inflated_load_monitor);
+    br(Assembler::NE, inflated_load_mark);
 
     // Pop lock-stack.
     DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
@@ -372,7 +424,10 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Regis
     ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 
     // Check header for monitor (0b10).
-    tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+    // Because we got here by popping (meaning we pushed in locked)
+    // there will be no monitor in the box. So we need to push back the obj
+    // so that the runtime can fix any potential anonymous owner.
+    tbnz(t1_mark, exact_log2(markWord::monitor_value), UseObjectMonitorTable ? push_and_slow_path : inflated);
 
     // Try to unlock. Transition lock bits 0b00 => 0b01
     assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
@@ -381,6 +436,7 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Regis
             /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
     br(Assembler::EQ, unlocked);
 
+    bind(push_and_slow_path);
     // Compare and exchange failed.
     // Restore lock-stack and handle the unlock in runtime.
     DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
@@ -391,7 +447,7 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Regis
 
 
   { // Handle inflated monitor.
-    bind(inflated_load_monitor);
+    bind(inflated_load_mark);
     ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 #ifdef ASSERT
     tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
@@ -412,12 +468,19 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Regis
     bind(check_done);
 #endif
 
-    // mark contains the tagged ObjectMonitor*.
-    const Register t1_monitor = t1_mark;
-    const uintptr_t monitor_tag = markWord::monitor_value;
+    const Register t1_monitor = t1;
+
+    if (!UseObjectMonitorTable) {
+      assert(t1_monitor == t1_mark, "should be the same here");
 
-    // Untag the monitor.
-    sub(t1_monitor, t1_mark, monitor_tag);
+      // Untag the monitor.
+      add(t1_monitor, t1_mark, -(int)markWord::monitor_value);
+    } else {
+      ldr(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
+      // null check with Flags == NE, no valid pointer below alignof(ObjectMonitor*)
+      cmp(t1_monitor, checked_cast<uint8_t>(alignof(ObjectMonitor*)));
+      br(Assembler::LO, slow_path);
+    }
 
     const Register t2_recursions = t2;
     Label not_recursive;

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp

@@ -39,8 +39,8 @@
   void fast_lock(Register object, Register box, Register tmp, Register tmp2, Register tmp3);
   void fast_unlock(Register object, Register box, Register tmp, Register tmp2);
   // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file.
-  void fast_lock_lightweight(Register object, Register t1, Register t2, Register t3);
-  void fast_unlock_lightweight(Register object, Register t1, Register t2, Register t3);
+  void fast_lock_lightweight(Register object, Register box, Register t1, Register t2, Register t3);
+  void fast_unlock_lightweight(Register object, Register box, Register t1, Register t2, Register t3);
 
   void string_compare(Register str1, Register str2,
                       Register cnt1, Register cnt2, Register result,

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

@@ -696,7 +696,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
     }
 
     if (LockingMode == LM_LIGHTWEIGHT) {
-      lightweight_lock(obj_reg, tmp, tmp2, tmp3, slow_case);
+      lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
       b(count);
     } else if (LockingMode == LM_LEGACY) {
       // Load (object->mark() | 1) into swap_reg
@@ -752,15 +752,9 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
     bind(slow_case);
 
     // Call the runtime routine for slow case
-    if (LockingMode == LM_LIGHTWEIGHT) {
-      call_VM(noreg,
-              CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj),
-              obj_reg);
-    } else {
-      call_VM(noreg,
-              CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
-              lock_reg);
-    }
+    call_VM(noreg,
+            CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
+            lock_reg);
     b(done);
 
     bind(count);

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -6750,9 +6750,9 @@ void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
 //  - obj: the object to be locked
 //  - t1, t2, t3: temporary registers, will be destroyed
 //  - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding).
-void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
+void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow) {
   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
-  assert_different_registers(obj, t1, t2, t3, rscratch1);
+  assert_different_registers(basic_lock, obj, t1, t2, t3, rscratch1);
 
   Label push;
   const Register top = t1;
@@ -6763,6 +6763,11 @@ void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Re
   // instruction emitted as it is part of C1's null check semantics.
   ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 
+  if (UseObjectMonitorTable) {
+    // Clear cache in case fast locking succeeds.
+    str(zr, Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))));
+  }
+
   // Check if the lock-stack is full.
   ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
   cmpw(top, (unsigned)LockStack::end_offset());

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1639,7 +1639,7 @@ class MacroAssembler: public Assembler {
   // Code for java.lang.Thread::onSpinWait() intrinsic.
   void spin_wait();
 
-  void lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow);
+  void lightweight_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow);
   void lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
 
 private:

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -1811,7 +1811,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ br(Assembler::NE, slow_path_lock);
     } else {
       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
-      __ lightweight_lock(obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
+      __ lightweight_lock(lock_reg, obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
     }
     __ bind(count);
     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));

src/hotspot/cpu/arm/interp_masm_arm.cpp

@@ -985,15 +985,7 @@ void InterpreterMacroAssembler::lock_object(Register Rlock) {
     bind(slow_case);
 
     // Call the runtime routine for slow case
-    if (LockingMode == LM_LIGHTWEIGHT) {
-      // Pass oop, not lock, in fast lock case. call_VM wants R1 though.
-      push(R1);
-      mov(R1, Robj);
-      call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj), R1);
-      pop(R1);
-    } else {
-      call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
-    }
+    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
     bind(done);
   }
 }

src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp

@@ -1043,11 +1043,7 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
     // None of the above fast optimizations worked so we have to get into the
     // slow case of monitor enter.
     bind(slow_case);
-    if (LockingMode == LM_LIGHTWEIGHT) {
-      call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj), object);
-    } else {
-      call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
-    }
+    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
     b(done);
     // }
     align(32, 12);