8374169: too many unsafe intrinsics (simplify unsafe intrinsic processing)

src/hotspot/share/c1/c1_Compiler.cpp

@@ -109,21 +109,20 @@ bool Compiler::is_intrinsic_supported(const methodHandle& method) {
 
 bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   switch (id) {
-  case vmIntrinsics::_compareAndSetLong:
+  case vmIntrinsics::_compareAndSetPrimitiveBitsMO:
+  case vmIntrinsics::_compareAndSetReferenceMO:
+    // all platforms must support at least T_OBJECT, T_INT, T_LONG
     break;
-  case vmIntrinsics::_getAndAddInt:
-    if (!VM_Version::supports_atomic_getadd4()) return false;
-    break;
-  case vmIntrinsics::_getAndAddLong:
-    if (!VM_Version::supports_atomic_getadd8()) return false;
-    break;
-  case vmIntrinsics::_getAndSetInt:
-    if (!VM_Version::supports_atomic_getset4()) return false;
-    break;
-  case vmIntrinsics::_getAndSetLong:
-    if (!VM_Version::supports_atomic_getset8()) return false;
+  case vmIntrinsics::_getAndOperatePrimitiveBitsMO:
+    if (!(VM_Version::supports_atomic_getadd4() ||
+          VM_Version::supports_atomic_getadd8() ||
+          VM_Version::supports_atomic_getset4() ||
+          VM_Version::supports_atomic_getset8())) {
+      // if any of the hardware ops are present, try the expansion
+      return false;
+    }
     break;
-  case vmIntrinsics::_getAndSetReference:
+  case vmIntrinsics::_getAndSetReferenceMO:
 #ifdef _LP64
     if (!UseCompressedOops && !VM_Version::supports_atomic_getset8()) return false;
     if (UseCompressedOops && !VM_Version::supports_atomic_getset4()) return false;
@@ -176,50 +175,10 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_dpow:
   case vmIntrinsics::_fmaD:
   case vmIntrinsics::_fmaF:
-  case vmIntrinsics::_getReference:
-  case vmIntrinsics::_getBoolean:
-  case vmIntrinsics::_getByte:
-  case vmIntrinsics::_getShort:
-  case vmIntrinsics::_getChar:
-  case vmIntrinsics::_getInt:
-  case vmIntrinsics::_getLong:
-  case vmIntrinsics::_getFloat:
-  case vmIntrinsics::_getDouble:
-  case vmIntrinsics::_putReference:
-  case vmIntrinsics::_putBoolean:
-  case vmIntrinsics::_putByte:
-  case vmIntrinsics::_putShort:
-  case vmIntrinsics::_putChar:
-  case vmIntrinsics::_putInt:
-  case vmIntrinsics::_putLong:
-  case vmIntrinsics::_putFloat:
-  case vmIntrinsics::_putDouble:
-  case vmIntrinsics::_getReferenceVolatile:
-  case vmIntrinsics::_getBooleanVolatile:
-  case vmIntrinsics::_getByteVolatile:
-  case vmIntrinsics::_getShortVolatile:
-  case vmIntrinsics::_getCharVolatile:
-  case vmIntrinsics::_getIntVolatile:
-  case vmIntrinsics::_getLongVolatile:
-  case vmIntrinsics::_getFloatVolatile:
-  case vmIntrinsics::_getDoubleVolatile:
-  case vmIntrinsics::_putReferenceVolatile:
-  case vmIntrinsics::_putBooleanVolatile:
-  case vmIntrinsics::_putByteVolatile:
-  case vmIntrinsics::_putShortVolatile:
-  case vmIntrinsics::_putCharVolatile:
-  case vmIntrinsics::_putIntVolatile:
-  case vmIntrinsics::_putLongVolatile:
-  case vmIntrinsics::_putFloatVolatile:
-  case vmIntrinsics::_putDoubleVolatile:
-  case vmIntrinsics::_getShortUnaligned:
-  case vmIntrinsics::_getCharUnaligned:
-  case vmIntrinsics::_getIntUnaligned:
-  case vmIntrinsics::_getLongUnaligned:
-  case vmIntrinsics::_putShortUnaligned:
-  case vmIntrinsics::_putCharUnaligned:
-  case vmIntrinsics::_putIntUnaligned:
-  case vmIntrinsics::_putLongUnaligned:
+  case vmIntrinsics::_getPrimitiveBitsMO:
+  case vmIntrinsics::_putPrimitiveBitsMO:
+  case vmIntrinsics::_getReferenceMO:
+  case vmIntrinsics::_putReferenceMO:
   case vmIntrinsics::_Preconditions_checkIndex:
   case vmIntrinsics::_Preconditions_checkLongIndex:
   case vmIntrinsics::_updateCRC32:
@@ -230,8 +189,6 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_updateDirectByteBufferCRC32C:
 #endif
   case vmIntrinsics::_vectorizedMismatch:
-  case vmIntrinsics::_compareAndSetInt:
-  case vmIntrinsics::_compareAndSetReference:
   case vmIntrinsics::_getCharStringU:
   case vmIntrinsics::_putCharStringU:
 #ifdef JFR_HAVE_INTRINSICS

src/hotspot/share/c1/c1_GraphBuilder.hpp

@@ -351,6 +351,7 @@ class GraphBuilder {
   // inliners
   bool try_inline(           ciMethod* callee, bool holder_known, bool ignore_return, Bytecodes::Code bc = Bytecodes::_illegal, Value receiver = nullptr);
   bool try_inline_intrinsics(ciMethod* callee, bool ignore_return = false);
+  bool try_inline_polymorphic_intrinsic(ciMethod* callee, bool ignore_return = false);
   bool try_inline_full(      ciMethod* callee, bool holder_known, bool ignore_return, Bytecodes::Code bc = Bytecodes::_illegal, Value receiver = nullptr);
   bool try_inline_jsr(int jsr_dest_bci);
 
@@ -373,10 +374,15 @@ class GraphBuilder {
   void pop_scope();
   void pop_scope_for_jsr();
 
-  void append_unsafe_get(ciMethod* callee, BasicType t, bool is_volatile);
-  void append_unsafe_put(ciMethod* callee, BasicType t, bool is_volatile);
-  void append_unsafe_CAS(ciMethod* callee);
-  void append_unsafe_get_and_set(ciMethod* callee, bool is_add);
+  void append_unsafe_get(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size);
+  void append_unsafe_put(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size);
+  void append_unsafe_CAS(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, int prefix_size);
+  void append_unsafe_get_and_set(ciMethod* callee, vmIntrinsics::MemoryOrder mo, BasicType t, vmIntrinsics::BitsOperation op, int prefix_size);
+  Value adjust_unsafe_container(Value x, BasicType from, BasicType to);
+  Value adjust_unsafe_offset(Value offset);
+  Bytecodes::Code  match_conversion(     Value x, Value* arg);
+  vmIntrinsics::ID match_unary_intrinsic(Value x, Value* arg);
+
   void append_char_access(ciMethod* callee, bool is_store);
   void append_alloc_array_copy(ciMethod* callee);
 

src/hotspot/share/c1/c1_Instruction.cpp

@@ -543,6 +543,42 @@ void BlockBegin::set_end(BlockEnd* new_end) { // Assumes that no predecessor of
 }
 
 
+Intrinsic*
+Intrinsic::with_polymorphic_prefix(vmIntrinsics::MemoryOrder memory_order,
+                                   BasicType basic_type,
+                                   vmIntrinsics::BitsOperation bits_op) {
+  vmIntrinsics::PolymorphicPrefix pfx = vmIntrinsics::polymorphic_prefix(id());
+  switch (pfx) {
+  case vmIntrinsics::PP_MO:
+    assert(memory_order != vmIntrinsics::MO_NONE, "");
+    assert(basic_type == T_OBJECT || basic_type == T_ILLEGAL, "");
+    assert(bits_op == vmIntrinsics::OP_NONE, "");
+    break;
+  case vmIntrinsics::PP_MO_BT:
+    assert(memory_order != vmIntrinsics::MO_NONE, "");
+    assert(is_java_primitive(basic_type), "");
+    assert(bits_op == vmIntrinsics::OP_NONE, "");
+    break;
+  case vmIntrinsics::PP_MO_BT_OP:
+    assert(memory_order != vmIntrinsics::MO_NONE, "");
+    assert(is_java_primitive(basic_type), "");
+    assert(bits_op != vmIntrinsics::OP_NONE, "");
+    break;
+  default:
+    ShouldNotReachHere();
+  }
+  // make sure previous settings are empty
+  assert(_basic_type == T_ILLEGAL, "");
+  assert(_memory_order == vmIntrinsics::MO_NONE, "");
+  assert(_bits_op == vmIntrinsics::OP_NONE, "");
+  // install the settings; some might still be empty
+  _memory_order = memory_order;
+  _basic_type = basic_type;
+  _bits_op = bits_op;
+  return this;
+}
+
+
 void BlockBegin::disconnect_edge(BlockBegin* from, BlockBegin* to) {
   // disconnect any edges between from and to
 #ifndef PRODUCT

src/hotspot/share/c1/c1_Instruction.hpp

@@ -1501,14 +1501,18 @@ LEAF(MonitorExit, AccessMonitor)
   }
 };
 
-
 LEAF(Intrinsic, StateSplit)
  private:
   vmIntrinsics::ID _id;
   ArgsNonNullState _nonnull_state;
   Values*          _args;
   Value            _recv;
 
+  // polymorphic prefix arguments, if applicable, set neutrally if not:
+  vmIntrinsics::MemoryOrder   _memory_order;
+  BasicType                   _basic_type;
+  vmIntrinsics::BitsOperation _bits_op;
+
  public:
   // preserves_state can be set to true for Intrinsics
   // which are guaranteed to preserve register state across any slow
@@ -1528,6 +1532,9 @@ LEAF(Intrinsic, StateSplit)
   , _id(id)
   , _args(args)
   , _recv(nullptr)
+  , _memory_order(vmIntrinsics::MO_NONE)
+  , _basic_type(T_ILLEGAL)
+  , _bits_op(vmIntrinsics::OP_NONE)
   {
     assert(args != nullptr, "args must exist");
     ASSERT_VALUES
@@ -1567,6 +1574,25 @@ LEAF(Intrinsic, StateSplit)
     StateSplit::input_values_do(f);
     for (int i = 0; i < _args->length(); i++) f->visit(_args->adr_at(i));
   }
+
+  // prefix arguments
+  vmIntrinsics::MemoryOrder memory_order() const {
+    assert(_memory_order != vmIntrinsics::MO_NONE, "must be present");
+    return _memory_order;
+  }
+  BasicType basic_type() const {
+    assert(_basic_type != T_ILLEGAL, "must be present");
+    return _basic_type;
+  }
+  vmIntrinsics::BitsOperation bits_op() const {
+    assert(_bits_op != vmIntrinsics::OP_NONE, "must be present");
+    return _bits_op;
+  }
+  // one-time setup after creation, required for polymorphic intrinsics
+  Intrinsic* with_polymorphic_prefix(
+      vmIntrinsics::MemoryOrder memory_order,
+      BasicType basic_type = T_OBJECT,
+      vmIntrinsics::BitsOperation bits_op = vmIntrinsics::OP_NONE);
 };
 
 
@@ -2139,14 +2165,15 @@ BASE(UnsafeOp, Instruction)
  private:
   Value _object;                                 // Object to be fetched from or mutated
   Value _offset;                                 // Offset within object
-  bool  _is_volatile;                            // true if volatile - dl/JSR166
+  vmIntrinsics::MemoryOrder _memory_order;       // MO_PLAIN, MO_VOLATILE - dl/JSR166
   BasicType _basic_type;                         // ValueType can not express byte-sized integers
 
  protected:
   // creation
-  UnsafeOp(BasicType basic_type, Value object, Value offset, bool is_put, bool is_volatile)
+  UnsafeOp(BasicType basic_type, Value object, Value offset, bool is_put,
+           vmIntrinsics::MemoryOrder memory_order)
     : Instruction(is_put ? voidType : as_ValueType(basic_type)),
-    _object(object), _offset(offset), _is_volatile(is_volatile), _basic_type(basic_type)
+    _object(object), _offset(offset), _memory_order(memory_order), _basic_type(basic_type)
   {
     //Note:  Unsafe ops are not not guaranteed to throw NPE.
     // Convservatively, Unsafe operations must be pinned though we could be
@@ -2159,7 +2186,7 @@ BASE(UnsafeOp, Instruction)
   BasicType basic_type()                         { return _basic_type; }
   Value object()                                 { return _object; }
   Value offset()                                 { return _offset; }
-  bool  is_volatile()                            { return _is_volatile; }
+  vmIntrinsics::MemoryOrder memory_order()       { return _memory_order; }
 
   // generic
   virtual void input_values_do(ValueVisitor* f)   { f->visit(&_object);
@@ -2170,14 +2197,14 @@ LEAF(UnsafeGet, UnsafeOp)
  private:
   bool _is_raw;
  public:
-  UnsafeGet(BasicType basic_type, Value object, Value offset, bool is_volatile)
-  : UnsafeOp(basic_type, object, offset, false, is_volatile)
+  UnsafeGet(BasicType basic_type, Value object, Value offset, vmIntrinsics::MemoryOrder memory_order)
+  : UnsafeOp(basic_type, object, offset, false, memory_order)
   {
     ASSERT_VALUES
     _is_raw = false;
   }
-  UnsafeGet(BasicType basic_type, Value object, Value offset, bool is_volatile, bool is_raw)
-  : UnsafeOp(basic_type, object, offset, false, is_volatile), _is_raw(is_raw)
+  UnsafeGet(BasicType basic_type, Value object, Value offset, vmIntrinsics::MemoryOrder memory_order, bool is_raw)
+  : UnsafeOp(basic_type, object, offset, false, memory_order), _is_raw(is_raw)
   {
     ASSERT_VALUES
   }
@@ -2191,8 +2218,8 @@ LEAF(UnsafePut, UnsafeOp)
  private:
   Value _value;                                  // Value to be stored
  public:
-  UnsafePut(BasicType basic_type, Value object, Value offset, Value value, bool is_volatile)
-  : UnsafeOp(basic_type, object, offset, true, is_volatile)
+  UnsafePut(BasicType basic_type, Value object, Value offset, Value value, vmIntrinsics::MemoryOrder memory_order)
+  : UnsafeOp(basic_type, object, offset, true, memory_order)
     , _value(value)
   {
     ASSERT_VALUES
@@ -2209,18 +2236,19 @@ LEAF(UnsafePut, UnsafeOp)
 LEAF(UnsafeGetAndSet, UnsafeOp)
  private:
   Value _value;                                  // Value to be stored
-  bool  _is_add;
+  vmIntrinsics::BitsOperation _bits_op;          // operation: swap, add
  public:
-  UnsafeGetAndSet(BasicType basic_type, Value object, Value offset, Value value, bool is_add)
-  : UnsafeOp(basic_type, object, offset, false, false)
+  UnsafeGetAndSet(BasicType basic_type, Value object, Value offset, Value value,
+                  vmIntrinsics::MemoryOrder memory_order, vmIntrinsics::BitsOperation bits_op)
+  : UnsafeOp(basic_type, object, offset, false, memory_order)
     , _value(value)
-    , _is_add(is_add)
+    , _bits_op(bits_op)
   {
     ASSERT_VALUES
   }
 
   // accessors
-  bool is_add() const                            { return _is_add; }
+  vmIntrinsics::BitsOperation bits_op() const    { return _bits_op; }
   Value value()                                  { return _value; }
 
   // generic

src/hotspot/share/c1/c1_InstructionPrinter.cpp

@@ -792,7 +792,11 @@ void InstructionPrinter::do_UnsafePut(UnsafePut* x) {
 }
 
 void InstructionPrinter::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {
-  print_unsafe_op(x, x->is_add()?"UnsafeGetAndSet (add)":"UnsafeGetAndSet");
+  switch (x->bits_op()) {
+  default:  assert(false, ""); //and fall through:
+  case vmIntrinsics::OP_SWAP:  print_unsafe_op(x, "UnsafeGetAndSet"); break;
+  case vmIntrinsics::OP_ADD:   print_unsafe_op(x, "UnsafeGetAndSet (add)"); break;
+  }
   output()->print(", value ");
   print_value(x->value());
   output()->put(')');