[vm] Use dmb ishst instead of dmb ish for store-store fence.

TEST=ci
Change-Id: Id7fb33bb4d9626ad420fd5aac2df86b47bd5c7bf
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/459662
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>

Author: rmacnak-google

runtime/platform/atomic.h

@@ -153,6 +153,28 @@ static inline T LoadRelaxed(const T* ptr) {
       std::memory_order_relaxed);
 }
 
+inline void StoreStoreFence() {
+#if defined(HOST_ARCH_ARM) && defined(_MSC_VER)
+  __dmb(_ARM_BARRIER_ISHST);
+#elif defined(HOST_ARCH_ARM64) && defined(_MSC_VER)
+  __dmb(_ARM64_BARRIER_ISHST);
+#elif defined(HOST_ARCH_ARM64) && defined(__GNUC__)
+  __asm__ __volatile__("dmb ishst" : : : "memory");
+#elif defined(HOST_ARCH_ARM) && defined(__GNUC__)
+  __asm__ __volatile__("dmb ishst" : : : "memory");
+#elif defined(HOST_ARCH_RISCV32) && defined(__GNUC__)
+  __asm__ __volatile__("fence w,w" : : : "memory");
+#elif defined(HOST_ARCH_RISCV64) && defined(__GNUC__)
+  __asm__ __volatile__("fence w,w" : : : "memory");
+#else
+  // GCC warns that TSAN doesn't understand thread fences.
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic ignored "-Wtsan"
+#endif
+  std::atomic_thread_fence(std::memory_order_release);
+#endif
+}
+
 }  // namespace dart
 
 #endif  // RUNTIME_PLATFORM_ATOMIC_H_

runtime/vm/compiler/assembler/assembler_arm.cc

@@ -527,11 +527,6 @@ void Assembler::strex(Register rd, Register rt, Register rn, Condition cond) {
   Emit(encoding);
 }
 
-void Assembler::dmb() {
-  // Emit a `dmb ish` instruction.
-  Emit(kDataMemoryBarrier);
-}
-
 static int32_t BitFieldExtractEncoding(bool sign_extend,
                                        Register rd,
                                        Register rn,

runtime/vm/compiler/assembler/assembler_arm.h

@@ -438,12 +438,12 @@ class Assembler : public AssemblerBase {
                    const Address& address,
                    OperandSize size = kFourBytes) override {
     Load(dst, address, size);
-    dmb();
+    dmb_ish();
   }
   void StoreRelease(Register src,
                     const Address& address,
                     OperandSize size = kFourBytes) override {
-    dmb();
+    dmb_ish();
     Store(src, address, size);
   }
 
@@ -469,6 +469,8 @@ class Assembler : public AssemblerBase {
   // Debugging and bringup support.
   void Breakpoint() override { bkpt(0); }
 
+  void StoreStoreFence() override { dmb_ishst(); }
+
   // Data-processing instructions.
   void and_(Register rd, Register rn, Operand o, Condition cond = AL);
   void ands(Register rd, Register rn, Operand o, Condition cond = AL);
@@ -601,7 +603,8 @@ class Assembler : public AssemblerBase {
   void ldrex(Register rd, Register rn, Condition cond = AL);
   void strex(Register rd, Register rt, Register rn, Condition cond = AL);
 
-  void dmb();
+  void dmb_ish() { Emit(kDMB_ISH); }
+  void dmb_ishst() { Emit(kDMB_ISHST); }
 
   // Media instructions.
   void sbfx(Register rd,
@@ -1041,13 +1044,13 @@ class Assembler : public AssemblerBase {
   void InitializeHeader(Register tags, Register object) {
     str(tags, FieldAddress(object, target::Object::tags_offset()));
 #if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
-    dmb();
+    StoreStoreFence();
 #endif
   }
   void InitializeHeaderUntagged(Register tags, Register object) {
     str(tags, Address(object, target::Object::tags_offset()));
 #if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
-    dmb();
+    StoreStoreFence();
 #endif
   }
 

runtime/vm/compiler/assembler/assembler_arm64.h

@@ -564,6 +564,8 @@ class Assembler : public AssemblerBase {
   // Debugging and bringup support.
   void Breakpoint() override { brk(0); }
 
+  void StoreStoreFence() override { dmb_ishst(); }
+
   void SetPrologueOffset() {
     if (prologue_offset_ == -1) {
       prologue_offset_ = CodeSize();
@@ -1242,7 +1244,8 @@ class Assembler : public AssemblerBase {
   // Breakpoint.
   void brk(uint16_t imm) { EmitExceptionGenOp(BRK, imm); }
 
-  void dmb() { Emit(kDataMemoryBarrier); }
+  void dmb_ish() { Emit(kDMB_ISH); }
+  void dmb_ishst() { Emit(kDMB_ISHST); }
 
   // Double floating point.
   bool fmovdi(VRegister vd, double immd) {
@@ -1963,13 +1966,13 @@ class Assembler : public AssemblerBase {
   void InitializeHeader(Register header, Register object) {
     str(header, FieldAddress(object, target::Object::tags_offset()));
 #if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
-    dmb();
+    StoreStoreFence();
 #endif
   }
   void InitializeHeaderUntagged(Register header, Register object) {
     str(header, Address(object, target::Object::tags_offset()));
 #if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
-    dmb();
+    StoreStoreFence();
 #endif
   }
 

runtime/vm/compiler/assembler/assembler_base.h

@@ -630,6 +630,8 @@ class AssemblerBase : public StackResource {
 
   virtual void Breakpoint() = 0;
 
+  virtual void StoreStoreFence() = 0;
+
   virtual void SmiTag(Register r) = 0;
 
   virtual void Bind(Label* label) = 0;

runtime/vm/compiler/assembler/assembler_ia32.h

@@ -1199,6 +1199,8 @@ class Assembler : public AssemblerBase {
   // Debugging and bringup support.
   void Breakpoint() override { int3(); }
 
+  void StoreStoreFence() override {}
+
   // Check if the given value is an integer value that can be directly
   // embedded into the code without additional XORing with jit_cookie.
   // We consider 16-bit integers, powers of two and corresponding masks

runtime/vm/compiler/assembler/assembler_riscv.h

@@ -1027,6 +1027,8 @@ class Assembler : public MicroAssembler {
   // Debugging and bringup support.
   void Breakpoint() override { trap(); }
 
+  void StoreStoreFence() override { fence(kWrite, kWrite); }
+
   void SetPrologueOffset() {
     if (prologue_offset_ == -1) {
       prologue_offset_ = CodeSize();

runtime/vm/compiler/assembler/assembler_x64.h

@@ -1345,6 +1345,8 @@ class Assembler : public AssemblerBase {
   // Debugging and bringup support.
   void Breakpoint() override { int3(); }
 
+  void StoreStoreFence() override {}
+
   static bool AddressCanHoldConstantIndex(const Object& constant,
                                           bool is_external,
                                           intptr_t cid,

runtime/vm/compiler/assembler/disassembler_arm.cc

@@ -1444,9 +1444,10 @@ void ARMDecoder::InstructionDecode(uword pc) {
   if (instr->ConditionField() == kSpecialCondition) {
     if (instr->InstructionBits() == static_cast<int32_t>(0xf57ff01f)) {
       Format(instr, "clrex");
-    } else if (instr->InstructionBits() ==
-               static_cast<int32_t>(kDataMemoryBarrier)) {
+    } else if (instr->InstructionBits() == static_cast<int32_t>(kDMB_ISH)) {
       Format(instr, "dmb ish");
+    } else if (instr->InstructionBits() == static_cast<int32_t>(kDMB_ISHST)) {
+      Format(instr, "dmb ishst");
     } else {
       if (instr->IsSIMDDataProcessing()) {
         DecodeSIMDDataProcessing(instr);

runtime/vm/compiler/assembler/disassembler_arm64.cc

@@ -994,10 +994,14 @@ void ARM64Decoder::DecodeSystem(Instr* instr) {
     return;
   }
 
-  if (instr->InstructionBits() == kDataMemoryBarrier) {
+  if (instr->InstructionBits() == kDMB_ISH) {
     Format(instr, "dmb ish");
     return;
   }
+  if (instr->InstructionBits() == kDMB_ISHST) {
+    Format(instr, "dmb ishst");
+    return;
+  }
 
   if ((instr->Bits(0, 8) == 0x1f) && (instr->Bits(12, 4) == 2) &&
       (instr->Bits(16, 3) == 3) && (instr->Bits(19, 2) == 0) &&