Replace llvm.memcpy et al's i1 isVolatile with i8 VolFlags

clang/lib/CodeGen/CGBuilder.h

@@ -298,16 +298,16 @@ class CGBuilderTy : public CGBuilderBaseTy {
 
   using CGBuilderBaseTy::CreateMemCpy;
   llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size,
-                               bool IsVolatile = false) {
+                               llvm::MemTransferVolatility VolFlags = {}) {
     return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
                         Src.getPointer(), Src.getAlignment().getAsAlign(), Size,
-                        IsVolatile);
+                        VolFlags);
   }
   llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size,
-                               bool IsVolatile = false) {
+                               llvm::MemTransferVolatility VolFlags = {}) {
     return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
                         Src.getPointer(), Src.getAlignment().getAsAlign(), Size,
-                        IsVolatile);
+                        VolFlags);
   }
 
   using CGBuilderBaseTy::CreateMemCpyInline;
@@ -319,10 +319,10 @@ class CGBuilderTy : public CGBuilderBaseTy {
 
   using CGBuilderBaseTy::CreateMemMove;
   llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size,
-                                bool IsVolatile = false) {
+                                llvm::MemTransferVolatility VolFlags = {}) {
     return CreateMemMove(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
                          Src.getPointer(), Src.getAlignment().getAsAlign(),
-                         Size, IsVolatile);
+                         Size, VolFlags);
   }
 
   using CGBuilderBaseTy::CreateMemSet;

clang/lib/CodeGen/CGBuiltin.cpp

@@ -3678,7 +3678,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                         E->getArg(0)->getExprLoc(), FD, 0);
     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
                         E->getArg(1)->getExprLoc(), FD, 1);
-    Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+    Builder.CreateMemCpy(Dest, Src, SizeVal);
     if (BuiltinID == Builtin::BImempcpy ||
         BuiltinID == Builtin::BI__builtin_mempcpy)
       return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(),
@@ -3717,7 +3717,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+    Builder.CreateMemCpy(Dest, Src, SizeVal);
     return RValue::get(Dest.getPointer());
   }
 
@@ -3743,7 +3743,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    Builder.CreateMemMove(Dest, Src, SizeVal);
     return RValue::get(Dest.getPointer());
   }
 
@@ -3756,7 +3756,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                         E->getArg(0)->getExprLoc(), FD, 0);
     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
                         E->getArg(1)->getExprLoc(), FD, 1);
-    Builder.CreateMemMove(Dest, Src, SizeVal, false);
+    Builder.CreateMemMove(Dest, Src, SizeVal);
     return RValue::get(Dest.getPointer());
   }
   case Builtin::BImemset:

clang/lib/CodeGen/CGDecl.cpp

@@ -1274,11 +1274,11 @@ static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D,
   }
 
   // Copy from a global.
-  auto *I =
-      Builder.CreateMemCpy(Loc,
-                           createUnnamedGlobalForMemcpyFrom(
-                               CGM, D, Builder, constant, Loc.getAlignment()),
-                           SizeVal, isVolatile);
+  auto *I = Builder.CreateMemCpy(
+      Loc,
+      createUnnamedGlobalForMemcpyFrom(CGM, D, Builder, constant,
+                                       Loc.getAlignment()),
+      SizeVal, llvm::MemTransferVolatility().Dst(isVolatile));
   if (IsAutoInit)
     I->addAnnotationMetadata("auto-init");
 }
@@ -1842,11 +1842,11 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type,
     llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur");
     Cur->addIncoming(Begin.getPointer(), OriginBB);
     CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize);
-    auto *I =
-        Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign),
-                             createUnnamedGlobalForMemcpyFrom(
-                                 CGM, D, Builder, Constant, ConstantAlign),
-                             BaseSizeInChars, isVolatile);
+    auto *I = Builder.CreateMemCpy(
+        Address(Cur, Int8Ty, CurAlign),
+        createUnnamedGlobalForMemcpyFrom(CGM, D, Builder, Constant,
+                                         ConstantAlign),
+        BaseSizeInChars, llvm::MemTransferVolatility().Dst(isVolatile));
     I->addAnnotationMetadata("auto-init");
     llvm::Value *Next =
         Builder.CreateInBoundsGEP(Int8Ty, Cur, BaseSizeInChars, "vla.next");

clang/lib/CodeGen/CGExprAgg.cpp

@@ -376,6 +376,9 @@ void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src,
   AggValueSlot srcAgg = AggValueSlot::forLValue(
       src, CGF, AggValueSlot::IsDestructed, needsGC(type),
       AggValueSlot::IsAliased, AggValueSlot::MayOverlap);
+  // A non-volatile src might have a volatile member.
+  if (!srcAgg.isVolatile() && CGF.hasVolatileMember(type))
+    srcAgg.setVolatile(true);
   EmitCopy(type, Dest, srcAgg);
 }
 
@@ -401,7 +404,7 @@ void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
   LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type);
   LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type);
   CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(),
-                        dest.isVolatile() || src.isVolatile());
+                        dest.isVolatile(), src.isVolatile());
 }
 
 /// Emit the initializer for a std::initializer_list initialized with a
@@ -2101,7 +2104,7 @@ AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
 
 void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
                                         AggValueSlot::Overlap_t MayOverlap,
-                                        bool isVolatile) {
+                                        bool isVolDst, bool isVolSrc) {
   assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
 
   Address DestPtr = Dest.getAddress(*this);
@@ -2183,8 +2186,8 @@ void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
   //   a = b;
   // }
   //
-  // we need to use a different call here.  We use isVolatile to indicate when
-  // either the source or the destination is volatile.
+  // we need to use a different call here.  We use isVolDst and isVolSrc to
+  // indicate when the destination and/or source is volatile.
 
   DestPtr = DestPtr.withElementType(Int8Ty);
   SrcPtr = SrcPtr.withElementType(Int8Ty);
@@ -2210,7 +2213,9 @@ void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
     }
   }
 
-  auto Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile);
+  auto Inst = Builder.CreateMemCpy(
+      DestPtr, SrcPtr, SizeVal,
+      llvm::MemTransferVolatility().Dst(isVolDst).Src(isVolSrc));
 
   // Determine the metadata to describe the position of any padding in this
   // memcpy, as well as the TBAA tags for the members of the struct, in case

clang/lib/CodeGen/CGNonTrivialStruct.cpp

@@ -526,7 +526,7 @@ struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
           llvm::ConstantInt::get(this->CGF->SizeTy, Size.getQuantity());
       DstAddr = DstAddr.withElementType(this->CGF->Int8Ty);
       SrcAddr = SrcAddr.withElementType(this->CGF->Int8Ty);
-      this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal, false);
+      this->CGF->Builder.CreateMemCpy(DstAddr, SrcAddr, SizeVal);
     } else {
       llvm::Type *Ty = llvm::Type::getIntNTy(
           this->CGF->getLLVMContext(),

clang/lib/CodeGen/CodeGenFunction.cpp

@@ -1978,8 +1978,8 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
     dest.getAlignment().alignmentOfArrayElement(baseSize);
 
   // memcpy the individual element bit-pattern.
-  Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src, baseSizeInChars,
-                       /*volatile*/ false);
+  Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src,
+                       baseSizeInChars);
 
   // Go to the next element.
   llvm::Value *next =
@@ -2054,14 +2054,14 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
     if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
 
     // Get and call the appropriate llvm.memcpy overload.
-    Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
+    Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal);
     return;
   }
 
   // Otherwise, just memset the whole thing to zero.  This is legal
   // because in LLVM, all default initializers (other than the ones we just
   // handled above) are guaranteed to have a bit pattern of all zeros.
-  Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
+  Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal);
 }
 
 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {

clang/lib/CodeGen/CodeGenFunction.h

@@ -2717,16 +2717,22 @@ class CodeGenFunction : public CodeGenTypeCache {
     EmitAggregateCopy(Dest, Src, Src.getType(), MayOverlap);
   }
 
+  void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
+                         AggValueSlot::Overlap_t MayOverlap,
+                         bool isVolatile = false) {
+    EmitAggregateCopy(Dest, Src, EltTy, MayOverlap, isVolatile, isVolatile);
+  }
+
   /// EmitAggregateCopy - Emit an aggregate copy.
   ///
-  /// \param isVolatile \c true iff either the source or the destination is
-  ///        volatile.
+  /// \param isVolDst \c true iff either the destination is volatile.
+  /// \param isVolSrc \c true iff either the source is volatile.
   /// \param MayOverlap Whether the tail padding of the destination might be
   ///        occupied by some other object. More efficient code can often be
   ///        generated if not.
   void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
-                         AggValueSlot::Overlap_t MayOverlap,
-                         bool isVolatile = false);
+                         AggValueSlot::Overlap_t MayOverlap, bool isVolDst,
+                         bool isVolSrc);
 
   /// GetAddrOfLocalVar - Return the address of a local variable.
   Address GetAddrOfLocalVar(const VarDecl *VD) {

clang/lib/CodeGen/Targets/X86.cpp

@@ -3115,7 +3115,7 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
     // register save area.
     if (TyAlign.getQuantity() > 8) {
       Address Tmp = CGF.CreateMemTemp(Ty);
-      CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
+      CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize);
       RegAddr = Tmp;
     }
 

clang/test/C/C2x/n2900_n3011_2.c

@@ -101,8 +101,8 @@ void test_nested_structs() {
   // CHECK-NEXT: %[[T1:.+]] = alloca %struct.T
   // CHECK-NEXT: %[[T2:.+]] = alloca %struct.T
   // CHECK-NEXT: %[[T3:.+]] = alloca %struct.T
-  // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr {{.*}} %[[T1]], ptr {{.*}} @[[CONST_T1]], i64 12, i1 false)
-  // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr {{.*}} %[[T2]], ptr {{.*}} @[[CONST_T2]], i64 12, i1 false)
+  // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr {{.*}} %[[T1]], ptr {{.*}} @[[CONST_T1]], i64 12, i8 0)
+  // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr {{.*}} %[[T2]], ptr {{.*}} @[[CONST_T2]], i64 12, i8 0)
   // CHECK-NEXT: call void @llvm.memset.p0.i64(ptr {{.*}} %[[T3]], i8 0, i64 12, i1 false)
 }
 

clang/test/CodeGen/2007-11-07-CopyAggregateAlign.c

@@ -2,6 +2,6 @@
 struct A { char s, t, u, v; short a; };
 // CHECK: %a = alloca %struct.A, align 2
 // CHECK: %b = alloca %struct.A, align 2
-// CHECK: call void @llvm.memcpy.p0.p0.{{.*}} align 2 {{.*}} align 2 {{.*}}, i1 false)
+// CHECK: call void @llvm.memcpy.p0.p0.{{.*}} align 2 {{.*}} align 2 {{.*}}, i8 0)
 
 void q(void) { struct A a, b; a = b; }

clang/test/CodeGen/CSKY/csky-abi.c

@@ -293,15 +293,15 @@ double f_va_3(char *fmt, ...) {
 // CHECK-NEXT:    [[ARGP_CUR4:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT5:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR4]], i32 4
 // CHECK-NEXT:    store ptr [[ARGP_NEXT5]], ptr [[VA]], align 4
-// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[TS]], ptr align 4 [[ARGP_CUR4]], i32 4, i1 false)
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[TS]], ptr align 4 [[ARGP_CUR4]], i32 4, i8 0)
 // CHECK-NEXT:    [[ARGP_CUR6:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT7:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR6]], i32 8
 // CHECK-NEXT:    store ptr [[ARGP_NEXT7]], ptr [[VA]], align 4
-// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[SS]], ptr align 4 [[ARGP_CUR6]], i32 8, i1 false)
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[SS]], ptr align 4 [[ARGP_CUR6]], i32 8, i8 0)
 // CHECK-NEXT:    [[ARGP_CUR8:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT9:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR8]], i32 16
 // CHECK-NEXT:    store ptr [[ARGP_NEXT9]], ptr [[VA]], align 4
-// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[LS]], ptr align 4 [[ARGP_CUR8]], i32 16, i1 false)
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[LS]], ptr align 4 [[ARGP_CUR8]], i32 16, i8 0)
 // CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
 int f_va_4(char *fmt, ...) {
   __builtin_va_list va;

clang/test/CodeGen/PowerPC/aix-alignment.c

@@ -24,8 +24,8 @@ double retDouble(double x) { return x; }
 
 // AIX32: define void @bar(ptr noalias sret(%struct.StructDouble) align 4 %agg.result, ptr noundef byval(%struct.StructDouble) align 4 %x)
 // AIX64: define void @bar(ptr noalias sret(%struct.StructDouble) align 4 %agg.result, ptr noundef byval(%struct.StructDouble) align 8 %x)
-// AIX32:   call void @llvm.memcpy.p0.p0.i32(ptr align 4 %agg.result, ptr align 4 %x, i32 16, i1 false)
-// AIX64:   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %agg.result, ptr align 8 %x, i64 16, i1 false)
+// AIX32:   call void @llvm.memcpy.p0.p0.i32(ptr align 4 %agg.result, ptr align 4 %x, i32 16, i8 0)
+// AIX64:   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %agg.result, ptr align 8 %x, i64 16, i8 0)
 StructDouble bar(StructDouble x) { return x; }
 
 // AIX:   define void @foo(ptr noundef %out, ptr noundef %in)

clang/test/CodeGen/PowerPC/aix-vaargs.c

@@ -45,8 +45,8 @@ void testva (int n, ...) {
 // AIX64-NEXT:  store ptr %argp.next, ptr %ap, align 8
 
 
-// AIX32-NEXT:  call void @llvm.memcpy.p0.p0.i32(ptr align 8 %t, ptr align 4 %argp.cur, i32 16, i1 false)
-// AIX64-NEXT:  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %t, ptr align 8 %argp.cur, i64 16, i1 false)
+// AIX32-NEXT:  call void @llvm.memcpy.p0.p0.i32(ptr align 8 %t, ptr align 4 %argp.cur, i32 16, i8 0)
+// AIX64-NEXT:  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %t, ptr align 8 %argp.cur, i64 16, i8 0)
 
 // CHECK-NEXT:  call void @llvm.va_copy(ptr %ap2, ptr %ap)
 
@@ -68,8 +68,8 @@ void testva (int n, ...) {
 
 // CHECK: declare void @llvm.va_start(ptr)
 
-// AIX32: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
-// AIX64: declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+// AIX32: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i8 immarg)
+// AIX64: declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i8 immarg)
 
 // CHECK: declare void @llvm.va_copy(ptr, ptr)
 // CHECK: declare void @llvm.va_end(ptr)

clang/test/CodeGen/PowerPC/ppc-varargs-struct.c

@@ -46,7 +46,7 @@ void testva (int n, ...)
 // CHECK-PPC:[[CONT]]
 // CHECK-PPC-NEXT:  [[VAARG_ADDR:%[a-z.0-9]+]] = phi ptr [ [[RAW_REGADDR]], %[[USING_REGS]] ], [ %argp.cur.aligned, %[[USING_OVERFLOW]] ]
 // CHECK-PPC-NEXT:  [[AGGR:%[a-z0-9]+]] = load ptr, ptr [[VAARG_ADDR]]
-// CHECK-PPC-NEXT:  call void @llvm.memcpy.p0.p0.i32(ptr align 8 %t, ptr align 8 [[AGGR]], i32 16, i1 false)
+// CHECK-PPC-NEXT:  call void @llvm.memcpy.p0.p0.i32(ptr align 8 %t, ptr align 8 [[AGGR]], i32 16, i8 0)
 
   int v = va_arg (ap, int);