Add bypass for i1xN

 - Automatically destructure structs if the type doesn't match exactly
 - Start emitting hard errors on signature mismatches

Author: sayantn

compiler/rustc_codegen_llvm/src/abi.rs

@@ -330,7 +330,17 @@ impl<'ll> FunctionSignature<'ll> {
 pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
     fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>;
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> FunctionSignature<'ll>;
+    /// When `do_verify` is set, this function performs checks for the signature of LLVM intrinsics
+    /// and emits a fatal error if it doesn't match. These checks are important,but somewhat expensive
+    /// So they are only used at function definitions, not at callsites
+    fn llvm_type(
+        &self,
+        cx: &CodegenCx<'ll, 'tcx>,
+        name: &[u8],
+        do_verify: bool,
+    ) -> FunctionSignature<'ll>;
+    /// **If this function is an LLVM intrinsic** checks if the LLVM signature provided matches with this
+    fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_ty: &'ll Type) -> bool;
     fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv;
 
@@ -343,31 +353,25 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
     );
 
     /// Apply attributes to a function call.
-    fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value);
+    fn apply_attrs_callsite(
+        &self,
+        bx: &mut Builder<'_, 'll, 'tcx>,
+        callsite: &'ll Value,
+        llfn: &'ll Value,
+    );
 }
 
-/// checks that the Rust signature of a llvm intrinsic is correct
-fn verify_intrinsic_signature<'ll, 'tcx>(
-    cx: &CodegenCx<'ll, 'tcx>,
-    fn_ty: &'ll Type,
-    rust_return_ty: &'ll Type,
-    rust_argument_tys: &[&'ll Type],
-    rust_is_variadic: bool,
-) -> bool {
-    let llvm_return_ty = cx.get_return_type(fn_ty);
-    let llvm_argument_tys = cx.func_params_types(fn_ty);
-    let llvm_is_variadic = cx.func_is_variadic(fn_ty);
-
-    if rust_is_variadic != llvm_is_variadic || rust_argument_tys.len() != llvm_argument_tys.len() {
-        return false;
+impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
+    pub(crate) fn set_intrinsic_attributes(
+        &self,
+        intrinsic: llvm::Intrinsic,
+        llfn_or_callsite: &'ll Value,
+    ) {
+        unsafe {
+            llvm::LLVMRustSetIntrinsicAttributes(self.llcx(), llfn_or_callsite, intrinsic.id());
+        }
     }
 
-    iter::once((&rust_return_ty, &llvm_return_ty))
-        .chain(iter::zip(rust_argument_tys, &llvm_argument_tys))
-        .all(|(rust_ty, llvm_ty)| cx.equate_ty(rust_ty, llvm_ty))
-}
-
-impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
     pub(crate) fn equate_ty(&self, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool {
         if rust_ty == llvm_ty {
             return true;
@@ -395,15 +399,32 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
             }
             TypeKind::BFloat => rust_ty == self.type_i16(),
             TypeKind::Vector => {
-                let llvm_element_count = self.vector_length(llvm_ty);
+                let llvm_element_count = self.vector_length(llvm_ty) as u64;
                 let llvm_element_ty = self.element_type(llvm_ty);
 
                 if llvm_element_ty == self.type_bf16() {
-                    rust_ty == self.type_vector(self.type_i16(), llvm_element_count as u64)
+                    rust_ty == self.type_vector(self.type_i16(), llvm_element_count)
+                } else if llvm_element_ty == self.type_i1() {
+                    let int_width = cmp::max(llvm_element_count.next_power_of_two(), 8);
+                    rust_ty == self.type_ix(int_width)
                 } else {
                     false
                 }
             }
+            TypeKind::Struct if self.type_kind(rust_ty) == TypeKind::Struct => {
+                let rust_element_tys = self.struct_element_types(rust_ty);
+                let llvm_element_tys = self.struct_element_types(llvm_ty);
+
+                if rust_element_tys.len() != llvm_element_tys.len() {
+                    return false;
+                }
+
+                iter::zip(rust_element_tys, llvm_element_tys).all(
+                    |(rust_element_ty, llvm_element_ty)| {
+                        self.equate_ty(rust_element_ty, llvm_element_ty)
+                    },
+                )
+            }
             _ => false,
         }
     }
@@ -482,7 +503,30 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         llargument_tys
     }
 
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> FunctionSignature<'ll> {
+    fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_fn_ty: &'ll Type) -> bool {
+        let rust_return_ty = self.llvm_return_type(cx);
+        let rust_argument_tys = self.llvm_argument_types(cx);
+
+        let llvm_return_ty = cx.get_return_type(llvm_fn_ty);
+        let llvm_argument_tys = cx.func_params_types(llvm_fn_ty);
+        let llvm_is_variadic = cx.func_is_variadic(llvm_fn_ty);
+
+        if self.c_variadic != llvm_is_variadic || rust_argument_tys.len() != llvm_argument_tys.len()
+        {
+            return false;
+        }
+
+        iter::once((rust_return_ty, llvm_return_ty))
+            .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
+            .all(|(rust_ty, llvm_ty)| cx.equate_ty(rust_ty, llvm_ty))
+    }
+
+    fn llvm_type(
+        &self,
+        cx: &CodegenCx<'ll, 'tcx>,
+        name: &[u8],
+        do_verify: bool,
+    ) -> FunctionSignature<'ll> {
         let return_ty = self.llvm_return_type(cx);
         let argument_tys = self.llvm_argument_types(cx);
 
@@ -493,30 +537,15 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 if !intrinsic.is_overloaded() {
                     // FIXME: also do this for overloaded intrinsics
                     let llvm_fn_ty = cx.intrinsic_type(intrinsic, &[]);
-                    if verify_intrinsic_signature(
-                        cx,
-                        llvm_fn_ty,
-                        return_ty,
-                        &argument_tys,
-                        self.c_variadic,
-                    ) {
-                        return FunctionSignature::Intrinsic(intrinsic, llvm_fn_ty);
-                    } else {
-                        // FIXME: ideally we would like to hard error, but there are edge cases like `i1xN`
-                        // and differences between struct representations that we have to tackle first
-
-                        let rust_fn_ty = if self.c_variadic {
-                            cx.type_variadic_func(&argument_tys, return_ty)
-                        } else {
-                            cx.type_func(&argument_tys, return_ty)
-                        };
-                        cx.tcx.dcx().note(format!(
-                            "Intrinsic signature mismatch for `{}`: expected signature `{llvm_fn_ty:?}`, found signature `{rust_fn_ty:?}`",
-                            str::from_utf8(name).unwrap()
-                        ));
-                        // fallback to the rust signature
-                        return FunctionSignature::Rust(rust_fn_ty);
+                    if do_verify {
+                        if !self.verify_intrinsic_signature(cx, llvm_fn_ty) {
+                            cx.tcx.dcx().fatal(format!(
+                                "Intrinsic signature mismatch for `{}`: expected signature `{llvm_fn_ty:?}`",
+                                str::from_utf8(name).unwrap()
+                            ));
+                        }
                     }
+                    return FunctionSignature::Intrinsic(intrinsic, llvm_fn_ty);
                 }
             } else {
                 // it's one of 2 cases,
@@ -674,7 +703,24 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         }
     }
 
-    fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) {
+    fn apply_attrs_callsite(
+        &self,
+        bx: &mut Builder<'_, 'll, 'tcx>,
+        callsite: &'ll Value,
+        llfn: &'ll Value,
+    ) {
+        // if we are using the LLVM signature, use the LLVM attributes otherwise it might be problematic
+        let name = llvm::get_value_name(llfn);
+        if name.starts_with(b"llvm.")
+            && let Some(intrinsic) = llvm::Intrinsic::lookup(name)
+        {
+            // FIXME: also do this for overloaded intrinsics
+            if !intrinsic.is_overloaded() {
+                bx.set_intrinsic_attributes(intrinsic, callsite);
+                return;
+            }
+        }
+
         let mut func_attrs = SmallVec::<[_; 2]>::new();
         if self.ret.layout.is_uninhabited() {
             func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx));