Skip to content

Enable constant folding across crates (weak linkage + comdat) #114816

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 41 additions & 6 deletions compiler/rustc_codegen_llvm/src/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

use crate::consts::const_alloc_to_llvm;
pub use crate::context::CodegenCx;
use crate::debuginfo::metadata::{
build_const_str_di_node, build_opaque_pointer_global_var_di_node,
};
use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True};
use crate::type_::Type;
use crate::value::Value;
Expand All @@ -11,7 +14,9 @@ use rustc_codegen_ssa::traits::*;
use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher};
use rustc_hir::def_id::DefId;
use rustc_middle::bug;
use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
use rustc_middle::mir::interpret::{
ConstAllocation, ConstAllocationDebugHint, GlobalAlloc, Scalar,
};
use rustc_middle::ty::TyCtxt;
use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType};
use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer};
Expand All @@ -20,6 +25,8 @@ use rustc_target::spec::Target;
use libc::{c_char, c_uint};
use std::fmt::Write;

const RUST_VERSION: &str = env!("CFG_RELEASE_NUM");

/*
* A note on nomenclature of linking: "extern", "foreign", and "upcall".
*
Expand Down Expand Up @@ -196,15 +203,23 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
.from_key(s)
.or_insert_with(|| {
let sc = self.const_bytes(s.as_bytes());
let sym = self.generate_local_symbol_name("str");
let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
let mut hasher = StableHasher::new();
s.hash_stable(&mut hcx, &mut hasher);
hasher.finish::<Hash128>()
});
let sym = format!("__rust_{RUST_VERSION}_conststr_{hash:032x}");
let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| {
bug!("symbol `{}` is already defined", sym);
});
unsafe {
llvm::LLVMSetInitializer(g, sc);
llvm::LLVMSetGlobalConstant(g, True);
llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage);
llvm::LLVMRustSetLinkage(g, llvm::Linkage::LinkOnceODRLinkage);
llvm::SetUniqueComdat(self.llmod, g);
llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden);
}
build_const_str_di_node(self, &sym, g);
(s.to_owned(), g)
})
.1;
Expand Down Expand Up @@ -249,18 +264,38 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) {
GlobalAlloc::Memory(alloc) => {
let init = const_alloc_to_llvm(self, alloc);
let debug_hint = alloc.1;
let alloc = alloc.inner();
let value = match alloc.mutability {
Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
_ => self.static_addr_of(init, alloc.align, None),
_ => {
let value = self.static_addr_of(init, alloc.align, None);
llvm::set_linkage(value, llvm::Linkage::LinkOnceODRLinkage);
llvm::set_visibility(value, llvm::Visibility::Hidden);
value
}
};
if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() {

if llvm::get_value_name(value).is_empty() {
let name_prefix = match debug_hint {
Some(ConstAllocationDebugHint::StrLiteral) => "str",
Some(ConstAllocationDebugHint::CallerLocation) => "callerloc",
Some(ConstAllocationDebugHint::TypeName) => "typename",
Some(ConstAllocationDebugHint::VTable) => "vtable",
None => "alloc",
};

let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
let mut hasher = StableHasher::new();
alloc.hash_stable(&mut hcx, &mut hasher);
hasher.finish::<Hash128>()
});
llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes());
let name = format!("__rust_{RUST_VERSION}_{name_prefix}_{hash:032x}");
llvm::set_value_name(value, name.as_bytes());
build_opaque_pointer_global_var_di_node(self, &name, value);
if alloc.mutability == Mutability::Not {
llvm::SetUniqueComdat(self.llmod, value);
}
}
(value, AddressSpace::DATA)
}
Expand Down
56 changes: 56 additions & 0 deletions compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1252,6 +1252,62 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo
}
}

/// Creates debug information for the given constant string.
///
/// Adds the created debuginfo nodes directly to the crate's IR.
pub fn build_const_str_di_node<'ll>(
cx: &CodegenCx<'ll, '_>,
var_name: &str,
const_str: &'ll Value,
) {
if cx.dbg_cx.is_none() {
return;
}

// Only create type information if full debuginfo is enabled
if cx.sess().opts.debuginfo != DebugInfo::Full {
return;
}

unsafe {
llvm::LLVMRustDIBuilderCreateConstStr(
DIB(cx),
var_name.as_ptr().cast(),
var_name.len(),
type_di_node(cx, cx.tcx.types.str_),
const_str,
);
}
}

/// Creates debug information for the given global variable which is a pointer to an
/// unknown or opaque type.
///
/// Adds the created debuginfo nodes directly to the crate's IR.
pub fn build_opaque_pointer_global_var_di_node<'ll>(
cx: &CodegenCx<'ll, '_>,
var_name: &str,
global: &'ll Value,
) {
if cx.dbg_cx.is_none() {
return;
}

// Only create type information if full debuginfo is enabled
if cx.sess().opts.debuginfo != DebugInfo::Full {
return;
}

unsafe {
llvm::LLVMRustDIBuilderCreateOpaquePointerStaticVariable(
DIB(cx),
var_name.as_ptr().cast(),
var_name.len(),
global,
);
}
}

/// Generates LLVM debuginfo for a vtable.
///
/// The vtable type looks like a struct with a field for each function pointer and super-trait
Expand Down
15 changes: 15 additions & 0 deletions compiler/rustc_codegen_llvm/src/llvm/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1948,6 +1948,21 @@ extern "C" {
AlignInBits: u32,
) -> &'a DIGlobalVariableExpression;

pub fn LLVMRustDIBuilderCreateConstStr<'a>(
Builder: &DIBuilder<'a>,
Name: *const c_char,
NameLen: size_t,
Ty: &'a DIType,
Val: &'a Value,
) -> &'a DIGlobalVariableExpression;

pub fn LLVMRustDIBuilderCreateOpaquePointerStaticVariable<'a>(
Builder: &DIBuilder<'a>,
Name: *const c_char,
NameLen: size_t,
Val: &'a Value,
) -> &'a DIGlobalVariableExpression;

pub fn LLVMRustDIBuilderCreateVariable<'a>(
Builder: &DIBuilder<'a>,
Tag: c_uint,
Expand Down
7 changes: 4 additions & 3 deletions compiler/rustc_const_eval/src/const_eval/eval_queries.rs
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,10 @@ pub(super) fn op_to_const<'tcx>(
(ecx.tcx.global_alloc(alloc_id).unwrap_memory(), offset.bytes())
}
(None, _offset) => (
ecx.tcx.mk_const_alloc(Allocation::from_bytes_byte_aligned_immutable(
b"" as &[u8],
)),
ecx.tcx.mk_const_alloc(
Allocation::from_bytes_byte_aligned_immutable(b"" as &[u8]),
None,
),
0,
),
};
Expand Down
18 changes: 9 additions & 9 deletions compiler/rustc_const_eval/src/interpret/intern.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use super::validity::RefTracking;
use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
use rustc_errors::ErrorGuaranteed;
use rustc_hir as hir;
use rustc_middle::mir::interpret::InterpResult;
use rustc_middle::mir::interpret::{ConstAllocationDebugHint, InterpResult};
use rustc_middle::ty::{self, layout::TyAndLayout, Ty};

use rustc_ast::Mutability;
Expand Down Expand Up @@ -104,11 +104,11 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
};
// This match is just a canary for future changes to `MemoryKind`, which most likely need
// changes in this function.
match kind {
MemoryKind::Stack
| MemoryKind::Machine(const_eval::MemoryKind::Heap)
| MemoryKind::CallerLocation => {}
}
let const_allocation_kind = match kind {
MemoryKind::Stack | MemoryKind::Machine(const_eval::MemoryKind::Heap) => None,
MemoryKind::CallerLocation => Some(ConstAllocationDebugHint::CallerLocation),
};

// Set allocation mutability as appropriate. This is used by LLVM to put things into
// read-only memory, and also by Miri when evaluating other globals that
// access this one.
Expand Down Expand Up @@ -136,7 +136,7 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
};
// link the alloc id to the actual allocation
leftover_allocations.extend(alloc.provenance().ptrs().iter().map(|&(_, alloc_id)| alloc_id));
let alloc = tcx.mk_const_alloc(alloc);
let alloc = tcx.mk_const_alloc(alloc, const_allocation_kind);
tcx.set_alloc_id_memory(alloc_id, alloc);
None
}
Expand Down Expand Up @@ -428,7 +428,7 @@ pub fn intern_const_alloc_recursive<
alloc.mutability = Mutability::Not;
}
}
let alloc = tcx.mk_const_alloc(alloc);
let alloc = tcx.mk_const_alloc(alloc, None);
tcx.set_alloc_id_memory(alloc_id, alloc);
for &(_, alloc_id) in alloc.inner().provenance().ptrs().iter() {
if leftover_allocations.insert(alloc_id) {
Expand Down Expand Up @@ -467,6 +467,6 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
f(self, &dest.clone().into())?;
let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1;
alloc.mutability = Mutability::Not;
Ok(self.tcx.mk_const_alloc(alloc))
Ok(self.tcx.mk_const_alloc(alloc, None))
}
}
5 changes: 3 additions & 2 deletions compiler/rustc_const_eval/src/interpret/intrinsics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ use rustc_hir::def_id::DefId;
use rustc_middle::mir::{
self,
interpret::{
Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar,
Allocation, ConstAllocation, ConstAllocationDebugHint, ConstValue, GlobalId, InterpResult,
PointerArithmetic, Scalar,
},
BinOp, NonDivergingIntrinsic,
};
Expand Down Expand Up @@ -47,7 +48,7 @@ fn numeric_intrinsic<Prov>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<
pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
let path = crate::util::type_name(tcx, ty);
let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
tcx.mk_const_alloc(alloc)
tcx.mk_const_alloc(alloc, Some(ConstAllocationDebugHint::TypeName))
}

/// The logic for all nullary intrinsics is implemented here. These intrinsics don't get evaluated
Expand Down
44 changes: 44 additions & 0 deletions compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1035,6 +1035,50 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable(
return wrap(VarExpr);
}

extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateConstStr(
LLVMRustDIBuilderRef Builder,
const char *Name, size_t NameLen,
LLVMMetadataRef Ty, LLVMValueRef V) {
llvm::GlobalVariable *InitVal = cast<llvm::GlobalVariable>(unwrap(V));

llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression(
/* context */ nullptr, StringRef(Name, NameLen),
/* linkageName */ StringRef(),
nullptr, 0, unwrapDI<DIType>(Ty), /* isLocalToUnit */ false,
/* isDefined */ true,
/* expr */ nullptr, /* decl */ nullptr,
/* templateParams */ nullptr,
/* alignInBits */ 0);

InitVal->setMetadata("dbg", VarExpr);

return wrap(VarExpr);
}

extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateOpaquePointerStaticVariable(
LLVMRustDIBuilderRef Builder,
const char *Name, size_t NameLen,
LLVMValueRef V) {
llvm::GlobalVariable *InitVal = cast<llvm::GlobalVariable>(unwrap(V));

DataLayout Layout = InitVal->getParent()->getDataLayout();
DIType *DebugType = Builder->createPointerType(nullptr, Layout.getTypeSizeInBits(
InitVal->getValueType()));

llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression(
/* context */ nullptr, StringRef(Name, NameLen),
/* linkageName */ StringRef(),
nullptr, 0, DebugType, /* isLocalToUnit */ false,
/* isDefined */ true,
/* expr */ nullptr, /* decl */ nullptr,
/* templateParams */ nullptr,
/* alignInBits */ 0);

InitVal->setMetadata("dbg", VarExpr);

return wrap(VarExpr);
}

extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable(
LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope,
const char *Name, size_t NameLen,
Expand Down
19 changes: 18 additions & 1 deletion compiler/rustc_middle/src/mir/interpret/allocation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,10 @@ impl hash::Hash for Allocation {
/// (`ConstAllocation`) are used quite a bit.
#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable)]
#[rustc_pass_by_value]
pub struct ConstAllocation<'tcx>(pub Interned<'tcx, Allocation>);
pub struct ConstAllocation<'tcx>(
pub Interned<'tcx, Allocation>,
pub Option<ConstAllocationDebugHint>,
);

impl<'tcx> fmt::Debug for ConstAllocation<'tcx> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
Expand All @@ -166,6 +169,20 @@ impl<'tcx> ConstAllocation<'tcx> {
}
}

/// Hint used to tag the name of a `ConstAllocation` in debug information so
/// that data in the final binary can be attributed back to the allocator.
#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable, Encodable, Decodable)]
pub enum ConstAllocationDebugHint {
/// A string literal.
StrLiteral,
/// Caller infromation (e.g., for panics).
CallerLocation,
/// The name of a type.
TypeName,
/// A vtable.
VTable,
}

/// We have our own error type that does not know about the `AllocId`; that information
/// is added when converting to `InterpError`.
#[derive(Debug)]
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_middle/src/mir/interpret/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar};

pub use self::allocation::{
alloc_range, AllocBytes, AllocError, AllocRange, AllocResult, Allocation, ConstAllocation,
InitChunk, InitChunkIter,
ConstAllocationDebugHint, InitChunk, InitChunkIter,
};

pub use self::pointer::{Pointer, PointerArithmetic, Provenance};
Expand Down
4 changes: 3 additions & 1 deletion compiler/rustc_middle/src/ty/codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for ty::Const<'tcx> {

impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for ConstAllocation<'tcx> {
fn encode(&self, e: &mut E) {
self.1.encode(e);
self.inner().encode(e)
}
}
Expand Down Expand Up @@ -356,7 +357,8 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for [ty::ValTre

impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ConstAllocation<'tcx> {
fn decode(decoder: &mut D) -> Self {
decoder.interner().mk_const_alloc(Decodable::decode(decoder))
let debug_hint = Decodable::decode(decoder);
decoder.interner().mk_const_alloc(Decodable::decode(decoder), debug_hint)
}
}

Expand Down
Loading