Intern ConstValue.

Author: cjgillot

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -3,7 +3,7 @@
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{
-    read_target_uint, AllocId, ConstValue, ErrorHandled, GlobalAlloc, Scalar,
+    read_target_uint, AllocId, ConstValue, ConstValueKind, ErrorHandled, GlobalAlloc, Scalar,
 };
 
 use cranelift_module::*;
@@ -115,9 +115,9 @@ pub(crate) fn codegen_const_value<'tcx>(
         return CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout);
     }
 
-    match const_val {
-        ConstValue::ZeroSized => unreachable!(), // we already handled ZST above
-        ConstValue::Scalar(x) => match x {
+    match *const_val.kind() {
+        ConstValueKind::ZeroSized => unreachable!(), // we already handled ZST above
+        ConstValueKind::Scalar(x) => match x {
             Scalar::Int(int) => {
                 if fx.clif_type(layout.ty).is_some() {
                     return CValue::const_val(fx, layout, int);
@@ -200,12 +200,12 @@ pub(crate) fn codegen_const_value<'tcx>(
                 CValue::by_val(val, layout)
             }
         },
-        ConstValue::Indirect { alloc_id, offset } => CValue::by_ref(
+        ConstValueKind::Indirect { alloc_id, offset } => CValue::by_ref(
             pointer_for_allocation(fx, alloc_id)
                 .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()),
             layout,
         ),
-        ConstValue::Slice { data, start, end } => {
+        ConstValueKind::Slice { data, start, end } => {
             let alloc_id = fx.tcx.reserve_and_set_memory_alloc(data);
             let ptr = pointer_for_allocation(fx, alloc_id)
                 .offset_i64(fx, i64::try_from(start).unwrap())

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -171,8 +171,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx)
                     .expect("simd_shuffle idx not const");
 
-                let idx_bytes = match idx_const {
-                    ConstValue::Indirect { alloc_id, offset } => {
+                let idx_bytes = match *idx_const.kind() {
+                    ConstValueKind::Indirect { alloc_id, offset } => {
                         let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory();
                         let size = Size::from_bytes(
                             4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */

compiler/rustc_codegen_ssa/src/common.rs

@@ -197,7 +197,7 @@ pub fn asm_const_to_str<'tcx>(
     const_value: ConstValue<'tcx>,
     ty_and_layout: TyAndLayout<'tcx>,
 ) -> String {
-    let ConstValue::Scalar(scalar) = const_value else {
+    let Some(scalar) = const_value.try_to_scalar() else {
         span_bug!(sp, "expected Scalar for promoted asm const, but got {:#?}", const_value)
     };
     let value = scalar.assert_bits(ty_and_layout.size);

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -7,7 +7,7 @@ use crate::traits::*;
 use crate::MemFlags;
 
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{alloc_range, ConstValue, Pointer, Scalar};
+use rustc_middle::mir::interpret::{alloc_range, ConstValue, ConstValueKind, Pointer, Scalar};
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::Ty;
 use rustc_target::abi::{self, Abi, Align, Size};
@@ -91,16 +91,16 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
     ) -> Self {
         let layout = bx.layout_of(ty);
 
-        let val = match val {
-            ConstValue::Scalar(x) => {
+        let val = match *val.kind() {
+            ConstValueKind::Scalar(x) => {
                 let Abi::Scalar(scalar) = layout.abi else {
                     bug!("from_const: invalid ByVal layout: {:#?}", layout);
                 };
                 let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
                 OperandValue::Immediate(llval)
             }
-            ConstValue::ZeroSized => return OperandRef::zero_sized(layout),
-            ConstValue::Slice { data, start, end } => {
+            ConstValueKind::ZeroSized => return OperandRef::zero_sized(layout),
+            ConstValueKind::Slice { data, start, end } => {
                 let Abi::ScalarPair(a_scalar, _) = layout.abi else {
                     bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
                 };
@@ -119,7 +119,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 let b_llval = bx.const_usize((end - start) as u64);
                 OperandValue::Pair(a_llval, b_llval)
             }
-            ConstValue::Indirect { alloc_id, offset } => {
+            ConstValueKind::Indirect { alloc_id, offset } => {
                 let alloc = bx.tcx().global_alloc(alloc_id).unwrap_memory();
                 return Self::from_const_alloc(bx, layout, alloc, offset);
             }

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -111,23 +111,24 @@ pub(super) fn op_to_const<'tcx>(
     ecx: &CompileTimeEvalContext<'_, 'tcx>,
     op: &OpTy<'tcx>,
 ) -> ConstValue<'tcx> {
+    let tcx = *ecx.tcx;
     // Handle ZST consistently and early.
     if op.layout.is_zst() {
-        return ConstValue::ZeroSized;
+        return ConstValue::zero_sized(tcx);
     }
 
-    // All scalar types should be stored as `ConstValue::Scalar`. This is needed to make
-    // `ConstValue::try_to_scalar` efficient; we want that to work for *all* constants of scalar
+    // All scalar types should be stored as `ConstValueKind::Scalar`. This is needed to make
+    // `ConstValueKind::try_to_scalar` efficient; we want that to work for *all* constants of scalar
     // type (it's used throughout the compiler and having it work just on literals is not enough)
     // and we want it to be fast (i.e., don't go to an `Allocation` and reconstruct the `Scalar`
     // from its byte-serialized form).
     let force_as_immediate = match op.layout.abi {
         Abi::Scalar(abi::Scalar::Initialized { .. }) => true,
-        // We don't *force* `ConstValue::Slice` for `ScalarPair`. This has the advantage that if the
+        // We don't *force* `ConstValueKind::Slice` for `ScalarPair`. This has the advantage that if the
         // input `op` is a place, then turning it into a `ConstValue` and back into a `OpTy` will
         // not have to generate any duplicate allocations (we preserve the original `AllocId` in
-        // `ConstValue::Indirect`). It means accessing the contents of a slice can be slow (since
-        // they can be stored as `ConstValue::Indirect`), but that's not relevant since we barely
+        // `ConstValueKind::Indirect`). It means accessing the contents of a slice can be slow (since
+        // they can be stored as `ConstValueKind::Indirect`), but that's not relevant since we barely
         // ever have to do this. (`try_get_slice_bytes_for_diagnostics` exists to provide this
         // functionality.)
         _ => false,
@@ -145,26 +146,26 @@ pub(super) fn op_to_const<'tcx>(
             // We know `offset` is relative to the allocation, so we can use `into_parts`.
             let (alloc_id, offset) = mplace.ptr().into_parts();
             let alloc_id = alloc_id.expect("cannot have `fake` place fot non-ZST type");
-            ConstValue::Indirect { alloc_id, offset }
+            ConstValue::from_memory(tcx, alloc_id, offset)
         }
         // see comment on `let force_as_immediate` above
         Right(imm) => match *imm {
-            Immediate::Scalar(x) => ConstValue::Scalar(x),
+            Immediate::Scalar(x) => ConstValue::from_scalar(tcx, x),
             Immediate::ScalarPair(a, b) => {
                 debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
                 // FIXME: assert that this has an appropriate type.
                 // Currently we actually get here for non-[u8] slices during valtree construction!
                 let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to actually allocated memory";
                 // We know `offset` is relative to the allocation, so we can use `into_parts`.
-                // We use `ConstValue::Slice` so that we don't have to generate an allocation for
-                // `ConstValue::Indirect` here.
+                // We use `ConstValueKind::Slice` so that we don't have to generate an allocation for
+                // `ConstValueKind::Indirect` here.
                 let (alloc_id, offset) = a.to_pointer(ecx).expect(msg).into_parts();
                 let alloc_id = alloc_id.expect(msg);
                 let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
                 let start = offset.bytes_usize();
                 let len = b.to_target_usize(ecx).expect(msg);
                 let len: usize = len.try_into().unwrap();
-                ConstValue::Slice { data, start, end: start + len }
+                ConstValue::from_slice(tcx, data, start, start + len)
             }
             Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
         },

compiler/rustc_const_eval/src/const_eval/mod.rs

@@ -30,7 +30,7 @@ pub(crate) fn const_caller_location(
     if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() {
         bug!("intern_const_alloc_recursive should not error in this case")
     }
-    ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr(), &tcx))
+    ConstValue::from_scalar(tcx, Scalar::from_maybe_pointer(loc_place.ptr(), &tcx))
 }
 
 // We forbid type-level constants that contain more than `VALTREE_MAX_NODES` nodes.

compiler/rustc_const_eval/src/const_eval/valtrees.rs

@@ -219,10 +219,10 @@ pub fn valtree_to_const_value<'tcx>(
     match ty.kind() {
         ty::FnDef(..) => {
             assert!(valtree.unwrap_branch().is_empty());
-            ConstValue::ZeroSized
+            ConstValue::zero_sized(tcx)
         }
         ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => match valtree {
-            ty::ValTree::Leaf(scalar_int) => ConstValue::Scalar(Scalar::Int(scalar_int)),
+            ty::ValTree::Leaf(scalar_int) => ConstValue::from_scalar(tcx, Scalar::Int(scalar_int)),
             ty::ValTree::Branch(_) => bug!(
                 "ValTrees for Bool, Int, Uint, Float or Char should have the form ValTree::Leaf"
             ),
@@ -237,7 +237,7 @@ pub fn valtree_to_const_value<'tcx>(
             let layout = tcx.layout_of(param_env_ty).unwrap();
             if layout.is_zst() {
                 // Fast path to avoid some allocations.
-                return ConstValue::ZeroSized;
+                return ConstValue::zero_sized(tcx);
             }
             if layout.abi.is_scalar()
                 && (matches!(ty.kind(), ty::Tuple(_))

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -64,24 +64,24 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
         sym::type_name => {
             ensure_monomorphic_enough(tcx, tp_ty)?;
             let alloc = alloc_type_name(tcx, tp_ty);
-            ConstValue::Slice { data: alloc, start: 0, end: alloc.inner().len() }
+            ConstValue::from_slice(tcx, alloc, 0, alloc.inner().len())
         }
         sym::needs_drop => {
             ensure_monomorphic_enough(tcx, tp_ty)?;
-            ConstValue::from_bool(tp_ty.needs_drop(tcx, param_env))
+            ConstValue::from_bool(tcx, tp_ty.needs_drop(tcx, param_env))
         }
         sym::pref_align_of => {
             // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
             let layout = tcx.layout_of(param_env.and(tp_ty)).map_err(|e| err_inval!(Layout(*e)))?;
-            ConstValue::from_target_usize(layout.align.pref.bytes(), &tcx)
+            ConstValue::from_target_usize(tcx, layout.align.pref.bytes())
         }
         sym::type_id => {
             ensure_monomorphic_enough(tcx, tp_ty)?;
-            ConstValue::from_u128(tcx.type_id_hash(tp_ty).as_u128())
+            ConstValue::from_u128(tcx, tcx.type_id_hash(tp_ty).as_u128())
         }
         sym::variant_count => match tp_ty.kind() {
             // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
-            ty::Adt(adt, _) => ConstValue::from_target_usize(adt.variants().len() as u64, &tcx),
+            ty::Adt(adt, _) => ConstValue::from_target_usize(tcx, adt.variants().len() as u64),
             ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
                 throw_inval!(TooGeneric)
             }
@@ -106,7 +106,7 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
             | ty::GeneratorWitnessMIR(_, _)
             | ty::Never
             | ty::Tuple(_)
-            | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
+            | ty::Error(_) => ConstValue::from_target_usize(tcx, 0u64),
         },
         other => bug!("`{}` is not a zero arg intrinsic", other),
     })

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -14,9 +14,9 @@ use rustc_span::Span;
 use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size};
 
 use super::{
-    alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, Frame, GlobalId,
-    InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, PlaceTy, Pointer,
-    Projectable, Provenance, Scalar,
+    alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, ConstValueKind,
+    Frame, GlobalId, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, PlaceTy,
+    Pointer, Projectable, Provenance, Scalar,
 };
 
 /// An `Immediate` represents a single immediate self-contained Rust value.
@@ -763,16 +763,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             })
         };
         let layout = from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(ty))?;
-        let op = match val_val {
-            ConstValue::Indirect { alloc_id, offset } => {
+        let op = match *val_val.kind() {
+            ConstValueKind::Indirect { alloc_id, offset } => {
                 // We rely on mutability being set correctly in that allocation to prevent writes
                 // where none should happen.
                 let ptr = self.global_base_pointer(Pointer::new(alloc_id, offset))?;
                 Operand::Indirect(MemPlace::from_ptr(ptr.into()))
             }
-            ConstValue::Scalar(x) => Operand::Immediate(adjust_scalar(x)?.into()),
-            ConstValue::ZeroSized => Operand::Immediate(Immediate::Uninit),
-            ConstValue::Slice { data, start, end } => {
+            ConstValueKind::Scalar(x) => Operand::Immediate(adjust_scalar(x)?.into()),
+            ConstValueKind::ZeroSized => Operand::Immediate(Immediate::Uninit),
+            ConstValueKind::Slice { data, start, end } => {
                 // We rely on mutability being set correctly in `data` to prevent writes
                 // where none should happen.
                 let ptr = Pointer::new(

compiler/rustc_const_eval/src/transform/promote_consts.rs

@@ -769,7 +769,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                         let unit = Rvalue::Use(Operand::Constant(Box::new(Constant {
                             span: statement.source_info.span,
                             user_ty: None,
-                            literal: ConstantKind::zero_sized(self.tcx.types.unit),
+                            literal: ConstantKind::zero_sized(self.tcx, self.tcx.types.unit),
                         })));
                         mem::replace(rhs, unit)
                     },

compiler/rustc_middle/src/mir/interpret/mod.rs

@@ -149,7 +149,7 @@ pub use self::error::{
     UnsupportedOpInfo, ValidationErrorInfo, ValidationErrorKind,
 };
 
-pub use self::value::{ConstAlloc, ConstValue, Scalar};
+pub use self::value::{ConstAlloc, ConstValue, ConstValueKind, Scalar};
 
 pub use self::allocation::{
     alloc_range, AllocBytes, AllocError, AllocRange, AllocResult, Allocation, ConstAllocation,