Enum layout optimizations

src/liballoc/arc.rs

@@ -76,11 +76,11 @@ use core::default::Default;
 use core::kinds::{Sync, Send};
 use core::mem::{min_align_of, size_of, drop};
 use core::mem;
+use core::nonzero::NonZero;
 use core::ops::{Drop, Deref};
 use core::option::Option;
 use core::option::Option::{Some, None};
-use core::ptr::RawPtr;
-use core::ptr;
+use core::ptr::{mod, RawPtr};
 use heap::deallocate;
 
 /// An atomically reference counted wrapper for shared state.
@@ -114,7 +114,7 @@ use heap::deallocate;
 pub struct Arc<T> {
     // FIXME #12808: strange name to try to avoid interfering with
     // field accesses of the contained type via Deref
-    _ptr: *mut ArcInner<T>,
+    _ptr: NonZero<*mut ArcInner<T>>,
 }
 
 unsafe impl<T: Sync + Send> Send for Arc<T> { }
@@ -130,7 +130,7 @@ unsafe impl<T: Sync + Send> Sync for Arc<T> { }
 pub struct Weak<T> {
     // FIXME #12808: strange name to try to avoid interfering with
     // field accesses of the contained type via Deref
-    _ptr: *mut ArcInner<T>,
+    _ptr: NonZero<*mut ArcInner<T>>,
 }
 
 unsafe impl<T: Sync + Send> Send for Weak<T> { }
@@ -165,7 +165,7 @@ impl<T> Arc<T> {
             weak: atomic::AtomicUint::new(1),
             data: data,
         };
-        Arc { _ptr: unsafe { mem::transmute(x) } }
+        Arc { _ptr: unsafe { NonZero::new(mem::transmute(x)) } }
     }
 
     /// Downgrades the `Arc<T>` to a `Weak<T>` reference.
@@ -194,7 +194,7 @@ impl<T> Arc<T> {
         // pointer is valid. Furthermore, we know that the `ArcInner` structure itself is `Sync`
         // because the inner data is `Sync` as well, so we're ok loaning out an immutable pointer
         // to these contents.
-        unsafe { &*self._ptr }
+        unsafe { &**self._ptr }
     }
 }
 
@@ -281,7 +281,7 @@ impl<T: Send + Sync + Clone> Arc<T> {
         // pointer that will ever be returned to T. Our reference count is guaranteed to be 1 at
         // this point, and we required the Arc itself to be `mut`, so we're returning the only
         // possible reference to the inner data.
-        let inner = unsafe { &mut *self._ptr };
+        let inner = unsafe { &mut **self._ptr };
         &mut inner.data
     }
 }
@@ -316,7 +316,8 @@ impl<T: Sync + Send> Drop for Arc<T> {
     fn drop(&mut self) {
         // This structure has #[unsafe_no_drop_flag], so this drop glue may run more than once (but
         // it is guaranteed to be zeroed after the first if it's run more than once)
-        if self._ptr.is_null() { return }
+        let ptr = *self._ptr;
+        if ptr.is_null() { return }
 
         // Because `fetch_sub` is already atomic, we do not need to synchronize with other threads
         // unless we are going to delete the object. This same logic applies to the below
@@ -346,7 +347,7 @@ impl<T: Sync + Send> Drop for Arc<T> {
 
         if self.inner().weak.fetch_sub(1, atomic::Release) == 1 {
             atomic::fence(atomic::Acquire);
-            unsafe { deallocate(self._ptr as *mut u8, size_of::<ArcInner<T>>(),
+            unsafe { deallocate(ptr as *mut u8, size_of::<ArcInner<T>>(),
                                 min_align_of::<ArcInner<T>>()) }
         }
     }
@@ -386,7 +387,7 @@ impl<T: Sync + Send> Weak<T> {
     #[inline]
     fn inner(&self) -> &ArcInner<T> {
         // See comments above for why this is "safe"
-        unsafe { &*self._ptr }
+        unsafe { &**self._ptr }
     }
 }
 
@@ -442,14 +443,16 @@ impl<T: Sync + Send> Drop for Weak<T> {
     /// } // implicit drop
     /// ```
     fn drop(&mut self) {
+        let ptr = *self._ptr;
+
         // see comments above for why this check is here
-        if self._ptr.is_null() { return }
+        if ptr.is_null() { return }
 
         // If we find out that we were the last weak pointer, then its time to deallocate the data
         // entirely. See the discussion in Arc::drop() about the memory orderings
         if self.inner().weak.fetch_sub(1, atomic::Release) == 1 {
             atomic::fence(atomic::Acquire);
-            unsafe { deallocate(self._ptr as *mut u8, size_of::<ArcInner<T>>(),
+            unsafe { deallocate(ptr as *mut u8, size_of::<ArcInner<T>>(),
                                 min_align_of::<ArcInner<T>>()) }
         }
     }

src/liballoc/rc.rs

@@ -150,11 +150,11 @@ use core::fmt;
 use core::hash::{mod, Hash};
 use core::kinds::marker;
 use core::mem::{transmute, min_align_of, size_of, forget};
+use core::nonzero::NonZero;
 use core::ops::{Deref, Drop};
 use core::option::Option;
 use core::option::Option::{Some, None};
-use core::ptr;
-use core::ptr::RawPtr;
+use core::ptr::{mod, RawPtr};
 use core::result::Result;
 use core::result::Result::{Ok, Err};
 
@@ -174,7 +174,7 @@ struct RcBox<T> {
 pub struct Rc<T> {
     // FIXME #12808: strange names to try to avoid interfering with field accesses of the contained
     // type via Deref
-    _ptr: *mut RcBox<T>,
+    _ptr: NonZero<*mut RcBox<T>>,
     _nosend: marker::NoSend,
     _noshare: marker::NoSync
 }
@@ -196,11 +196,11 @@ impl<T> Rc<T> {
                 // there is an implicit weak pointer owned by all the strong pointers, which
                 // ensures that the weak destructor never frees the allocation while the strong
                 // destructor is running, even if the weak pointer is stored inside the strong one.
-                _ptr: transmute(box RcBox {
+                _ptr: NonZero::new(transmute(box RcBox {
                     value: value,
                     strong: Cell::new(1),
                     weak: Cell::new(1)
-                }),
+                })),
                 _nosend: marker::NoSend,
                 _noshare: marker::NoSync
             }
@@ -281,7 +281,7 @@ pub fn try_unwrap<T>(rc: Rc<T>) -> Result<T, Rc<T>> {
             let val = ptr::read(&*rc); // copy the contained object
             // destruct the box and skip our Drop
             // we can ignore the refcounts because we know we're unique
-            deallocate(rc._ptr as *mut u8, size_of::<RcBox<T>>(),
+            deallocate(*rc._ptr as *mut u8, size_of::<RcBox<T>>(),
                         min_align_of::<RcBox<T>>());
             forget(rc);
             Ok(val)
@@ -311,7 +311,7 @@ pub fn try_unwrap<T>(rc: Rc<T>) -> Result<T, Rc<T>> {
 #[experimental]
 pub fn get_mut<'a, T>(rc: &'a mut Rc<T>) -> Option<&'a mut T> {
     if is_unique(rc) {
-        let inner = unsafe { &mut *rc._ptr };
+        let inner = unsafe { &mut **rc._ptr };
         Some(&mut inner.value)
     } else {
         None
@@ -343,7 +343,7 @@ impl<T: Clone> Rc<T> {
         // pointer that will ever be returned to T. Our reference count is guaranteed to be 1 at
         // this point, and we required the `Rc<T>` itself to be `mut`, so we're returning the only
         // possible reference to the inner value.
-        let inner = unsafe { &mut *self._ptr };
+        let inner = unsafe { &mut **self._ptr };
         &mut inner.value
     }
 }
@@ -391,7 +391,8 @@ impl<T> Drop for Rc<T> {
     /// ```
     fn drop(&mut self) {
         unsafe {
-            if !self._ptr.is_null() {
+            let ptr = *self._ptr;
+            if !ptr.is_null() {
                 self.dec_strong();
                 if self.strong() == 0 {
                     ptr::read(&**self); // destroy the contained object
@@ -401,7 +402,7 @@ impl<T> Drop for Rc<T> {
                     self.dec_weak();
 
                     if self.weak() == 0 {
-                        deallocate(self._ptr as *mut u8, size_of::<RcBox<T>>(),
+                        deallocate(ptr as *mut u8, size_of::<RcBox<T>>(),
                                    min_align_of::<RcBox<T>>())
                     }
                 }
@@ -618,7 +619,7 @@ impl<T: fmt::Show> fmt::Show for Rc<T> {
 pub struct Weak<T> {
     // FIXME #12808: strange names to try to avoid interfering with
     // field accesses of the contained type via Deref
-    _ptr: *mut RcBox<T>,
+    _ptr: NonZero<*mut RcBox<T>>,
     _nosend: marker::NoSend,
     _noshare: marker::NoSync
 }
@@ -682,12 +683,13 @@ impl<T> Drop for Weak<T> {
     /// ```
     fn drop(&mut self) {
         unsafe {
-            if !self._ptr.is_null() {
+            let ptr = *self._ptr;
+            if !ptr.is_null() {
                 self.dec_weak();
                 // the weak count starts at 1, and will only go to zero if all the strong pointers
                 // have disappeared.
                 if self.weak() == 0 {
-                    deallocate(self._ptr as *mut u8, size_of::<RcBox<T>>(),
+                    deallocate(ptr as *mut u8, size_of::<RcBox<T>>(),
                                min_align_of::<RcBox<T>>())
                 }
             }
@@ -742,12 +744,12 @@ trait RcBoxPtr<T> {
 
 impl<T> RcBoxPtr<T> for Rc<T> {
     #[inline(always)]
-    fn inner(&self) -> &RcBox<T> { unsafe { &(*self._ptr) } }
+    fn inner(&self) -> &RcBox<T> { unsafe { &(**self._ptr) } }
 }
 
 impl<T> RcBoxPtr<T> for Weak<T> {
     #[inline(always)]
-    fn inner(&self) -> &RcBox<T> { unsafe { &(*self._ptr) } }
+    fn inner(&self) -> &RcBox<T> { unsafe { &(**self._ptr) } }
 }
 
 #[cfg(test)]