copy_nonverlapping_memory for hashes, round_up_to_next

Author: pczarn

src/libstd/collections/hash/table.rs

@@ -450,9 +450,7 @@ impl<K, V> RawTable<K, V> {
             };
         }
 
-        let elem_size = size_of::<(K, V)>() + size_of::<u64>();
-        let size = capacity.checked_mul(elem_size).expect("capacity overflow");
-        let hashes = allocate(size, align::<K, V>());
+        let hashes = allocate(checked_size_generic::<K, V>(capacity), align::<K, V>());
         if hashes.is_null() { ::alloc::oom() }
 
         RawTable {
@@ -483,25 +481,39 @@ impl<K, V> RawTable<K, V> {
     }
 
     pub fn grow_inplace(&mut self, capacity: uint) -> bool {
-        if self.middle.0.is_null() || capacity < self.capacity {
+        assert!(capacity.is_power_of_two());
+        assert!(capacity >= self.capacity);
+
+        if self.middle.0.is_null() {
             return false;
         }
 
-        let size = self.capacity * (size_of::<u64>() + size_of::<(K, V)>());
-        let new_size = (size_of::<u64>() + size_of::<(K, V)>()).checked_mul(capacity)
-                                                               .expect("capacity overflow");
+        let new_size = checked_size_generic::<K, V>(capacity);
+
         unsafe {
             let ptr = self.middle.0.offset(-(self.capacity as isize)) as *mut u8;
             let is_inplace = reallocate_inplace(ptr,
-                                                size,
+                                                size_generic::<K, V>(self.capacity),
                                                 new_size,
                                                 align::<K, V>()) >= new_size;
 
             if is_inplace {
-                let hashes = self.middle.0.offset((capacity - self.capacity) as isize) as *mut Option<SafeHash>;
-                copy_memory(hashes, self.middle.0 as *const Option<SafeHash>, self.capacity);
+                let cap_diff = (capacity - self.capacity) as isize;
+                let hashes = self.middle.0.offset(cap_diff) as *mut Option<SafeHash>;
+                // Copy the array of hashes. Maybe it's already in cache.
+                if size_of::<(K, V)>() >= size_of::<Option<SafeHash>>() {
+                    // The regions of memory occupied by old and new hash arrays are disjoint.
+                    // before: [KVKVKVKV|h h h h ]
+                    // after:  [KVKVKVKV|KVKVKVKV|h h h h h h h h ]
+                    copy_nonoverlapping_memory(hashes, self.middle.0 as *const _, self.capacity);
+                } else {
+                    // before: [KVKVKVKV|h   h   |h   h   ]
+                    // after:  [KVKVKVKV|KVKVKVKV|h   h   h   h   h   h   h   h   ]
+                    copy_memory(hashes, self.middle.0 as *const _, self.capacity);
+                }
                 zero_memory(hashes.offset(self.capacity as int), capacity - self.capacity);
-                self.middle = Unique(self.middle.0.offset((capacity - self.capacity) as isize));
+
+                self.middle = Unique(self.middle.0.offset(cap_diff));
                 self.capacity = capacity;
             }
 
@@ -536,6 +548,42 @@ impl<K, V> RawTable<K, V> {
     }
 }
 
+/// Rounds up to a multiple of a power of two. Returns the closest multiple
+/// of `target_alignment` that is higher or equal to `unrounded`.
+///
+/// # Panics
+///
+/// Panics if `target_alignment` is not a power of two.
+fn round_up_to_next(unrounded: uint, target_alignment: uint) -> uint {
+    assert!(target_alignment.is_power_of_two());
+    (unrounded + target_alignment - 1) & !(target_alignment - 1)
+}
+
+#[test]
+fn test_rounding() {
+    assert_eq!(round_up_to_next(0, 4), 0);
+    assert_eq!(round_up_to_next(1, 4), 4);
+    assert_eq!(round_up_to_next(2, 4), 4);
+    assert_eq!(round_up_to_next(3, 4), 4);
+    assert_eq!(round_up_to_next(4, 4), 4);
+    assert_eq!(round_up_to_next(5, 4), 8);
+}
+
+#[inline]
+fn size_generic<K, V>(capacity: usize) -> usize {
+    let hash_align = min_align_of::<Option<SafeHash>>();
+    round_up_to_next(size_of::<(K, V)>() * capacity, hash_align) + size_of::<SafeHash>() * capacity
+}
+
+fn checked_size_generic<K, V>(capacity: usize) -> usize {
+    let size = size_generic::<K, V>(capacity);
+    let elem_size = size_of::<(K, V)>() + size_of::<SafeHash>();
+    assert!(size >= capacity.checked_mul(elem_size).expect("capacity overflow"),
+            "capacity overflow");
+    size
+}
+
+#[inline]
 fn align<K, V>() -> usize {
     cmp::max(mem::min_align_of::<(K, V)>(), mem::min_align_of::<u64>())
 }
@@ -579,7 +627,6 @@ impl<K, V, M: Clone> Clone for RawFullBuckets<K, V, M> {
             raw: self.raw,
             hashes_end: self.hashes_end,
             table: self.table.clone(),
-            // marker: marker::ContravariantLifetime,
         }
     }
 }
@@ -693,11 +740,9 @@ impl<K, V> Drop for RawTable<K, V> {
             }
         }
 
-        let size = self.capacity * (size_of::<u64>() + size_of::<(K, V)>());
-
         unsafe {
             let ptr = self.middle.0.offset(-(self.capacity as isize)) as *mut u8;
-            deallocate(ptr, size, align::<K, V>());
+            deallocate(ptr, size_generic::<K, V>(self.capacity), align::<K, V>());
         }
     }
 }