Description
In https://bugzilla.mozilla.org/show_bug.cgi?id=1787715 (currently not public for security reasons) we ran into a stack overflow when flate2 calls Box::<CompressorOxide>>::default() in DeflateBackend::make() here:
https://github.com/rust-lang/flate2-rs/blob/cc5ed7f817cc5e5712b2bb924ed19cab4f389a47/src/ffi/rust.rs#L131
DeflateBackend::make() is getting compiled with a call to chkstk(65664)
A reduced version of this is here:
https://rust.godbolt.org/z/hTE3785xY
pub const LZ_CODE_BUF_SIZE: usize = 64 * 1024;
struct LZOxide {
pub codes: [u8; LZ_CODE_BUF_SIZE],
}
impl LZOxide {
#[inline(never)]
const fn new() -> Self {
LZOxide {
codes: [0; LZ_CODE_BUF_SIZE],
}
}
}
pub struct CompressorOxide {
lz: LZOxide,
huff: Box<u32>,
pub codes: [u8; LZ_CODE_BUF_SIZE],
}
impl Default for CompressorOxide {
#[inline(always)]
fn default() -> Self {
CompressorOxide {
lz: LZOxide::new(),
huff: Box::default(),
codes: [0; LZ_CODE_BUF_SIZE],
}
}
}
pub fn f() -> Box<CompressorOxide> {
Box::default()
}A further reduced version that doesn't require any of the special box syntax in Default() is here:
https://rust.godbolt.org/z/8e5EPafzb
pub const LZ_CODE_BUF_SIZE: usize = 64 * 1024;
struct LZOxide {
pub codes: [u8; LZ_CODE_BUF_SIZE],
}
impl LZOxide {
#[inline(never)]
const fn new() -> Self {
LZOxide {
codes: [0; LZ_CODE_BUF_SIZE],
}
}
}
pub struct CompressorOxide {
lz: LZOxide,
huff: Box<u32>,
//pub codes: [u8; LZ_CODE_BUF_SIZE],
}
impl Default for CompressorOxide {
#[inline(always)]
fn default() -> Self {
CompressorOxide {
lz: LZOxide::new(),
huff: Box::default(),
//codes: [0; LZ_CODE_BUF_SIZE],
}
}
}
use std::ptr;
use std::alloc::{Layout, alloc};
pub fn g() -> Box<CompressorOxide> {
let layout = Layout::new::<CompressorOxide>();
let ptr =
unsafe { alloc(layout) as *mut CompressorOxide };
assert!(!ptr.is_null());
unsafe { ptr::write(ptr, CompressorOxide {
lz: LZOxide::new(),
huff: Box::default(),
});
Box::from_raw(ptr)}
}This seems to have to do with LZOxide::new() not being inlined. If it's inlined the problem goes away. If the field initialization order of CompressorOxide is swapped
so that we call Box::<u32>::default() before LZOxide::new() the problem also goes away.