Use a variable-length integer for many serialization wrappers

The lightning protocol uses u16s for lengths in most cases. As our
serialization framework primarily targets that, we must as well.
However, because we may serialize objects that have  more than 65K
entries, we want to be able to store larger values. Thus, we define
a variable length integer here which is backwards-compatible but
treats 0xffff as "read eight more bytes".

This doesn't address any specific known issue, but feels like good
practice just in case.

Author: TheBlueMatt

lightning/src/util/ser.rs

@@ -383,6 +383,35 @@ impl Readable for BigSize {
 	}
 }
 
+/// The lightning protocol uses u16s for lengths in most cases. As our serialization framework
+/// primarily targets that, we must as well. However, because we may serialize objects that have
+/// more than 65K entries, we need to be able to store larger values. Thus, we define a variable
+/// length integer here which is backwards-compatible but treats 0xffff as "read eight more bytes".
+struct U16Or64(pub u64);
+impl Writeable for U16Or64 {
+	#[inline]
+	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+		if self.0 < 0xffff {
+			(self.0 as u16).write(writer)
+		} else {
+			0xffffu16.write(writer)?;
+			(self.0 - 0xfffe).write(writer)
+		}
+	}
+}
+
+impl Readable for U16Or64 {
+	#[inline]
+	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+		let mut val: u64 = <u16 as Readable>::read(r)? as u64;
+		if val == 0xffff {
+			val = <u64 as Readable>::read(r)?
+				.checked_add(0xfffe).ok_or(DecodeError::InvalidValue)?;
+		}
+		Ok(U16Or64(val))
+	}
+}
+
 /// In TLV we occasionally send fields which only consist of, or potentially end with, a
 /// variable-length integer which is simply truncated by skipping high zero bytes. This type
 /// encapsulates such integers implementing [`Readable`]/[`Writeable`] for them.
@@ -597,7 +626,7 @@ macro_rules! impl_for_map {
 		{
 			#[inline]
 			fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-				(self.len() as u16).write(w)?;
+				U16Or64(self.len() as u64).write(w)?;
 				for (key, value) in self.iter() {
 					key.write(w)?;
 					value.write(w)?;
@@ -611,9 +640,9 @@ macro_rules! impl_for_map {
 		{
 			#[inline]
 			fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-				let len: u16 = Readable::read(r)?;
-				let mut ret = $constr(len as usize);
-				for _ in 0..len {
+				let len: U16Or64 = Readable::read(r)?;
+				let mut ret = $constr(len.0 as usize);
+				for _ in 0..len.0 {
 					let k = K::read(r)?;
 					let v_opt = V::read(r)?;
 					if let Some(v) = v_opt {
@@ -637,7 +666,7 @@ where T: Writeable + Eq + Hash
 {
 	#[inline]
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-		(self.len() as u16).write(w)?;
+		U16Or64(self.len() as u64).write(w)?;
 		for item in self.iter() {
 			item.write(w)?;
 		}
@@ -650,9 +679,9 @@ where T: Readable + Eq + Hash
 {
 	#[inline]
 	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-		let len: u16 = Readable::read(r)?;
-		let mut ret = HashSet::with_capacity(len as usize);
-		for _ in 0..len {
+		let len: U16Or64 = Readable::read(r)?;
+		let mut ret = HashSet::with_capacity(cmp::min(len.0 as usize, MAX_BUF_SIZE / core::mem::size_of::<T>()));
+		for _ in 0..len.0 {
 			if !ret.insert(T::read(r)?) {
 				return Err(DecodeError::InvalidValue)
 			}
@@ -667,7 +696,7 @@ macro_rules! impl_for_vec {
 		impl<$($name : Writeable),*> Writeable for Vec<$ty> {
 			#[inline]
 			fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-				(self.len() as u16).write(w)?;
+				U16Or64(self.len() as u64).write(w)?;
 				for elem in self.iter() {
 					elem.write(w)?;
 				}
@@ -678,9 +707,9 @@ macro_rules! impl_for_vec {
 		impl<$($name : Readable),*> Readable for Vec<$ty> {
 			#[inline]
 			fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-				let len: u16 = Readable::read(r)?;
-				let mut ret = Vec::with_capacity(cmp::min(len as usize, MAX_BUF_SIZE / core::mem::size_of::<$ty>()));
-				for _ in 0..len {
+				let len: U16Or64 = Readable::read(r)?;
+				let mut ret = Vec::with_capacity(cmp::min(len.0 as usize, MAX_BUF_SIZE / core::mem::size_of::<$ty>()));
+				for _ in 0..len.0 {
 					if let Some(val) = MaybeReadable::read(r)? {
 						ret.push(val);
 					}
@@ -694,18 +723,23 @@ macro_rules! impl_for_vec {
 impl Writeable for Vec<u8> {
 	#[inline]
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-		(self.len() as u16).write(w)?;
+		U16Or64(self.len() as u64).write(w)?;
 		w.write_all(&self)
 	}
 }
 
 impl Readable for Vec<u8> {
 	#[inline]
 	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-		let len: u16 = Readable::read(r)?;
-		let mut ret = Vec::with_capacity(len as usize);
-		ret.resize(len as usize, 0);
-		r.read_exact(&mut ret)?;
+		let mut len: U16Or64 = Readable::read(r)?;
+		let mut ret = Vec::new();
+		while len.0 > 0 {
+			let readamt = cmp::min(len.0 as usize, MAX_BUF_SIZE);
+			let readstart = ret.len();
+			ret.resize(readstart + readamt, 0);
+			r.read_exact(&mut ret[readstart..])?;
+			len.0 -= readamt as u64;
+		}
 		Ok(ret)
 	}
 }
@@ -1040,7 +1074,7 @@ impl Readable for () {
 impl Writeable for String {
 	#[inline]
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-		(self.len() as u16).write(w)?;
+		U16Or64(self.len() as u64).write(w)?;
 		w.write_all(self.as_bytes())
 	}
 }