Strict conversion of UHC text to Unicode

Previously, mbstring would accept a lot of things which were not valid
UHC text. No more.

- Don't allow single-byte control characters to appear where the 2nd
  byte of a multi-byte character should be.
- Validate that the 2nd byte of a multi-byte character is in the
  expected range.
- Treat it as an error if a multi-byte character is truncated.

Also add a test suite to confirm that UHC conversion (both to and from
Unicode) works according to spec.

Author: alexdowad

ext/mbstring/libmbfl/filters/mbfilter_uhc.c

@@ -36,6 +36,8 @@
 #define UNICODE_TABLE_UHC_DEF
 #include "unicode_table_uhc.h"
 
+static int mbfl_filt_conv_uhc_wchar_flush(mbfl_convert_filter *filter);
+
 static const unsigned char mblen_table_uhc[] = { /* 0x81-0xFE */
   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -74,7 +76,7 @@ const struct mbfl_convert_vtbl vtbl_uhc_wchar = {
 	mbfl_filt_conv_common_ctor,
 	NULL,
 	mbfl_filt_conv_uhc_wchar,
-	mbfl_filt_conv_common_flush,
+	mbfl_filt_conv_uhc_wchar_flush,
 	NULL,
 };
 
@@ -99,7 +101,7 @@ mbfl_filt_conv_uhc_wchar(int c, mbfl_convert_filter *filter)
 	case 0:
 		if (c >= 0 && c < 0x80) { /* latin */
 			CK((*filter->output_function)(c, filter->data));
-		} else if (c > 0x80 && c < 0xff && c != 0xc9) { /* dbcs lead byte */
+		} else if (c > 0x80 && c < 0xfe && c != 0xc9) { /* dbcs lead byte */
 			filter->status = 1;
 			filter->cache = c;
 		} else {
@@ -113,23 +115,23 @@ mbfl_filt_conv_uhc_wchar(int c, mbfl_convert_filter *filter)
 		filter->status = 0;
 		c1 = filter->cache;
 
-		if (c1 >= 0x81 && c1 <= 0xa0) {
+		if (c1 >= 0x81 && c1 <= 0xa0 && c >= 0x41 && c <= 0xfe) {
 			w = (c1 - 0x81)*190 + (c - 0x41);
 			if (w >= 0 && w < uhc1_ucs_table_size) {
 				flag = 1;
 				w = uhc1_ucs_table[w];
 			} else {
 				w = 0;
 			}
-		} else if (c1 >= 0xa1 && c1 <= 0xc6) {
+		} else if (c1 >= 0xa1 && c1 <= 0xc6 && c >= 0x41 && c <= 0xfe) {
 			w = (c1 - 0xa1)*190 + (c - 0x41);
 			if (w >= 0 && w < uhc2_ucs_table_size) {
 				flag = 2;
 				w = uhc2_ucs_table[w];
 			} else {
 				w = 0;
 			}
-		} else if (c1 >= 0xc7 && c1 <= 0xfe) {
+		} else if (c1 >= 0xc7 && c1 < 0xfe && c >= 0xa1 && c <= 0xfe) {
 			w = (c1 - 0xc7)*94 + (c - 0xa1);
 			if (w >= 0 && w < uhc3_ucs_table_size) {
 				flag = 3;
@@ -147,14 +149,10 @@ mbfl_filt_conv_uhc_wchar(int c, mbfl_convert_filter *filter)
 			}
 			CK((*filter->output_function)(w, filter->data));
 		} else {
-			if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */
-				CK((*filter->output_function)(c, filter->data));
-			} else {
-				w = (c1 << 8) | c;
-				w &= MBFL_WCSGROUP_MASK;
-				w |= MBFL_WCSGROUP_THROUGH;
-				CK((*filter->output_function)(w, filter->data));
-			}
+			w = (c1 << 8) | c;
+			w &= MBFL_WCSGROUP_MASK;
+			w |= MBFL_WCSGROUP_THROUGH;
+			CK((*filter->output_function)(w, filter->data));
 		}
 		break;
 
@@ -186,6 +184,7 @@ mbfl_filt_conv_wchar_uhc(int c, mbfl_convert_filter *filter)
 	} else if (c >= ucs_r2_uhc_table_min && c < ucs_r2_uhc_table_max) {
 		s = ucs_r2_uhc_table[c - ucs_r2_uhc_table_min];
 	}
+
 	if (s <= 0) {
 		c1 = c & ~MBFL_WCSPLANE_MASK;
 		if (c1 == MBFL_WCSPLANE_UHC) {
@@ -197,6 +196,7 @@ mbfl_filt_conv_wchar_uhc(int c, mbfl_convert_filter *filter)
 			s = -1;
 		}
 	}
+
 	if (s >= 0) {
 		if (s < 0x80) { /* latin */
 			CK((*filter->output_function)(s, filter->data));
@@ -210,3 +210,17 @@ mbfl_filt_conv_wchar_uhc(int c, mbfl_convert_filter *filter)
 
 	return c;
 }
+
+static int mbfl_filt_conv_uhc_wchar_flush(mbfl_convert_filter *filter)
+{
+	if (filter->status == 1) {
+		/* 2-byte character was truncated */
+		CK((*filter->output_function)(filter->cache | MBFL_WCSGROUP_THROUGH, filter->data));
+	}
+
+	if (filter->flush_function) {
+		(*filter->flush_function)(filter->data);
+	}
+
+	return 0;
+}