SJIS-2004 encoding conversion: handle invalid (or truncated) 2nd byte for Kanji correctly

If the 2nd byte of a 2-byte character is invalid, then mb_substitute_character()
should be respected. Instead, what mbstring was doing was 'swallowing' the
first byte, then emitting the 2nd byte as if it was an ASCII character.

Likewise, if the 2nd byte is missing, instead of just keeping quiet, report an
illegal character as specified by mb_substitute_character().

Author: alexdowad

ext/mbstring/libmbfl/filters/mbfilter_euc_jp_2004.c

@@ -62,6 +62,6 @@ const struct mbfl_convert_vtbl vtbl_wchar_eucjp2004 = {
 	mbfl_filt_conv_common_ctor,
 	NULL,
 	mbfl_filt_conv_wchar_jis2004,
-	mbfl_filt_conv_jis2004_flush,
+	mbfl_filt_conv_wchar_jis2004_flush,
 	NULL,
 };

ext/mbstring/libmbfl/filters/mbfilter_iso2022jp_2004.c

@@ -62,6 +62,6 @@ const struct mbfl_convert_vtbl vtbl_wchar_2022jp_2004 = {
 	mbfl_filt_conv_common_ctor,
 	NULL,
 	mbfl_filt_conv_wchar_jis2004,
-	mbfl_filt_conv_jis2004_flush,
+	mbfl_filt_conv_wchar_jis2004_flush,
 	NULL,
 };

ext/mbstring/libmbfl/filters/mbfilter_sjis_2004.c

@@ -57,7 +57,7 @@ const struct mbfl_convert_vtbl vtbl_sjis2004_wchar = {
 	mbfl_filt_conv_common_ctor,
 	NULL,
 	mbfl_filt_conv_jis2004_wchar,
-	mbfl_filt_conv_common_flush,
+	mbfl_filt_conv_jis2004_wchar_flush,
 	NULL,
 };
 
@@ -67,7 +67,7 @@ const struct mbfl_convert_vtbl vtbl_wchar_sjis2004 = {
 	mbfl_filt_conv_common_ctor,
 	NULL,
 	mbfl_filt_conv_wchar_jis2004,
-	mbfl_filt_conv_jis2004_flush,
+	mbfl_filt_conv_wchar_jis2004_flush,
 	NULL,
 };
 
@@ -202,6 +202,9 @@ mbfl_filt_conv_jis2004_wchar(int c, mbfl_convert_filter *filter)
 		} else if (filter->from->no_encoding == mbfl_no_encoding_sjis2004) {
 			if (c >= 0x40 && c <= 0xfc && c != 0x7f) {
 				SJIS_DECODE(c1, c, s1, s2);
+			} else {
+				CK((*filter->output_function)(c | MBFL_WCSGROUP_THROUGH, filter->data));
+				break;
 			}
 		} else {
 			s1 = c1;
@@ -471,6 +474,14 @@ mbfl_filt_conv_jis2004_wchar(int c, mbfl_convert_filter *filter)
 	return c;
 }
 
+int mbfl_filt_conv_jis2004_wchar_flush(mbfl_convert_filter *filter)
+{
+	if (filter->status & 0xF) {
+		CK((*filter->output_function)(filter->cache | MBFL_WCSGROUP_THROUGH, filter->data));
+	}
+	return 0;
+}
+
 int
 mbfl_filt_conv_wchar_jis2004(int c, mbfl_convert_filter *filter) {
 	int k;
@@ -665,7 +676,7 @@ mbfl_filt_conv_wchar_jis2004(int c, mbfl_convert_filter *filter) {
 }
 
 int
-mbfl_filt_conv_jis2004_flush(mbfl_convert_filter *filter)
+mbfl_filt_conv_wchar_jis2004_flush(mbfl_convert_filter *filter)
 {
 	int k, c1, c2, s1, s2;
 

ext/mbstring/libmbfl/filters/mbfilter_sjis_2004.h

@@ -39,7 +39,8 @@ extern const struct mbfl_convert_vtbl vtbl_wchar_sjis2004;
 int mbfl_filt_conv_jis2004_wchar(int c, mbfl_convert_filter *filter);
 int mbfl_filt_conv_wchar_jis2004(int c, mbfl_convert_filter *filter);
 
-int mbfl_filt_conv_jis2004_flush(mbfl_convert_filter *filter);
+int mbfl_filt_conv_wchar_jis2004_flush(mbfl_convert_filter *filter);
+int mbfl_filt_conv_jis2004_wchar_flush(mbfl_convert_filter *filter);
 
 #endif /* MBFL_MBFILTER_SJIS_2004_H */