Add test suite for UTF-{7,8,16,32}

Also fix a couple small problems with UTF-32 and UTF-8 support:

- UTF-32 would pass very large codepoints (>= 0x80000000), which are
  not valid.
- UTF-8 would sometimes emit two error marker characters for a single
  bad input byte.

Author: alexdowad

ext/mbstring/libmbfl/filters/mbfilter_utf32.c

@@ -131,7 +131,7 @@ const struct mbfl_convert_vtbl vtbl_wchar_utf32le = {
 
 static int emit_char_if_valid(int n, mbfl_convert_filter *filter)
 {
-	if (n < MBFL_WCSPLANE_UTF32MAX && (n < 0xD800 || n > 0xDFFF)) {
+	if (n >= 0 && n < MBFL_WCSPLANE_UTF32MAX && (n < 0xD800 || n > 0xDFFF)) {
 		CK((*filter->output_function)(n, filter->data));
 	} else {
 		n = (n & MBFL_WCSGROUP_MASK) | MBFL_WCSGROUP_THROUGH;

ext/mbstring/libmbfl/filters/mbfilter_utf8.c

@@ -86,19 +86,11 @@ const struct mbfl_convert_vtbl vtbl_wchar_utf8 = {
 
 int mbfl_filt_put_invalid_char(int c, mbfl_convert_filter *filter)
 {
-	int w;
-	w = c & MBFL_WCSGROUP_MASK;
-	w |= MBFL_WCSGROUP_THROUGH;
-	filter->status = 0;
-	filter->cache = 0;
-	CK((*filter->output_function)(w, filter->data));
+	filter->status = filter->cache = 0;
+	CK((*filter->output_function)((c & MBFL_WCSGROUP_MASK) | MBFL_WCSGROUP_THROUGH, filter->data));
 	return 0;
 }
 
-
-/*
- * UTF-8 => wchar
- */
 int mbfl_filt_conv_utf8_wchar(int c, mbfl_convert_filter *filter)
 {
 	int s, c1;
@@ -131,7 +123,8 @@ int mbfl_filt_conv_utf8_wchar(int c, mbfl_convert_filter *filter)
 			CK((*filter->output_function)(s, filter->data));
 		} else {
 			CK(mbfl_filt_put_invalid_char(filter->cache, filter));
-			goto retry;
+			if (c < 0x80 || (c >= 0xc2 && c <= 0xf4))
+				goto retry;
 		}
 		break;
 	case 0x20: /* 3byte code 2nd char: 0:0xa0-0xbf,D:0x80-9F,1-C,E-F:0x80-0x9f */
@@ -146,7 +139,8 @@ int mbfl_filt_conv_utf8_wchar(int c, mbfl_convert_filter *filter)
 			filter->status++;
 		} else {
 			CK(mbfl_filt_put_invalid_char(filter->cache, filter));
-			goto retry;
+			if (c < 0x80 || (c >= 0xc2 && c <= 0xf4))
+				goto retry;
 		}
 		break;
 	case 0x30: /* 4byte code 2nd char: 0:0x90-0xbf,1-3:0x80-0xbf,4:0x80-0x8f */
@@ -161,7 +155,8 @@ int mbfl_filt_conv_utf8_wchar(int c, mbfl_convert_filter *filter)
 			filter->status++;
 		} else {
 			CK(mbfl_filt_put_invalid_char(filter->cache, filter));
-			goto retry;
+			if (c < 0x80 || (c >= 0xc2 && c <= 0xf4))
+				goto retry;
 		}
 		break;
 	case 0x31: /* 4byte code 3rd char: 0x80-0xbf */
@@ -170,7 +165,8 @@ int mbfl_filt_conv_utf8_wchar(int c, mbfl_convert_filter *filter)
 			filter->status++;
 		} else {
 			CK(mbfl_filt_put_invalid_char(filter->cache, filter));
-			goto retry;
+			if (c < 0x80 || (c >= 0xc2 && c <= 0xf4))
+				goto retry;
 		}
 		break;
 	default:
@@ -183,27 +179,21 @@ int mbfl_filt_conv_utf8_wchar(int c, mbfl_convert_filter *filter)
 
 int mbfl_filt_conv_utf8_wchar_flush(mbfl_convert_filter *filter)
 {
-	int status, cache;
-
-	status = filter->status;
-	cache = filter->cache;
+	int status = filter->status, cache = filter->cache;
 
-	filter->status = 0;
-	filter->cache = 0;
+	filter->status = filter->cache = 0;
 
-	if (status != 0) {
+	if (status) {
 		CK(mbfl_filt_put_invalid_char(cache, filter));
 	}
 
-	if (filter->flush_function != NULL) {
+	if (filter->flush_function) {
 		(*filter->flush_function)(filter->data);
 	}
+
 	return 0;
 }
 
-/*
- * wchar => UTF-8
- */
 int mbfl_filt_conv_wchar_utf8(int c, mbfl_convert_filter *filter)
 {
 	if (c >= 0 && c < 0x110000) {

ext/mbstring/tests/illformed_utf_sequences.phpt

@@ -22,28 +22,28 @@ var_dump(chk_enc("\x31\x32\x33", 0));
 var_dump(chk_enc("\x41\x42\x43", 0));
 var_dump(chk_enc("\xc0\xb1\xc0\xb2\xc0\xb3", 6));
 var_dump(chk_enc("\xc1\x81\xc1\x82\xc1\x83", 6));
-var_dump(chk_enc("\xe0\x80\xb1\xe0\x80\xb2\xe0\x80\xb3", 9));
-var_dump(chk_enc("\xe0\x81\x81\xe0\x81\x82\xe0\x81\x83", 9));
-var_dump(chk_enc("\xf0\x80\x80\xb1\xf0\x80\x80\xb2\xf0\x80\x80\xb3", 12));
-var_dump(chk_enc("\xf0\x80\x81\x81\xf0\x80\x81\x82\xf0\x81\x83", 11));
+var_dump(chk_enc("\xe0\x80\xb1\xe0\x80\xb2\xe0\x80\xb3", 6));
+var_dump(chk_enc("\xe0\x81\x81\xe0\x81\x82\xe0\x81\x83", 6));
+var_dump(chk_enc("\xf0\x80\x80\xb1\xf0\x80\x80\xb2\xf0\x80\x80\xb3", 9));
+var_dump(chk_enc("\xf0\x80\x81\x81\xf0\x80\x81\x82\xf0\x81\x83", 8));
 var_dump(chk_enc("\xf8\x80\x80\x80\xb1\xf8\x80\x80\x80\xb2\xf8\x80\x80\x80\xb3", 15));
 var_dump(chk_enc("\xf8\x80\x80\x81\x81\xf8\x80\x80\x81\x82\xf8\x80\x80\x81\x83", 15));
 var_dump(chk_enc("\xfc\x80\x80\x80\x80\xb1\xfc\x80\x80\x80\x80\xb2\xfc\x80\x80\x80\x80\xb3", 18));
 var_dump(chk_enc("\xfc\x80\x80\x80\x81\x81\xfc\x80\x80\x80\x81\x82\xfc\x80\x80\x80\x81\x83", 18));
 
 var_dump(chk_enc("\xc2\xa2\xc2\xa3\xc2\xa5", 0));
-var_dump(chk_enc("\xe0\x82\xa2\xe0\x82\xa3\xe0\x82\xa5", 9));
-var_dump(chk_enc("\xf0\x80\x82\xa2\xf0\x80\x82\xa3\xf0\x80\x82\xa5", 12));
+var_dump(chk_enc("\xe0\x82\xa2\xe0\x82\xa3\xe0\x82\xa5", 6));
+var_dump(chk_enc("\xf0\x80\x82\xa2\xf0\x80\x82\xa3\xf0\x80\x82\xa5", 9));
 var_dump(chk_enc("\xf8\x80\x80\x82\xa2\xf8\x80\x80\x82\xa3\xf8\x80\x80\x82\xa5", 15));
 var_dump(chk_enc("\xfc\x80\x80\x80\x82\xa2\xfc\x80\x80\x80\x82\xa3\xfc\x80\x80\x80\x82\xa5", 18));
 
 var_dump(chk_enc("\xc1\xbf", 2));
 var_dump(chk_enc("\xc2\x80", 0));
 var_dump(chk_enc("\xdf\xbf", 0));
-var_dump(chk_enc("\xe0\x9f\xff", 3));
+var_dump(chk_enc("\xe0\x9f\xff", 2));
 var_dump(chk_enc("\xe0\xa0\x80", 2));
 var_dump(chk_enc("\xef\xbf\xbf", 0));
-var_dump(chk_enc("\xf0\x8f\xbf\xbf", 4));
+var_dump(chk_enc("\xf0\x8f\xbf\xbf", 3));
 var_dump(chk_enc("\xf0\x90\x80\x80", 0));
 var_dump(chk_enc("\xf7\xbf\xbf\xbf", 4));
 var_dump(chk_enc("\xf8\x87\xbf\xbf\xbf", 5));
@@ -58,7 +58,7 @@ echo "UTF-8 and surrogates area\n";
 $out = '';
 $cnt = 0;
 for ($i = 0xd7ff; $i <= 0xe000; ++$i) {
-    $s = chk_enc(pack('C3', 0xe0 | ($i >> 12), 0x80 | ($i >> 6) & 0x3f, 0x80 | $i & 0x3f), 3);
+    $s = chk_enc(pack('C3', 0xe0 | ($i >> 12), 0x80 | ($i >> 6) & 0x3f, 0x80 | $i & 0x3f), 2);
     if ($s === false) {
         $cnt++;
     } else {