Fix new conversion filters for mobile SJIS variants ('0' at end of buffer)

Previously, I had adjusted this code so that if a character which could
be part of a special Docomo/Softbank/KDDI 'keypad' emoji appeared at
the end of one buffer, and the 'keypad' character appeared at the
beginning of the next, they would still be combined. However, this
broke the handling of such a character appearing at the end of one
buffer, and a character which is NOT 'keypad' appearing at the
beginning of the next.

This was found while fuzzing the new implementation of
mb_decode_numericentity.

Author: alexdowad

ext/mbstring/libmbfl/filters/mbfilter_sjis_mobile.c

@@ -929,7 +929,7 @@ static void mb_wchar_to_sjis_docomo(uint32_t *in, size_t len, mb_convert_buf *bu
 {
 	unsigned char *out, *limit;
 	MB_CONVERT_BUF_LOAD(buf, out, limit);
-	MB_CONVERT_BUF_ENSURE(buf, out, limit, len);
+	MB_CONVERT_BUF_ENSURE(buf, out, limit, len + (buf->state ? 1 : 0));
 
 	uint32_t w;
 	unsigned int s = 0;
@@ -939,14 +939,15 @@ static void mb_wchar_to_sjis_docomo(uint32_t *in, size_t len, mb_convert_buf *bu
 		w = buf->state;
 		buf->state = 0;
 		if (len) {
-			goto process_possible_keypad;
+			goto reprocess_wchar;
 		} else {
 			goto emit_output;
 		}
 	}
 
 	while (len--) {
 		w = *in++;
+reprocess_wchar:
 		s = 0;
 
 		if (w >= ucs_a1_jis_table_min && w < ucs_a1_jis_table_max) {
@@ -1018,7 +1019,6 @@ static void mb_wchar_to_sjis_docomo(uint32_t *in, size_t len, mb_convert_buf *bu
 					break;
 				}
 			}
-process_possible_keypad: ;
 			uint32_t w2 = *in++; len--;
 			if (w2 == 0x20E3) {
 				if (w == '#') {
@@ -1160,7 +1160,7 @@ static void mb_wchar_to_sjis_kddi(uint32_t *in, size_t len, mb_convert_buf *buf,
 {
 	unsigned char *out, *limit;
 	MB_CONVERT_BUF_LOAD(buf, out, limit);
-	MB_CONVERT_BUF_ENSURE(buf, out, limit, len);
+	MB_CONVERT_BUF_ENSURE(buf, out, limit, len + (buf->state ? 1 : 0));
 
 	uint32_t w;
 	unsigned int s = 0;
@@ -1169,18 +1169,15 @@ static void mb_wchar_to_sjis_kddi(uint32_t *in, size_t len, mb_convert_buf *buf,
 		w = buf->state;
 		buf->state = 0;
 		if (len) {
-			if (w >= NFLAGS('A')) {
-				goto process_possible_flag;
-			} else {
-				goto process_possible_keypad;
-			}
+			goto reprocess_wchar;
 		} else {
 			goto emit_output;
 		}
 	}
 
 	while (len--) {
 		w = *in++;
+reprocess_wchar:
 		s = 0;
 
 		if (w >= ucs_a1_jis_table_min && w < ucs_a1_jis_table_max) {
@@ -1246,7 +1243,6 @@ static void mb_wchar_to_sjis_kddi(uint32_t *in, size_t len, mb_convert_buf *buf,
 					break;
 				}
 			}
-process_possible_keypad: ;
 			uint32_t w2 = *in++; len--;
 			if (w2 == 0x20E3) {
 				if (w == '#') {
@@ -1271,7 +1267,6 @@ process_possible_keypad: ;
 				}
 				break;
 			}
-process_possible_flag: ;
 			uint32_t w2 = *in++; len--;
 			if (w2 >= NFLAGS('B') && w2 <= NFLAGS('U')) { /* B for GB, U for RU */
 				for (int i = 0; i < 10; i++) {
@@ -1472,7 +1467,7 @@ static void mb_wchar_to_sjis_sb(uint32_t *in, size_t len, mb_convert_buf *buf, b
 {
 	unsigned char *out, *limit;
 	MB_CONVERT_BUF_LOAD(buf, out, limit);
-	MB_CONVERT_BUF_ENSURE(buf, out, limit, len);
+	MB_CONVERT_BUF_ENSURE(buf, out, limit, len + (buf->state ? 1 : 0));
 
 	uint32_t w;
 	unsigned int s = 0;
@@ -1481,18 +1476,15 @@ static void mb_wchar_to_sjis_sb(uint32_t *in, size_t len, mb_convert_buf *buf, b
 		w = buf->state;
 		buf->state = 0;
 		if (len) {
-			if (w >= NFLAGS('A')) {
-				goto process_possible_flag;
-			} else {
-				goto process_possible_keypad;
-			}
+			goto reprocess_wchar;
 		} else {
 			goto emit_output;
 		}
 	}
 
 	while (len--) {
 		w = *in++;
+reprocess_wchar:
 		s = 0;
 
 		if (w >= ucs_a1_jis_table_min && w < ucs_a1_jis_table_max) {
@@ -1558,7 +1550,6 @@ static void mb_wchar_to_sjis_sb(uint32_t *in, size_t len, mb_convert_buf *buf, b
 					break;
 				}
 			}
-process_possible_keypad: ;
 			uint32_t w2 = *in++; len--;
 			if (w2 == 0x20E3) {
 				if (w == '#') {
@@ -1577,13 +1568,12 @@ process_possible_keypad: ;
 				if (end) {
 					MB_CONVERT_ERROR(buf, out, limit, w, mb_wchar_to_sjis_sb);
 				} else {
-					/* Reprocess `w` when this function is called again with another buffer
-					 * of wchars */
+					/* Reprocess `w` when this function is called again with
+					 * another buffer of wchars */
 					buf->state = w;
 				}
 				break;
 			}
-process_possible_flag: ;
 			uint32_t w2 = *in++; len--;
 			if (w2 >= NFLAGS('B') && w2 <= NFLAGS('U')) { /* B for GB, U for RU */
 				for (int i = 0; i < 10; i++) {

ext/mbstring/tests/sjis_mobile_encodings.phpt

@@ -324,6 +324,20 @@ for ($i = 0; $i <= 256; $i++) {
   convertValidString(str_repeat("\x00a", $i) . "\x00\x30\x20\xE3", str_repeat('a', $i) . "\xF7\xC5", 'UTF-16BE', 'SJIS-Mobile#SOFTBANK');
 }
 
+// Regression test for 0-9 appearing at end of one buffer and U+203E NOT appearing
+// at the beginning of the next
+for ($i = 0; $i <= 256; $i++) {
+  convertValidString(str_repeat("\x000", $i), str_repeat('0', $i), 'UTF-16BE', 'SJIS-Mobile#DOCOMO');
+  convertValidString(str_repeat("\x000", $i), str_repeat('0', $i), 'UTF-16BE', 'SJIS-Mobile#KDDI');
+  convertValidString(str_repeat("\x000", $i), str_repeat('0', $i), 'UTF-16BE', 'SJIS-Mobile#SOFTBANK');
+}
+
+// Regression test for not making enough space in output buffer when 0-9 appeared
+// at the end of one buffer and was re-processed together with the next
+// This crazy-looking string was found by a fuzzer
+$str = "\x04\xff\x930\x00\xffUTF7~'F\x00A\x00\xffA\x0018030@\x00[\x1b\$EEEEE\x5C\x80(8~\x00F\x00zgb-18030$\x008~\x00F\x00z-gb-18EUC_JP-2004\x00z-g0\x0018030\x00b-18030$\x008~\x00F\x00z-gb-18EUC_JP-2004\x00z-g0\x0018030\x00";
+mb_convert_encoding($str, 'SJIS-Mobile#SOFTBANK', 'SJIS-Mobile#SOFTBANK');
+
 ?>
 --EXPECT--
 SJIS-Mobile#DOCOMO verification and conversion works on all valid characters