Implement mb_encode_mimeheader using fast text conversion filters

The behavior of the new mb_encode_mimeheader implementation closely
follows the old implementation, except for three points:

• The old implementation was missing a call to the mbfl_convert_filter
  flush function. So it would sometimes truncate the input string just
  before its end.

• The old implementation would drop zero bytes when QPrint-encoding.
  So for example, if you tried to QPrint-encode the UTF-32BE string
  "\x00\x00\x12\x34", its QPrint-encoding would be "=12=34", which
  does not decode to a valid UTF-32BE string. This is now fixed.

• In some rare corner cases, the new implementation will choose to
  Base64-encode or QPrint-encode the input string, where the old
  implementation would have just added newlines to it. Specifically,
  this can happen when there is a non-space ASCII character, followed
  by a large number of ASCII spaces, followed by a non-ASCII character.

The new implementation is around 2.5-8x faster than the old one,
depending on the text encoding and transfer encoding used. Performance
gains are greater with Base64 transfer encoding than with QPrint
transfer encoding; this is not because QPrint-encoding bytes is slow,
but because QPrint-encoded output is much bigger than Base64-encoded
output and takes more lines, so we have to go through the process of
finding the right place to break a line many more times.

Author: alexdowad

ext/mbstring/libmbfl/filters/mbfilter_base64.c

@@ -99,15 +99,13 @@ int mbfl_filt_conv_base64enc(int c, mbfl_convert_filter *filter)
 		filter->cache |= (c & 0xff) << 8;
 	} else {
 		filter->status &= ~0xff;
-		if ((filter->status & MBFL_BASE64_STS_MIME_HEADER) == 0) {
-			n = (filter->status & 0xff00) >> 8;
-			if (n > 72) {
-				CK((*filter->output_function)(0x0d, filter->data));		/* CR */
-				CK((*filter->output_function)(0x0a, filter->data));		/* LF */
-				filter->status &= ~0xff00;
-			}
-			filter->status += 0x400;
+		n = (filter->status & 0xff00) >> 8;
+		if (n > 72) {
+			CK((*filter->output_function)(0x0d, filter->data));		/* CR */
+			CK((*filter->output_function)(0x0a, filter->data));		/* LF */
+			filter->status &= ~0xff00;
 		}
+		filter->status += 0x400;
 		n = filter->cache | (c & 0xff);
 		CK((*filter->output_function)(mbfl_base64_table[(n >> 18) & 0x3f], filter->data));
 		CK((*filter->output_function)(mbfl_base64_table[(n >> 12) & 0x3f], filter->data));
@@ -129,11 +127,9 @@ int mbfl_filt_conv_base64enc_flush(mbfl_convert_filter *filter)
 	filter->cache = 0;
 	/* flush fragments */
 	if (status >= 1) {
-		if ((filter->status & MBFL_BASE64_STS_MIME_HEADER) == 0) {
-			if (len > 72){
-				CK((*filter->output_function)(0x0d, filter->data));		/* CR */
-				CK((*filter->output_function)(0x0a, filter->data));		/* LF */
-			}
+		if (len > 72){
+			CK((*filter->output_function)(0x0d, filter->data));		/* CR */
+			CK((*filter->output_function)(0x0a, filter->data));		/* LF */
 		}
 		CK((*filter->output_function)(mbfl_base64_table[(cache >> 18) & 0x3f], filter->data));
 		CK((*filter->output_function)(mbfl_base64_table[(cache >> 12) & 0x3f], filter->data));

ext/mbstring/libmbfl/filters/mbfilter_qprint.c

@@ -29,7 +29,6 @@
 
 #include "mbfilter.h"
 #include "mbfilter_qprint.h"
-#include "unicode_prop.h"
 
 static size_t mb_qprint_to_wchar(unsigned char **in, size_t *in_len, uint32_t *buf, size_t bufsize, unsigned int *state);
 static void mb_wchar_to_qprint(uint32_t *in, size_t len, mb_convert_buf *buf, bool end);
@@ -96,28 +95,25 @@ int mbfl_filt_conv_qprintenc(int c, mbfl_convert_filter *filter)
 			break;
 		}
 
-		if ((filter->status & MBFL_QPRINT_STS_MIME_HEADER) == 0) {
-			if (s == 0x0a || (s == 0x0d && c != 0x0a)) {	/* line feed */
-				CK((*filter->output_function)(0x0d, filter->data));		/* CR */
-				CK((*filter->output_function)(0x0a, filter->data));		/* LF */
-				filter->status &= ~0xff00;
-				break;
-			} else if (s == 0x0d) {
-				break;
-			}
+		if (s == '\n' || (s == '\r' && c != '\n')) {	/* line feed */
+			CK((*filter->output_function)('\r', filter->data));
+			CK((*filter->output_function)('\n', filter->data));
+			filter->status &= ~0xff00;
+			break;
+		} else if (s == 0x0d) {
+			break;
 		}
 
-		if ((filter->status & MBFL_QPRINT_STS_MIME_HEADER) == 0  && n >= 72) {	/* soft line feed */
-			CK((*filter->output_function)(0x3d, filter->data));		/* '=' */
-			CK((*filter->output_function)(0x0d, filter->data));		/* CR */
-			CK((*filter->output_function)(0x0a, filter->data));		/* LF */
+		if (n >= 72) {	/* soft line feed */
+			CK((*filter->output_function)('=', filter->data));
+			CK((*filter->output_function)('\r', filter->data));
+			CK((*filter->output_function)('\n', filter->data));
 			filter->status &= ~0xff00;
 		}
 
-		if (s <= 0 || s >= 0x80 || s == 0x3d		/* not ASCII or '=' */
-		   || ((filter->status & MBFL_QPRINT_STS_MIME_HEADER) && mime_char_needs_qencode[s])) {
+		if (s <= 0 || s >= 0x80 || s == '=') { /* not ASCII or '=' */
 			/* hex-octet */
-			CK((*filter->output_function)(0x3d, filter->data));		/* '=' */
+			CK((*filter->output_function)('=', filter->data));
 			n = (s >> 4) & 0xf;
 			if (n < 10) {
 				n += 48;		/* '0' */
@@ -132,14 +128,10 @@ int mbfl_filt_conv_qprintenc(int c, mbfl_convert_filter *filter)
 				n += 55;
 			}
 			CK((*filter->output_function)(n, filter->data));
-			if ((filter->status & MBFL_QPRINT_STS_MIME_HEADER) == 0) {
-				filter->status += 0x300;
-			}
+			filter->status += 0x300;
 		} else {
 			CK((*filter->output_function)(s, filter->data));
-			if ((filter->status & MBFL_QPRINT_STS_MIME_HEADER) == 0) {
-				filter->status += 0x100;
-			}
+			filter->status += 0x100;
 		}
 		break;
 	}