Use fast text conversion filters for mb_strpos, mb_stripos, mb_substr, etc

This boosts the performance of mb_strpos, mb_stripos, mb_strrpos,
mb_strripos, mb_strstr, mb_stristr, mb_strrchr, and mb_strrichr when
used on non-UTF-8 strings. mb_substr is also faster.

With UTF-8 input, there is no appreciable difference in performance for
mb_strpos, mb_stripos, mb_strrpos, etc. This is expected, since the only
real difference here (aside from shorter and simpler code) is that the
new text conversion code is used when converting non-UTF-8 input strings
to UTF-8. (This is done because internally, mb_strpos, etc. work only
on UTF-8 text.)

For ASCII, speed is boosted by 30-65%. For other legacy text encodings,
the degree of performance improvement will depend on how slow the
legacy conversion code was.

One other minor, but notable difference is that strings encoded using
UTF-8 variants from Japanese mobile vendors (SoftBank, KDDI, Docomo)
will not undergo encoding conversion but will be processed "as is". It
is expected that this will result in a large performance boost for
such input strings; but realistically, the number of users who work
with such strings is probably minute.

I was not originally planning to include mb_substr in this commit, but
fuzzing of the reimplemented mb_strstr revealed that mb_substr needed
to be reimplemented, too; using the old mbfl_substr, which was based
on the old text conversion filters, in combination with functions which
use the new text conversion filters caused bugs.

The performance boost for mb_substr varies from 10%-500%, depending
on the encoding and input string used.

Author: alexdowad

ext/mbstring/libmbfl/mbfl/mbfilter.c

@@ -429,42 +429,6 @@ const mbfl_encoding *mbfl_identify_encoding(mbfl_string *string, const mbfl_enco
 	return enc;
 }
 
-/*
- *  strlen
- */
-size_t mbfl_strlen(const mbfl_string *string)
-{
-	size_t len = 0;
-	const mbfl_encoding *encoding = string->encoding;
-
-	if (encoding->flag & MBFL_ENCTYPE_SBCS) {
-		len = string->len;
-	} else if (encoding->flag & MBFL_ENCTYPE_WCS2) {
-		len = string->len/2;
-	} else if (encoding->flag & MBFL_ENCTYPE_WCS4) {
-		len = string->len/4;
-	} else if (encoding->mblen_table) {
-		const unsigned char *mbtab = encoding->mblen_table;
-		unsigned char *p = string->val, *e = p + string->len;
-		while (p < e) {
-			p += mbtab[*p];
-			len++;
-		}
-	} else {
-		uint32_t wchar_buf[128];
-		unsigned char *in = string->val;
-		size_t in_len = string->len;
-		unsigned int state = 0;
-
-		while (in_len) {
-			len += encoding->to_wchar(&in, &in_len, wchar_buf, 128, &state);
-		}
-	}
-
-	return len;
-}
-
-
 /*
  *  strpos
  */
@@ -528,136 +492,6 @@ collector_strpos(int c, void* data)
 	return 0;
 }
 
-static const unsigned char *mbfl_find_offset_utf8(
-		const unsigned char *str, const unsigned char *end, ssize_t offset) {
-	if (offset < 0) {
-		const unsigned char *pos = end;
-		while (offset < 0) {
-			if (pos <= str) {
-				return NULL;
-			}
-
-			unsigned char c = *(--pos);
-			if (c < 0x80) {
-				++offset;
-			} else if ((c & 0xc0) != 0x80) {
-				++offset;
-			}
-		}
-		return pos;
-	} else {
-		const unsigned char *u8_tbl = mbfl_encoding_utf8.mblen_table;
-		const unsigned char *pos = str;
-		while (offset-- > 0) {
-			if (pos >= end) {
-				return NULL;
-			}
-			pos += u8_tbl[*pos];
-		}
-		return pos;
-	}
-}
-
-static size_t mbfl_pointer_to_offset_utf8(const unsigned char *start, const unsigned char *pos) {
-	size_t result = 0;
-	while (pos > start) {
-		unsigned char c = *--pos;
-		if (c < 0x80) {
-			++result;
-		} else if ((c & 0xc0) != 0x80) {
-			++result;
-		}
-	}
-	return result;
-}
-
-size_t
-mbfl_strpos(
-    mbfl_string *haystack,
-    mbfl_string *needle,
-    ssize_t offset,
-    int reverse)
-{
-	size_t result;
-	mbfl_string _haystack_u8, _needle_u8;
-	const mbfl_string *haystack_u8, *needle_u8 = NULL;
-	const unsigned char *offset_pointer;
-
-	if (haystack->encoding->no_encoding != mbfl_no_encoding_utf8) {
-		mbfl_string_init(&_haystack_u8);
-		haystack_u8 = mbfl_convert_encoding(haystack, &_haystack_u8, &mbfl_encoding_utf8);
-		if (haystack_u8 == NULL) {
-			result = MBFL_ERROR_ENCODING;
-			goto out;
-		}
-	} else {
-		haystack_u8 = haystack;
-	}
-
-	if (needle->encoding->no_encoding != mbfl_no_encoding_utf8) {
-		mbfl_string_init(&_needle_u8);
-		needle_u8 = mbfl_convert_encoding(needle, &_needle_u8, &mbfl_encoding_utf8);
-		if (needle_u8 == NULL) {
-			result = MBFL_ERROR_ENCODING;
-			goto out;
-		}
-	} else {
-		needle_u8 = needle;
-	}
-
-	offset_pointer = mbfl_find_offset_utf8(
-		haystack_u8->val, haystack_u8->val + haystack_u8->len, offset);
-	if (!offset_pointer) {
-		result = MBFL_ERROR_OFFSET;
-		goto out;
-	}
-
-	result = MBFL_ERROR_NOT_FOUND;
-	if (haystack_u8->len < needle_u8->len) {
-		goto out;
-	}
-
-	const char *found_pos;
-	if (!reverse) {
-		found_pos = zend_memnstr(
-			(const char *) offset_pointer,
-			(const char *) needle_u8->val, needle_u8->len,
-			(const char *) haystack_u8->val + haystack_u8->len);
-	} else {
-		if (offset >= 0) {
-			found_pos = zend_memnrstr(
-				(const char *) offset_pointer,
-				(const char *) needle_u8->val, needle_u8->len,
-				(const char *) haystack_u8->val + haystack_u8->len);
-		} else {
-			size_t needle_len = mbfl_strlen(needle_u8);
-			offset_pointer = mbfl_find_offset_utf8(
-				offset_pointer, haystack_u8->val + haystack_u8->len, needle_len);
-			if (!offset_pointer) {
-				offset_pointer = haystack_u8->val + haystack_u8->len;
-			}
-
-			found_pos = zend_memnrstr(
-				(const char *) haystack_u8->val,
-				(const char *) needle_u8->val, needle_u8->len,
-				(const char *) offset_pointer);
-		}
-	}
-
-	if (found_pos) {
-		result = mbfl_pointer_to_offset_utf8(haystack_u8->val, (const unsigned char *) found_pos);
-	}
-
-out:
-	if (haystack_u8 == &_haystack_u8) {
-		mbfl_string_clear(&_haystack_u8);
-	}
-	if (needle_u8 == &_needle_u8) {
-		mbfl_string_clear(&_needle_u8);
-	}
-	return result;
-}
-
 /*
  *  substr_count
  */
@@ -727,176 +561,6 @@ mbfl_substr_count(
 	return result;
 }
 
-/*
- *  substr
- */
-struct collector_substr_data {
-	mbfl_convert_filter *next_filter;
-	size_t start;
-	size_t stop;
-	size_t output;
-};
-
-static int
-collector_substr(int c, void* data)
-{
-	struct collector_substr_data *pc = (struct collector_substr_data*)data;
-
-	if (pc->output >= pc->stop) {
-		return -1;
-	}
-
-	if (pc->output >= pc->start) {
-		(*pc->next_filter->filter_function)(c, pc->next_filter);
-	}
-
-	pc->output++;
-
-	return 0;
-}
-
-mbfl_string *
-mbfl_substr(
-    mbfl_string *string,
-    mbfl_string *result,
-    size_t from,
-    size_t length)
-{
-	const mbfl_encoding *encoding = string->encoding;
-	size_t n, k, len, start, end;
-	unsigned m;
-	unsigned char *p, *w;
-
-	mbfl_string_init(result);
-	result->encoding = string->encoding;
-
-	if ((encoding->flag & (MBFL_ENCTYPE_SBCS | MBFL_ENCTYPE_WCS2 | MBFL_ENCTYPE_WCS4)) ||
-	   encoding->mblen_table != NULL) {
-		len = string->len;
-		if (encoding->flag & MBFL_ENCTYPE_SBCS) {
-			start = from;
-		} else if (encoding->flag & MBFL_ENCTYPE_WCS2) {
-			start = from*2;
-		} else if (encoding->flag & MBFL_ENCTYPE_WCS4) {
-			start = from*4;
-		} else {
-			const unsigned char *mbtab = encoding->mblen_table;
-			start = 0;
-			n = 0;
-			k = 0;
-			p = string->val;
-			/* search start position */
-			while (k <= from) {
-				start = n;
-				if (n >= len) {
-					break;
-				}
-				m = mbtab[*p];
-				n += m;
-				p += m;
-				k++;
-			}
-		}
-
-		if (length == MBFL_SUBSTR_UNTIL_END) {
-			end = len;
-		} else if (encoding->flag & MBFL_ENCTYPE_SBCS) {
-			end = start + length;
-		} else if (encoding->flag & MBFL_ENCTYPE_WCS2) {
-			end = start + length*2;
-		} else if (encoding->flag & MBFL_ENCTYPE_WCS4) {
-			end = start + length*4;
-		} else {
-			const unsigned char *mbtab = encoding->mblen_table;
-			end = start;
-			n = start;
-			k = 0;
-			p = string->val + start;
-			/* detect end position */
-			while (k <= length) {
-				end = n;
-				if (n >= len) {
-					break;
-				}
-				m = mbtab[*p];
-				n += m;
-				p += m;
-				k++;
-			}
-		}
-
-		if (start > len) {
-			start = len;
-		}
-		if (end > len) {
-			end = len;
-		}
-		if (start > end) {
-			start = end;
-		}
-
-		/* allocate memory and copy */
-		n = end - start;
-		result->len = 0;
-		result->val = w = (unsigned char*)emalloc(n + 1);
-		result->len = n;
-		memcpy(w, string->val + start, n);
-		w[n] = '\0';
-	} else {
-		mbfl_memory_device device;
-		struct collector_substr_data pc;
-		mbfl_convert_filter *decoder;
-		mbfl_convert_filter *encoder;
-
-		if (length == MBFL_SUBSTR_UNTIL_END) {
-			length = mbfl_strlen(string) - from;
-		}
-
-		mbfl_memory_device_init(&device, length + 1, 0);
-		mbfl_string_init(result);
-		result->encoding = string->encoding;
-		/* output code filter */
-		decoder = mbfl_convert_filter_new(
-		    &mbfl_encoding_wchar,
-		    string->encoding,
-		    mbfl_memory_device_output, 0, &device);
-		/* wchar filter */
-		encoder = mbfl_convert_filter_new(
-		    string->encoding,
-		    &mbfl_encoding_wchar,
-		    collector_substr, 0, &pc);
-		if (decoder == NULL || encoder == NULL) {
-			mbfl_convert_filter_delete(encoder);
-			mbfl_convert_filter_delete(decoder);
-			return NULL;
-		}
-		pc.next_filter = decoder;
-		pc.start = from;
-		pc.stop = from + length;
-		pc.output = 0;
-
-		/* feed data */
-		p = string->val;
-		n = string->len;
-		if (p != NULL) {
-			while (n > 0) {
-				if ((*encoder->filter_function)(*p++, encoder) < 0) {
-					break;
-				}
-				n--;
-			}
-		}
-
-		mbfl_convert_filter_flush(encoder);
-		mbfl_convert_filter_flush(decoder);
-		result = mbfl_memory_device_result(&device, result);
-		mbfl_convert_filter_delete(encoder);
-		mbfl_convert_filter_delete(decoder);
-	}
-
-	return result;
-}
-
 /*
  *  strcut
  */

ext/mbstring/libmbfl/mbfl/mbfilter.h

@@ -190,24 +190,11 @@ static inline int mbfl_is_error(size_t len) {
 	return len >= (size_t) -16;
 }
 
-/*
- * strlen
- */
-MBFLAPI extern size_t
-mbfl_strlen(const mbfl_string *string);
-
 #define MBFL_ERROR_NOT_FOUND ((size_t) -1)
 #define MBFL_ERROR_ENCODING ((size_t) -4)
 #define MBFL_ERROR_EMPTY ((size_t) -8)
 #define MBFL_ERROR_OFFSET ((size_t) -16)
 
-/*
- * strpos.
- * Errors: MBFL_ERROR_NOT_FOUND, MBFL_ERROR_ENCODING, MBFL_ERROR_OFFSET
- */
-MBFLAPI extern size_t
-mbfl_strpos(mbfl_string *haystack, mbfl_string *needle, ssize_t offset, int reverse);
-
 /*
  * substr_count
  */
@@ -219,12 +206,6 @@ mbfl_substr_count(mbfl_string *haystack, mbfl_string *needle);
  */
 #define MBFL_SUBSTR_UNTIL_END ((size_t) -1)
 
-/*
- * substr
- */
-MBFLAPI extern mbfl_string *
-mbfl_substr(mbfl_string *string, mbfl_string *result, size_t from, size_t length);
-
 /*
  * strcut
  */