changeset:   94711:af8089217cc6
branch:      2.7
parent:      94708:7052206ad381
user:        Serhiy Storchaka <storchaka@gmail.com>
date:        Sat Feb 21 01:19:17 2015 +0200
files:       Lib/test/test_multibytecodec.py Misc/NEWS Modules/cjkcodecs/multibytecodec.c
description:
Issue #23215: Multibyte codecs with custom error handlers that ignores errors
consumed too much memory and raised SystemError or MemoryError.
Original patch by Aleksi Torhamo.


diff -r 7052206ad381 -r af8089217cc6 Lib/test/test_multibytecodec.py
--- a/Lib/test/test_multibytecodec.py	Sat Feb 21 00:34:20 2015 +0200
+++ b/Lib/test/test_multibytecodec.py	Sat Feb 21 01:19:17 2015 +0200
@@ -43,6 +43,13 @@
         self.assertRaises(IndexError, dec,
                           'apple\x92ham\x93spam', 'test.cjktest')
 
+    def test_errorcallback_custom_ignore(self):
+        # Issue #23215: MemoryError with custom error handlers and multibyte codecs
+        data = 100 * unichr(0xdc00)
+        codecs.register_error("test.ignore", codecs.ignore_errors)
+        for enc in ALL_CJKENCODINGS:
+            self.assertEqual(data.encode(enc, "test.ignore"), b'')
+
     def test_codingspec(self):
         for enc in ALL_CJKENCODINGS:
             code = '# coding: {}\n'.format(enc)
diff -r 7052206ad381 -r af8089217cc6 Misc/NEWS
--- a/Misc/NEWS	Sat Feb 21 00:34:20 2015 +0200
+++ b/Misc/NEWS	Sat Feb 21 01:19:17 2015 +0200
@@ -18,6 +18,10 @@
 Library
 -------
 
+- Issue #23215: Multibyte codecs with custom error handlers that ignores errors
+  consumed too much memory and raised SystemError or MemoryError.
+  Original patch by Aleksi Torhamo.
+
 - Issue #5700: io.FileIO() called flush() after closing the file.
   flush() was not called in close() if closefd=False.
 
diff -r 7052206ad381 -r af8089217cc6 Modules/cjkcodecs/multibytecodec.c
--- a/Modules/cjkcodecs/multibytecodec.c	Sat Feb 21 00:34:20 2015 +0200
+++ b/Modules/cjkcodecs/multibytecodec.c	Sat Feb 21 01:19:17 2015 +0200
@@ -170,8 +170,10 @@
     orgsize = PyString_GET_SIZE(buf->outobj);
     incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
 
-    if (orgsize > PY_SSIZE_T_MAX - incsize)
+    if (orgsize > PY_SSIZE_T_MAX - incsize) {
+        PyErr_NoMemory();
         return -1;
+    }
 
     if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)
         return -1;
@@ -182,11 +184,11 @@
 
     return 0;
 }
-#define REQUIRE_ENCODEBUFFER(buf, s) {                                  \
-    if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end)             \
+#define REQUIRE_ENCODEBUFFER(buf, s) do {                               \
+    if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf)             \
         if (expand_encodebuffer(buf, s) == -1)                          \
             goto errorexit;                                             \
-}
+} while(0)
 
 static int
 expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)
@@ -205,11 +207,11 @@
 
     return 0;
 }
-#define REQUIRE_DECODEBUFFER(buf, s) {                                  \
-    if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end)             \
+#define REQUIRE_DECODEBUFFER(buf, s) do {                               \
+    if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf)             \
         if (expand_decodebuffer(buf, s) == -1)                          \
             goto errorexit;                                             \
-}
+} while(0)
 
 
 /**
@@ -327,10 +329,11 @@
     }
 
     retstrsize = PyString_GET_SIZE(retstr);
-    REQUIRE_ENCODEBUFFER(buf, retstrsize);
-
-    memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);
-    buf->outbuf += retstrsize;
+    if (retstrsize > 0) {
+        REQUIRE_ENCODEBUFFER(buf, retstrsize);
+        memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);
+        buf->outbuf += retstrsize;
+    }
 
     newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
     if (newpos < 0 && !PyErr_Occurred())