[2.7] [3.5] bpo-30070: Fixed leaks and crashes in errors handling in the parser module. (GH-1131). (GH-1185) (#1189)

(cherry picked from commit a79f4c2).
(cherry picked from commit 952a05e)

Author: serhiy-storchaka

Lib/test/test_parser.py

@@ -1,4 +1,6 @@
+import copy
 import parser
+import pickle
 import unittest
 import sys
 import struct
@@ -314,6 +316,52 @@ def test_junk(self):
         # not even remotely valid:
         self.check_bad_tree((1, 2, 3), "<junk>")
 
+    def test_illegal_terminal(self):
+        tree = \
+            (257,
+             (267,
+              (268,
+               (269,
+                (274,
+                 (1,))),
+               (4, ''))),
+             (4, ''),
+             (0, ''))
+        self.check_bad_tree(tree, "too small items in terminal node")
+        tree = \
+            (257,
+             (267,
+              (268,
+               (269,
+                (274,
+                 (1, u'pass'))),
+               (4, ''))),
+             (4, ''),
+             (0, ''))
+        self.check_bad_tree(tree, "non-string second item in terminal node")
+        tree = \
+            (257,
+             (267,
+              (268,
+               (269,
+                (274,
+                 (1, 'pass', '0', 0))),
+               (4, ''))),
+             (4, ''),
+             (0, ''))
+        self.check_bad_tree(tree, "non-integer third item in terminal node")
+        tree = \
+            (257,
+             (267,
+              (268,
+               (269,
+                (274,
+                 (1, 'pass', 0, 0))),
+               (4, ''))),
+             (4, ''),
+             (0, ''))
+        self.check_bad_tree(tree, "too many items in terminal node")
+
     def test_illegal_yield_1(self):
         # Illegal yield statement: def f(): return 1; yield 1
         tree = \
@@ -541,6 +589,18 @@ def test_missing_import_source(self):
              (4, ''), (0, ''))
         self.check_bad_tree(tree, "from import a")
 
+    def test_illegal_encoding(self):
+        # Illegal encoding declaration
+        tree = \
+            (339,
+             (257, (0, '')))
+        self.check_bad_tree(tree, "missed encoding")
+        tree = \
+            (339,
+             (257, (0, '')),
+              u'iso-8859-1')
+        self.check_bad_tree(tree, "non-string encoding")
+
 
 class CompileTestCase(unittest.TestCase):
 
@@ -602,6 +662,21 @@ def test_trigger_memory_error(self):
 class STObjectTestCase(unittest.TestCase):
     """Test operations on ST objects themselves"""
 
+    def test_copy_pickle(self):
+        sts = [
+            parser.expr('2 + 3'),
+            parser.suite('x = 2; y = x + 3'),
+            parser.expr('list(x**3 for x in range(20))')
+        ]
+        for st in sts:
+            st_copy = copy.copy(st)
+            self.assertEqual(st_copy.totuple(), st.totuple())
+            st_copy = copy.deepcopy(st)
+            self.assertEqual(st_copy.totuple(), st.totuple())
+            for proto in range(pickle.HIGHEST_PROTOCOL+1):
+                st_copy = pickle.loads(pickle.dumps(st, proto))
+                self.assertEqual(st_copy.totuple(), st.totuple())
+
     check_sizeof = support.check_sizeof
 
     @support.cpython_only

Misc/NEWS

@@ -42,6 +42,8 @@ Extension Modules
 Library
 -------
 
+- bpo-30070: Fixed leaks and crashes in errors handling in the parser module.
+
 - bpo-30061: Fixed crashes in IOBase methods next() and readlines() when
   readline() or next() respectively return non-sizeable object.
   Fixed possible other errors caused by not checking results of PyObject_Size(),

Modules/parsermodule.c

@@ -740,6 +740,9 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
     Py_ssize_t i;
     int  err;
 
+    if (len < 0) {
+        return NULL;
+    }
     for (i = 1; i < len; ++i) {
         /* elem must always be a sequence, however simple */
         PyObject* elem = PySequence_GetItem(tuple, i);
@@ -766,67 +769,83 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
             PyErr_SetObject(parser_error, err);
             Py_XDECREF(err);
             Py_XDECREF(elem);
-            return (0);
+            return NULL;
         }
         if (ISTERMINAL(type)) {
             Py_ssize_t len = PyObject_Size(elem);
             PyObject *temp;
 
             if ((len != 2) && (len != 3)) {
                 err_string("terminal nodes must have 2 or 3 entries");
-                return 0;
+                Py_DECREF(elem);
+                return NULL;
             }
             temp = PySequence_GetItem(elem, 1);
-            if (temp == NULL)
-                return 0;
+            if (temp == NULL) {
+                Py_DECREF(elem);
+                return NULL;
+            }
             if (!PyString_Check(temp)) {
                 PyErr_Format(parser_error,
                              "second item in terminal node must be a string,"
                              " found %s",
                              Py_TYPE(temp)->tp_name);
                 Py_DECREF(temp);
-                return 0;
+                Py_DECREF(elem);
+                return NULL;
             }
             if (len == 3) {
                 PyObject *o = PySequence_GetItem(elem, 2);
-                if (o != NULL) {
-                    if (PyInt_Check(o))
-                        *line_num = PyInt_AS_LONG(o);
-                    else {
-                        PyErr_Format(parser_error,
-                                     "third item in terminal node must be an"
-                                     " integer, found %s",
-                                     Py_TYPE(temp)->tp_name);
-                        Py_DECREF(o);
-                        Py_DECREF(temp);
-                        return 0;
-                    }
+                if (o == NULL) {
+                    Py_DECREF(temp);
+                    Py_DECREF(elem);
+                    return NULL;
+                }
+                if (PyInt_Check(o))
+                    *line_num = PyInt_AS_LONG(o);
+                else {
+                    PyErr_Format(parser_error,
+                                 "third item in terminal node must be an"
+                                 " integer, found %s",
+                                 Py_TYPE(temp)->tp_name);
                     Py_DECREF(o);
+                    Py_DECREF(temp);
+                    Py_DECREF(elem);
+                    return NULL;
                 }
+                Py_DECREF(o);
             }
             len = PyString_GET_SIZE(temp) + 1;
             strn = (char *)PyObject_MALLOC(len);
-            if (strn != NULL)
-                (void) memcpy(strn, PyString_AS_STRING(temp), len);
+            if (strn == NULL) {
+                Py_DECREF(temp);
+                Py_DECREF(elem);
+                PyErr_NoMemory();
+                return NULL;
+            }
+            (void) memcpy(strn, PyString_AS_STRING(temp), len);
             Py_DECREF(temp);
         }
         else if (!ISNONTERMINAL(type)) {
             /*
              *  It has to be one or the other; this is an error.
              *  Raise an exception.
              */
-            PyObject *err = Py_BuildValue("os", elem, "unknown node type.");
+            PyObject *err = Py_BuildValue("Os", elem, "unknown node type.");
             PyErr_SetObject(parser_error, err);
             Py_XDECREF(err);
-            Py_XDECREF(elem);
-            return (0);
+            Py_DECREF(elem);
+            return NULL;
         }
         err = PyNode_AddChild(root, type, strn, *line_num, 0);
         if (err == E_NOMEM) {
+            Py_DECREF(elem);
             PyObject_FREE(strn);
-            return (node *) PyErr_NoMemory();
+            PyErr_NoMemory();
+            return NULL;
         }
         if (err == E_OVERFLOW) {
+            Py_DECREF(elem);
             PyObject_FREE(strn);
             PyErr_SetString(PyExc_ValueError,
                             "unsupported number of child nodes");
@@ -837,14 +856,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
             node* new_child = CHILD(root, i - 1);
 
             if (new_child != build_node_children(elem, new_child, line_num)) {
-                Py_XDECREF(elem);
-                return (0);
+                Py_DECREF(elem);
+                return NULL;
             }
         }
         else if (type == NEWLINE) {     /* It's true:  we increment the     */
             ++(*line_num);              /* line number *after* the newline! */
         }
-        Py_XDECREF(elem);
+        Py_DECREF(elem);
     }
     return root;
 }
@@ -879,8 +898,23 @@ build_node_tree(PyObject *tuple)
 
         if (num == encoding_decl) {
             encoding = PySequence_GetItem(tuple, 2);
+            if (encoding == NULL) {
+                PyErr_SetString(parser_error, "missed encoding");
+                return NULL;
+            }
+            if (!PyString_Check(encoding)) {
+                PyErr_Format(parser_error,
+                             "encoding must be a string, found %.200s",
+                             Py_TYPE(encoding)->tp_name);
+                Py_DECREF(encoding);
+                return NULL;
+            }
             /* tuple isn't borrowed anymore here, need to DECREF */
             tuple = PySequence_GetSlice(tuple, 0, 2);
+            if (tuple == NULL) {
+                Py_DECREF(encoding);
+                return NULL;
+            }
         }
         res = PyNode_New(num);
         if (res != NULL) {
@@ -892,19 +926,27 @@ build_node_tree(PyObject *tuple)
                 Py_ssize_t len;
                 len = PyString_GET_SIZE(encoding) + 1;
                 res->n_str = (char *)PyObject_MALLOC(len);
-                if (res->n_str != NULL)
-                    (void) memcpy(res->n_str, PyString_AS_STRING(encoding), len);
-                Py_DECREF(encoding);
-                Py_DECREF(tuple);
+                if (res->n_str == NULL) {
+                    PyNode_Free(res);
+                    Py_DECREF(encoding);
+                    Py_DECREF(tuple);
+                    PyErr_NoMemory();
+                    return NULL;
+                }
+                (void) memcpy(res->n_str, PyString_AS_STRING(encoding), len);
             }
         }
+        if (encoding != NULL) {
+            Py_DECREF(encoding);
+            Py_DECREF(tuple);
+        }
     }
     else {
         /*  The tuple is illegal -- if the number is neither TERMINAL nor
          *  NONTERMINAL, we can't use it.  Not sure the implementation
          *  allows this condition, but the API doesn't preclude it.
          */
-        PyObject *err = Py_BuildValue("os", tuple,
+        PyObject *err = Py_BuildValue("Os", tuple,
                                       "Illegal component tuple.");
         PyErr_SetObject(parser_error, err);
         Py_XDECREF(err);
@@ -3371,7 +3413,6 @@ parser__pickler(PyObject *self, PyObject *args)
             result = Py_BuildValue("O(O)", pickle_constructor, tuple);
             Py_DECREF(tuple);
         }
-        Py_DECREF(empty_dict);
         Py_DECREF(newargs);
     }
   finally: