[3.6] bpo-9146: Raise a ValueError if OpenSSL fails to init a hash func (#3274)

* [3.6] bpo-9146: Raise a ValueError if OpenSSL fails to init a hash func. (GH-1777)

This helps people in weird FIPS mode environments where common things
like MD5 are not available in the binary as a matter of policy.
(cherry picked from commit 07244a8)

* Include a NEWS entry.

Author: gpshead

Misc/NEWS.d/next/Library/2017-09-03-14-10-00.bps-9146._-oo-_.rst

@@ -0,0 +1,3 @@
+Fix a segmentation fault in _hashopenssl when standard hash functions
+such as md5 are not available in the linked OpenSSL library.  As in
+some special FIPS-140 build environments.

Modules/_hashopenssl.c

@@ -139,7 +139,10 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
             process = MUNCH_SIZE;
         else
             process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int);
-        EVP_DigestUpdate(self->ctx, (const void*)cp, process);
+        if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) {
+            _setException(PyExc_ValueError);
+            break;
+        }
         len -= process;
         cp += process;
     }
@@ -209,7 +212,10 @@ EVP_digest(EVPobject *self, PyObject *unused)
         return _setException(PyExc_ValueError);
     }
     digest_size = EVP_MD_CTX_size(temp_ctx);
-    EVP_DigestFinal(temp_ctx, digest, NULL);
+    if (!EVP_DigestFinal(temp_ctx, digest, NULL)) {
+        _setException(PyExc_ValueError);
+        return NULL;
+    }
 
     retval = PyBytes_FromStringAndSize((const char *)digest, digest_size);
     EVP_MD_CTX_free(temp_ctx);
@@ -237,7 +243,10 @@ EVP_hexdigest(EVPobject *self, PyObject *unused)
         return _setException(PyExc_ValueError);
     }
     digest_size = EVP_MD_CTX_size(temp_ctx);
-    EVP_DigestFinal(temp_ctx, digest, NULL);
+    if (!EVP_DigestFinal(temp_ctx, digest, NULL)) {
+        _setException(PyExc_ValueError);
+        return NULL;
+    }
 
     EVP_MD_CTX_free(temp_ctx);
 
@@ -362,7 +371,12 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds)
             PyBuffer_Release(&view);
         return -1;
     }
-    EVP_DigestInit(self->ctx, digest);
+    if (!EVP_DigestInit(self->ctx, digest)) {
+        _setException(PyExc_ValueError);
+        if (data_obj)
+            PyBuffer_Release(&view);
+        return -1;
+    }
 
     self->name = name_obj;
     Py_INCREF(self->name);
@@ -461,7 +475,11 @@ EVPnew(PyObject *name_obj,
     if (initial_ctx) {
         EVP_MD_CTX_copy(self->ctx, initial_ctx);
     } else {
-        EVP_DigestInit(self->ctx, digest);
+        if (!EVP_DigestInit(self->ctx, digest)) {
+            _setException(PyExc_ValueError);
+            Py_DECREF(self);
+            return NULL;
+        }
     }
 
     if (cp && len) {
@@ -902,6 +920,8 @@ generate_hash_name_list(void)
  *  the generic one passing it a python string and are noticeably
  *  faster than calling a python new() wrapper.  Thats important for
  *  code that wants to make hashes of a bunch of small strings.
+ *  The first call will lazy-initialize, which reports an exception
+ *  if initialization fails.
  */
 #define GEN_CONSTRUCTOR(NAME)  \
     static PyObject * \
@@ -914,6 +934,17 @@ generate_hash_name_list(void)
         if (!PyArg_ParseTuple(args, "|O:" #NAME , &data_obj)) { \
             return NULL; \
         } \
+     \
+        if (CONST_new_ ## NAME ## _ctx_p == NULL) { \
+            EVP_MD_CTX *ctx_p = EVP_MD_CTX_new(); \
+            if (!EVP_get_digestbyname(#NAME) || \
+                !EVP_DigestInit(ctx_p, EVP_get_digestbyname(#NAME))) { \
+                _setException(PyExc_ValueError); \
+                EVP_MD_CTX_free(ctx_p); \
+                return NULL; \
+            } \
+            CONST_new_ ## NAME ## _ctx_p = ctx_p; \
+        } \
      \
         if (data_obj) \
             GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); \
@@ -942,10 +973,6 @@ generate_hash_name_list(void)
 #define INIT_CONSTRUCTOR_CONSTANTS(NAME)  do { \
     if (CONST_ ## NAME ## _name_obj == NULL) { \
         CONST_ ## NAME ## _name_obj = PyUnicode_FromString(#NAME); \
-        if (EVP_get_digestbyname(#NAME)) { \
-            CONST_new_ ## NAME ## _ctx_p = EVP_MD_CTX_new(); \
-            EVP_DigestInit(CONST_new_ ## NAME ## _ctx_p, EVP_get_digestbyname(#NAME)); \
-        } \
     } \
 } while (0);