Out-of-bounds array reads in Python/ast.c

[bradlarsen]

BPO	36495
Nosy	@gvanrossum, @bradlarsen
PRs	bpo-36495: Fix two out-of-bounds array reads #12641

^{Note: these values reflect the state of the issue at the time it was migrated and might not reflect the current state.}

Show more details

GitHub fields:

assignee = None
closed_at = <Date 2019-04-05.19:48:22.939>
created_at = <Date 2019-03-31.14:52:52.408>
labels = ['type-security', 'interpreter-core', 'build', '3.8']
title = 'Out-of-bounds array reads in Python/ast.c'
updated_at = <Date 2019-04-05.19:48:22.938>
user = 'https://github.com/bradlarsen'

bugs.python.org fields:

activity = <Date 2019-04-05.19:48:22.938>
actor = 'levkivskyi'
assignee = 'none'
closed = True
closed_date = <Date 2019-04-05.19:48:22.939>
closer = 'levkivskyi'
components = ['Build', 'Interpreter Core']
creation = <Date 2019-03-31.14:52:52.408>
creator = 'blarsen'
dependencies = []
files = []
hgrepos = []
issue_num = 36495
keywords = ['patch']
message_count = 2.0
messages = ['339260', '339296']
nosy_count = 2.0
nosy_names = ['gvanrossum', 'blarsen']
pr_nums = ['12641']
priority = 'normal'
resolution = 'fixed'
stage = 'resolved'
status = 'closed'
superseder = None
type = 'security'
url = 'https://bugs.python.org/issue36495'
versions = ['Python 3.8']

[bradlarsen]

There are currently 2 places in Python/ast.c on master where an out-of-bounds
array read can occur.

Both were introduced with the merge of of typed_ast into CPython in commit
dcfcd14 (bpo-35766, #55854).

In both places, the out-of-bounds array read occurs when an index variable is
incremented, then used before checking that it is still valid.

The first out-of-bounds read, in handle_keywordonly_args(), around line 1403:

    case vfpdef:
    case tfpdef:
        if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
            expression = ast_for_expr(c, CHILD(n, i + 2));
            if (!expression)
                goto error;
            asdl_seq_SET(kwdefaults, j, expression);
            i += 2; /* '=' and test */
        }
        else { /* setting NULL if no default value exists */
            asdl_seq_SET(kwdefaults, j, NULL);
        }
        if (NCH(ch) == 3) {
            /* ch is NAME ':' test */
            annotation = ast_for_expr(c, CHILD(ch, 2));
            if (!annotation)
                goto error;
        }
        else {
            annotation = NULL;
        }
        ch = CHILD(ch, 0);
        argname = NEW_IDENTIFIER(ch);
        if (!argname)
            goto error;
        if (forbidden_name(c, argname, ch, 0))
            goto error;
        arg = arg(argname, annotation, NULL, LINENO(ch), ch->n_col_offset,
                  ch->n_end_lineno, ch->n_end_col_offset,
                  c->c_arena);
        if (!arg)
            goto error;
        asdl_seq_SET(kwonlyargs, j++, arg);
        i += 1; /* the name */
        if (TYPE(CHILD(n, i)) == COMMA)           /* HERE, OOB read */
            i += 1; /* the comma, if present */
        break;

The second out-of-bounds read, in ast_for_arguments(), around line 1602:

/* ... */
case DOUBLESTAR:
    ch = CHILD(n, i+1);  /* tfpdef */
    assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);
    kwarg = ast_for_arg(c, ch);
    if (!kwarg)
        return NULL;
    i += 2; /* the double star and the name */
    if (TYPE(CHILD(n, i)) == COMMA)           /* HERE, OOB read */
        i += 1; /* the comma, if present */
    break;
/* ... */

You might see these out-of-bounds reads as crashes at various points during the
build process if you configure as so:

    $ clang --version
    clang version 8.0.0 (tags/RELEASE_800/final)
    Target: x86_64-unknown-linux-gnu
    Thread model: posix
    InstalledDir: /usr/local/bin

    $ clang++ --version
    clang version 8.0.0 (tags/RELEASE_800/final)
    Target: x86_64-unknown-linux-gnu
    Thread model: posix
    InstalledDir: /usr/local/bin

    $ export ASAN_OPTIONS=detect_leaks=0 CC=clang CXX=clang++
    $ ./configure --with-address-sanitizer --without-pydebug

    $ make   # might fail partway through, but a python binary should still be produced

Finally, to see crashes from the out-of-bounds reads:

    $ ./python.exe -c 'def foo(f, *args, kw=None): pass'
    

=================================================================
==59698==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x613000006740 at pc 0x0000009aff20 bp 0x7ffe94660260 sp 0x7ffe94660258
READ of size 2 at 0x613000006740 thread T0
#0 0x9aff1f in handle_keywordonly_args /cpython/Python/ast.c:1403:21
#1 0x9af034 in ast_for_arguments /cpython/Python/ast.c:1588:31
#2 0x9bb174 in ast_for_funcdef_impl /cpython/Python/ast.c:1748:12
#3 0x99ce99 in PyAST_FromNodeObject /cpython/Python/ast.c:806:25
#4 0x7bc4a2 in PyParser_ASTFromStringObject /cpython/Python/pythonrun.c:1216:15
#5 0x7ba4cf in PyRun_StringFlags /cpython/Python/pythonrun.c:963:11
#6 0x7ba422 in PyRun_SimpleStringFlags /cpython/Python/pythonrun.c:461:9
#7 0x512c5e in pymain_run_command /cpython/Modules/main.c:220:11
#8 0x512c5e in pymain_run_python /cpython/Modules/main.c:501
#9 0x512c5e in _Py_RunMain /cpython/Modules/main.c:583
#10 0x5144e2 in pymain_main /cpython/Modules/main.c:612:12
#11 0x51485f in _Py_UnixMain /cpython/Modules/main.c:636:12
#12 0x7f62e8f92b96 in __libc_start_main /build/glibc-OTsEL5/glibc-2.27/csu/../csu/libc-start.c:310
#13 0x437729 in _start (/cpython/python.exe+0x437729)

0x613000006740 is located 0 bytes to the right of 384-byte region [0x6130000065c0,0x613000006740)
allocated by thread T0 here:
    #0 0x4e34ef in realloc /tmp/tmp.XYTE7P6bCb/final/llvm.src/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:165:3
    #1 0x8fa635 in PyNode_AddChild /cpython/Parser/node.c:120:22
    #2 0x9d16f6 in shift /cpython/Parser/parser.c:114:11
    #3 0x9d16f6 in PyParser_AddToken /cpython/Parser/parser.c:285
    #4 0x8fbee3 in parsetok /cpython/Parser/parsetok.c:332:14
    #5 0x7bc44a in PyParser_ASTFromStringObject /cpython/Python/pythonrun.c:1206:15
    #6 0x7ba4cf in PyRun_StringFlags /cpython/Python/pythonrun.c:963:11
    #7 0x7ba422 in PyRun_SimpleStringFlags /cpython/Python/pythonrun.c:461:9
    #8 0x512c5e in pymain_run_command /cpython/Modules/main.c:220:11
    #9 0x512c5e in pymain_run_python /cpython/Modules/main.c:501
    #10 0x512c5e in _Py_RunMain /cpython/Modules/main.c:583
    #11 0x5144e2 in pymain_main /cpython/Modules/main.c:612:12
    #12 0x51485f in _Py_UnixMain /cpython/Modules/main.c:636:12
    #13 0x7f62e8f92b96 in __libc_start_main /build/glibc-OTsEL5/glibc-2.27/csu/../csu/libc-start.c:310

SUMMARY: AddressSanitizer: heap-buffer-overflow /cpython/Python/ast.c:1403:21 in handle_keywordonly_args
Shadow bytes around the buggy address:
  0x0c267fff8c90: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c267fff8ca0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fa fa
  0x0c267fff8cb0: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
  0x0c267fff8cc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x0c267fff8cd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
=>0x0c267fff8ce0: 00 00 00 00 00 00 00 00[fa]fa fa fa fa fa fa fa
  0x0c267fff8cf0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x0c267fff8d00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x0c267fff8d10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x0c267fff8d20: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c267fff8d30: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
  Shadow gap:              cc
==59698==ABORTING

    $ ./python.exe -c 'def foo(f, **kws): pass'
    

=================================================================
==59696==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x61000001cc00 at pc 0x0000009af45c bp 0x7fff799ca580 sp 0x7fff799ca578
READ of size 2 at 0x61000001cc00 thread T0
#0 0x9af45b in ast_for_arguments /cpython/Python/ast.c:1602:21
#1 0x9bb174 in ast_for_funcdef_impl /cpython/Python/ast.c:1748:12
#2 0x99ce99 in PyAST_FromNodeObject /cpython/Python/ast.c:806:25
#3 0x7bc4a2 in PyParser_ASTFromStringObject /cpython/Python/pythonrun.c:1216:15
#4 0x7ba4cf in PyRun_StringFlags /cpython/Python/pythonrun.c:963:11
#5 0x7ba422 in PyRun_SimpleStringFlags /cpython/Python/pythonrun.c:461:9
#6 0x512c5e in pymain_run_command /cpython/Modules/main.c:220:11
#7 0x512c5e in pymain_run_python /cpython/Modules/main.c:501
#8 0x512c5e in _Py_RunMain /cpython/Modules/main.c:583
#9 0x5144e2 in pymain_main /cpython/Modules/main.c:612:12
#10 0x51485f in _Py_UnixMain /cpython/Modules/main.c:636:12
#11 0x7f0c78595b96 in __libc_start_main /build/glibc-OTsEL5/glibc-2.27/csu/../csu/libc-start.c:310
#12 0x437729 in _start (/cpython/python.exe+0x437729)

0x61000001cc00 is located 0 bytes to the right of 192-byte region [0x61000001cb40,0x61000001cc00)
allocated by thread T0 here:
    #0 0x4e34ef in realloc /tmp/tmp.XYTE7P6bCb/final/llvm.src/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:165:3
    #1 0x8fa635 in PyNode_AddChild /cpython/Parser/node.c:120:22
    #2 0x9d16f6 in shift /cpython/Parser/parser.c:114:11
    #3 0x9d16f6 in PyParser_AddToken /cpython/Parser/parser.c:285
    #4 0x8fbee3 in parsetok /cpython/Parser/parsetok.c:332:14
    #5 0x7bc44a in PyParser_ASTFromStringObject /cpython/Python/pythonrun.c:1206:15
    #6 0x7ba4cf in PyRun_StringFlags /cpython/Python/pythonrun.c:963:11
    #7 0x7ba422 in PyRun_SimpleStringFlags /cpython/Python/pythonrun.c:461:9
    #8 0x512c5e in pymain_run_command /cpython/Modules/main.c:220:11
    #9 0x512c5e in pymain_run_python /cpython/Modules/main.c:501
    #10 0x512c5e in _Py_RunMain /cpython/Modules/main.c:583
    #11 0x5144e2 in pymain_main /cpython/Modules/main.c:612:12
    #12 0x51485f in _Py_UnixMain /cpython/Modules/main.c:636:12
    #13 0x7f0c78595b96 in __libc_start_main /build/glibc-OTsEL5/glibc-2.27/csu/../csu/libc-start.c:310

SUMMARY: AddressSanitizer: heap-buffer-overflow /cpython/Python/ast.c:1602:21 in ast_for_arguments
Shadow bytes around the buggy address:
  0x0c207fffb930: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c207fffb940: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
  0x0c207fffb950: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x0c207fffb960: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
  0x0c207fffb970: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
=>0x0c207fffb980:[fa]fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
  0x0c207fffb990: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x0c207fffb9a0: fa fa fa fa fa fa fa fa 00 00 00 00 00 00 00 00
  0x0c207fffb9b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x0c207fffb9c0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c207fffb9d0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
  Shadow gap:              cc
==59696==ABORTING

[gvanrossum]

New changeset a4d7836 by Guido van Rossum (Brad Larsen) in branch 'master':
bpo-36495: Fix two out-of-bounds array reads (GH-12641)
a4d7836