Skip to content

Commit 63b8e0c

Browse files
isidenticalisidentical
authored
bpo-40528: Improve AST generation script to do builds simultaneously (GH-19968)
- Switch from getopt to argparse. - Removed the limitation of not being able to produce both C and H simultaneously. This will make it run faster since it parses the asdl definition once and uses the generated tree to generate both the header and the C source.
1 parent 7b7a21b commit 63b8e0c

File tree

3 files changed

+83
-102
lines changed

3 files changed

+83
-102
lines changed

‎Makefile.pre.in‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -845,17 +845,15 @@ regen-pegen:
845845

846846
.PHONY=regen-ast
847847
regen-ast:
848-
# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h
848+
# Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py
849849
$(MKDIR_P) $(srcdir)/Include
850-
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
851-
-h $(srcdir)/Include/Python-ast.h.new \
852-
$(srcdir)/Parser/Python.asdl
853-
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
854-
# Regenerate Python/Python-ast.c using Parser/asdl_c.py -c
855850
$(MKDIR_P) $(srcdir)/Python
856851
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
857-
-c $(srcdir)/Python/Python-ast.c.new \
858-
$(srcdir)/Parser/Python.asdl
852+
$(srcdir)/Parser/Python.asdl \
853+
-H $(srcdir)/Include/Python-ast.h.new \
854+
-C $(srcdir)/Python/Python-ast.c.new
855+
856+
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
859857
$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
860858

861859
.PHONY: regen-opcode

‎PCbuild/regen.vcxproj‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -176,20 +176,15 @@
176176
<Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" />
177177
</Target>
178178
<Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
179-
<!-- Regenerate Include/Python-ast.h using Parser/asdl_c.py -h -->
180-
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -h &quot;$(IntDir)Python-ast.h&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
179+
<!-- Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py -h -->
180+
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot;" />
181181
<Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
182182
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
183183
</Copy>
184-
<Warning Text="Python-ast.h updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
185-
</Target>
186-
<Target Name="_RegenAST_C" AfterTargets="_RegenGrammar">
187-
<!-- Regenerate Python/Python-ast.c using Parser/asdl_c.py -c -->
188-
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -c &quot;$(IntDir)Python-ast.c&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
189184
<Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
190-
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
185+
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
191186
</Copy>
192-
<Warning Text="Python-ast.c updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
187+
<Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
193188
</Target>
194189
<Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
195190
<!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->

‎Parser/asdl_c.py‎

Lines changed: 73 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
#! /usr/bin/env python
22
"""Generate C code from an ASDL description."""
33

4-
import os, sys
4+
import os
5+
import sys
6+
7+
from argparse import ArgumentParser
8+
from pathlib import Path
59

610
import asdl
711

812
TABSIZE = 4
913
MAX_COL = 80
14+
AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n"
1015

1116
def get_c_type(name):
1217
"""Return a string for the C name of the type.
@@ -1369,94 +1374,77 @@ def generate_module_def(f, mod):
13691374
f.write(' return 1;\n')
13701375
f.write('};\n\n')
13711376

1372-
1373-
common_msg = "/* File automatically generated by %s. */\n\n"
1374-
1375-
def main(srcfile, dump_module=False):
1376-
argv0 = sys.argv[0]
1377-
components = argv0.split(os.sep)
1378-
# Always join with '/' so different OS does not keep changing the file
1379-
argv0 = '/'.join(components[-2:])
1380-
auto_gen_msg = common_msg % argv0
1381-
mod = asdl.parse(srcfile)
1377+
def write_header(f, mod):
1378+
f.write('#ifndef Py_PYTHON_AST_H\n')
1379+
f.write('#define Py_PYTHON_AST_H\n')
1380+
f.write('#ifdef __cplusplus\n')
1381+
f.write('extern "C" {\n')
1382+
f.write('#endif\n')
1383+
f.write('\n')
1384+
f.write('#ifndef Py_LIMITED_API\n')
1385+
f.write('#include "asdl.h"\n')
1386+
f.write('\n')
1387+
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
1388+
f.write('\n')
1389+
c = ChainOfVisitors(TypeDefVisitor(f),
1390+
StructVisitor(f))
1391+
c.visit(mod)
1392+
f.write("// Note: these macros affect function definitions, not only call sites.\n")
1393+
PrototypeVisitor(f).visit(mod)
1394+
f.write("\n")
1395+
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
1396+
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
1397+
f.write("int PyAST_Check(PyObject* obj);\n")
1398+
f.write("#endif /* !Py_LIMITED_API */\n")
1399+
f.write('\n')
1400+
f.write('#ifdef __cplusplus\n')
1401+
f.write('}\n')
1402+
f.write('#endif\n')
1403+
f.write('#endif /* !Py_PYTHON_AST_H */\n')
1404+
1405+
def write_source(f, mod):
1406+
f.write('#include <stddef.h>\n')
1407+
f.write('\n')
1408+
f.write('#include "Python.h"\n')
1409+
f.write('#include "%s-ast.h"\n' % mod.name)
1410+
f.write('#include "structmember.h" // PyMemberDef\n')
1411+
f.write('\n')
1412+
1413+
generate_module_def(f, mod)
1414+
1415+
v = ChainOfVisitors(
1416+
PyTypesDeclareVisitor(f),
1417+
PyTypesVisitor(f),
1418+
Obj2ModPrototypeVisitor(f),
1419+
FunctionVisitor(f),
1420+
ObjVisitor(f),
1421+
Obj2ModVisitor(f),
1422+
ASTModuleVisitor(f),
1423+
PartingShots(f),
1424+
)
1425+
v.visit(mod)
1426+
1427+
def main(input_file, c_file, h_file, dump_module=False):
1428+
auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
1429+
mod = asdl.parse(input_file)
13821430
if dump_module:
13831431
print('Parsed Module:')
13841432
print(mod)
13851433
if not asdl.check(mod):
13861434
sys.exit(1)
1387-
if H_FILE:
1388-
with open(H_FILE, "w") as f:
1389-
f.write(auto_gen_msg)
1390-
f.write('#ifndef Py_PYTHON_AST_H\n')
1391-
f.write('#define Py_PYTHON_AST_H\n')
1392-
f.write('#ifdef __cplusplus\n')
1393-
f.write('extern "C" {\n')
1394-
f.write('#endif\n')
1395-
f.write('\n')
1396-
f.write('#ifndef Py_LIMITED_API\n')
1397-
f.write('#include "asdl.h"\n')
1398-
f.write('\n')
1399-
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
1400-
f.write('\n')
1401-
c = ChainOfVisitors(TypeDefVisitor(f),
1402-
StructVisitor(f))
1403-
1404-
c.visit(mod)
1405-
f.write("// Note: these macros affect function definitions, not only call sites.\n")
1406-
PrototypeVisitor(f).visit(mod)
1407-
f.write("\n")
1408-
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
1409-
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
1410-
f.write("int PyAST_Check(PyObject* obj);\n")
1411-
f.write("#endif /* !Py_LIMITED_API */\n")
1412-
f.write('\n')
1413-
f.write('#ifdef __cplusplus\n')
1414-
f.write('}\n')
1415-
f.write('#endif\n')
1416-
f.write('#endif /* !Py_PYTHON_AST_H */\n')
1417-
1418-
if C_FILE:
1419-
with open(C_FILE, "w") as f:
1420-
f.write(auto_gen_msg)
1421-
f.write('#include <stddef.h>\n')
1422-
f.write('\n')
1423-
f.write('#include "Python.h"\n')
1424-
f.write('#include "%s-ast.h"\n' % mod.name)
1425-
f.write('#include "structmember.h" // PyMemberDef\n')
1426-
f.write('\n')
1427-
1428-
generate_module_def(f, mod)
1429-
1430-
v = ChainOfVisitors(
1431-
PyTypesDeclareVisitor(f),
1432-
PyTypesVisitor(f),
1433-
Obj2ModPrototypeVisitor(f),
1434-
FunctionVisitor(f),
1435-
ObjVisitor(f),
1436-
Obj2ModVisitor(f),
1437-
ASTModuleVisitor(f),
1438-
PartingShots(f),
1439-
)
1440-
v.visit(mod)
1435+
for file, writer in (c_file, write_source), (h_file, write_header):
1436+
if file is not None:
1437+
with file.open("w") as f:
1438+
f.write(auto_gen_msg)
1439+
writer(f, mod)
1440+
print(file, "regenerated.")
14411441

14421442
if __name__ == "__main__":
1443-
import getopt
1444-
1445-
H_FILE = ''
1446-
C_FILE = ''
1447-
dump_module = False
1448-
opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
1449-
for o, v in opts:
1450-
if o == '-h':
1451-
H_FILE = v
1452-
elif o == '-c':
1453-
C_FILE = v
1454-
elif o == '-d':
1455-
dump_module = True
1456-
if H_FILE and C_FILE:
1457-
print('Must specify exactly one output file')
1458-
sys.exit(1)
1459-
elif len(args) != 1:
1460-
print('Must specify single input file')
1461-
sys.exit(1)
1462-
main(args[0], dump_module)
1443+
parser = ArgumentParser()
1444+
parser.add_argument("input_file", type=Path)
1445+
parser.add_argument("-C", "--c-file", type=Path, default=None)
1446+
parser.add_argument("-H", "--h-file", type=Path, default=None)
1447+
parser.add_argument("-d", "--dump-module", action="store_true")
1448+
1449+
options = parser.parse_args()
1450+
main(**vars(options))

0 commit comments

Comments
 (0)