Skip to content

Commit ec3d34c

Browse files
miss-islingtonpitrou
miss-islington
authored and
pitrou
committed
[3.6] bpo-31536: Avoid wholesale rebuild after make regen-all (GH-3678) (#3797)
bpo-31536: Avoid wholesale rebuild after `make regen-all`
1 parent befc956 commit ec3d34c

File tree

4 files changed

+86
-51
lines changed

4 files changed

+86
-51
lines changed

‎Makefile.pre.in‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ PYTHON= python$(EXE)
230230
BUILDPYTHON= python$(BUILDEXE)
231231

232232
PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@
233+
UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py
233234
PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
234235
_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
235236
BUILD_GNU_TYPE= @build@
@@ -696,12 +697,14 @@ regen-importlib: Programs/_freeze_importlib
696697
# from Lib/importlib/_bootstrap_external.py using _freeze_importlib
697698
./Programs/_freeze_importlib \
698699
$(srcdir)/Lib/importlib/_bootstrap_external.py \
699-
$(srcdir)/Python/importlib_external.h
700+
$(srcdir)/Python/importlib_external.h.new
701+
$(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new
700702
# Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py
701703
# using _freeze_importlib
702704
./Programs/_freeze_importlib \
703705
$(srcdir)/Lib/importlib/_bootstrap.py \
704-
$(srcdir)/Python/importlib.h
706+
$(srcdir)/Python/importlib.h.new
707+
$(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new
705708

706709

707710
############################################################################
@@ -775,8 +778,10 @@ regen-grammar: $(PGEN)
775778
# from Grammar/Grammar using pgen
776779
@$(MKDIR_P) Include
777780
$(PGEN) $(srcdir)/Grammar/Grammar \
778-
$(srcdir)/Include/graminit.h \
779-
$(srcdir)/Python/graminit.c
781+
$(srcdir)/Include/graminit.h.new \
782+
$(srcdir)/Python/graminit.c.new
783+
$(UPDATE_FILE) $(srcdir)/Include/graminit.h $(srcdir)/Include/graminit.h.new
784+
$(UPDATE_FILE) $(srcdir)/Python/graminit.c $(srcdir)/Python/graminit.c.new
780785

781786
Parser/grammar.o: $(srcdir)/Parser/grammar.c \
782787
$(srcdir)/Include/token.h \
@@ -794,21 +799,24 @@ regen-ast:
794799
# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h
795800
$(MKDIR_P) $(srcdir)/Include
796801
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
797-
-h $(srcdir)/Include \
802+
-h $(srcdir)/Include/Python-ast.h.new \
798803
$(srcdir)/Parser/Python.asdl
804+
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
799805
# Regenerate Python/Python-ast.c using Parser/asdl_c.py -c
800806
$(MKDIR_P) $(srcdir)/Python
801807
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
802-
-c $(srcdir)/Python \
808+
-c $(srcdir)/Python/Python-ast.c.new \
803809
$(srcdir)/Parser/Python.asdl
810+
$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
804811

805812
.PHONY: regen-opcode
806813
regen-opcode:
807814
# Regenerate Include/opcode.h from Lib/opcode.py
808815
# using Tools/scripts/generate_opcode_h.py
809816
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \
810817
$(srcdir)/Lib/opcode.py \
811-
$(srcdir)/Include/opcode.h
818+
$(srcdir)/Include/opcode.h.new
819+
$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
812820

813821
Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
814822

@@ -865,7 +873,8 @@ regen-opcode-targets:
865873
# Regenerate Python/opcode_targets.h from Lib/opcode.py
866874
# using Python/makeopcodetargets.py
867875
$(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \
868-
$(srcdir)/Python/opcode_targets.h
876+
$(srcdir)/Python/opcode_targets.h.new
877+
$(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new
869878

870879
Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h
871880

@@ -892,7 +901,8 @@ regen-typeslots:
892901
# using Objects/typeslots.py
893902
$(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \
894903
< $(srcdir)/Include/typeslots.h \
895-
$(srcdir)/Objects/typeslots.inc
904+
$(srcdir)/Objects/typeslots.inc.new
905+
$(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new
896906

897907
############################################################################
898908
# Header files

‎Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Avoid wholesale rebuild after `make regen-all` if nothing changed.

‎Parser/asdl_c.py‎

Lines changed: 38 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1281,59 +1281,55 @@ def main(srcfile, dump_module=False):
12811281
print(mod)
12821282
if not asdl.check(mod):
12831283
sys.exit(1)
1284-
if INC_DIR:
1285-
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
1286-
f = open(p, "w")
1287-
f.write(auto_gen_msg)
1288-
f.write('#include "asdl.h"\n\n')
1289-
c = ChainOfVisitors(TypeDefVisitor(f),
1290-
StructVisitor(f),
1291-
PrototypeVisitor(f),
1292-
)
1293-
c.visit(mod)
1294-
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
1295-
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
1296-
f.write("int PyAST_Check(PyObject* obj);\n")
1297-
f.close()
1298-
1299-
if SRC_DIR:
1300-
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
1301-
f = open(p, "w")
1302-
f.write(auto_gen_msg)
1303-
f.write('#include <stddef.h>\n')
1304-
f.write('\n')
1305-
f.write('#include "Python.h"\n')
1306-
f.write('#include "%s-ast.h"\n' % mod.name)
1307-
f.write('\n')
1308-
f.write("static PyTypeObject AST_type;\n")
1309-
v = ChainOfVisitors(
1310-
PyTypesDeclareVisitor(f),
1311-
PyTypesVisitor(f),
1312-
Obj2ModPrototypeVisitor(f),
1313-
FunctionVisitor(f),
1314-
ObjVisitor(f),
1315-
Obj2ModVisitor(f),
1316-
ASTModuleVisitor(f),
1317-
PartingShots(f),
1318-
)
1319-
v.visit(mod)
1320-
f.close()
1284+
if H_FILE:
1285+
with open(H_FILE, "w") as f:
1286+
f.write(auto_gen_msg)
1287+
f.write('#include "asdl.h"\n\n')
1288+
c = ChainOfVisitors(TypeDefVisitor(f),
1289+
StructVisitor(f),
1290+
PrototypeVisitor(f),
1291+
)
1292+
c.visit(mod)
1293+
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
1294+
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
1295+
f.write("int PyAST_Check(PyObject* obj);\n")
1296+
1297+
if C_FILE:
1298+
with open(C_FILE, "w") as f:
1299+
f.write(auto_gen_msg)
1300+
f.write('#include <stddef.h>\n')
1301+
f.write('\n')
1302+
f.write('#include "Python.h"\n')
1303+
f.write('#include "%s-ast.h"\n' % mod.name)
1304+
f.write('\n')
1305+
f.write("static PyTypeObject AST_type;\n")
1306+
v = ChainOfVisitors(
1307+
PyTypesDeclareVisitor(f),
1308+
PyTypesVisitor(f),
1309+
Obj2ModPrototypeVisitor(f),
1310+
FunctionVisitor(f),
1311+
ObjVisitor(f),
1312+
Obj2ModVisitor(f),
1313+
ASTModuleVisitor(f),
1314+
PartingShots(f),
1315+
)
1316+
v.visit(mod)
13211317

13221318
if __name__ == "__main__":
13231319
import getopt
13241320

1325-
INC_DIR = ''
1326-
SRC_DIR = ''
1321+
H_FILE = ''
1322+
C_FILE = ''
13271323
dump_module = False
13281324
opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
13291325
for o, v in opts:
13301326
if o == '-h':
1331-
INC_DIR = v
1327+
H_FILE = v
13321328
if o == '-c':
1333-
SRC_DIR = v
1329+
C_FILE = v
13341330
if o == '-d':
13351331
dump_module = True
1336-
if INC_DIR and SRC_DIR:
1332+
if H_FILE and C_FILE:
13371333
print('Must specify exactly one output file')
13381334
sys.exit(1)
13391335
elif len(args) != 1:

‎Tools/scripts/update_file.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
"""
2+
A script that replaces an old file with a new one, only if the contents
3+
actually changed. If not, the new file is simply deleted.
4+
5+
This avoids wholesale rebuilds when a code (re)generation phase does not
6+
actually change the in-tree generated code.
7+
"""
8+
9+
import os
10+
import sys
11+
12+
13+
def main(old_path, new_path):
14+
with open(old_path, 'rb') as f:
15+
old_contents = f.read()
16+
with open(new_path, 'rb') as f:
17+
new_contents = f.read()
18+
if old_contents != new_contents:
19+
os.replace(new_path, old_path)
20+
else:
21+
os.unlink(new_path)
22+
23+
24+
if __name__ == '__main__':
25+
if len(sys.argv) != 3:
26+
print("Usage: %s <path to be updated> <path with new contents>" % (sys.argv[0],))
27+
sys.exit(1)
28+
main(sys.argv[1], sys.argv[2])

0 commit comments

Comments
 (0)