Repository navigation
Expand file tree
/
Copy pathsubprocess.py
More file actions
2509 lines (2149 loc) · 98.1 KB
/
Copy pathsubprocess.py
File metadata and controls
2509 lines (2149 loc) · 98.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# subprocess - Subprocesses with accessible I/O streams
#
# For more information about this module, see PEP 324.
#
# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
#
# Licensed to PSF under a Contributor Agreement.
r"""Subprocesses with accessible I/O streams
This module allows you to spawn processes, connect to their
input/output/error pipes, and obtain their return codes.
For a complete description of this module see the Python documentation.
Main API
========
run(...): Runs a command, waits for it to complete, then returns a
CompletedProcess instance.
Popen(...): A class for flexibly executing a command in a new process
Constants
---------
DEVNULL: Special value that indicates that os.devnull should be used
PIPE: Special value that indicates a pipe should be created
STDOUT: Special value that indicates that stderr should go to stdout
Older API
=========
call(...): Runs a command, waits for it to complete, then returns
the return code.
check_call(...): Same as call() but raises CalledProcessError()
if return code is not 0
check_output(...): Same as check_call() but returns the contents of
stdout instead of a return code
getoutput(...): Runs a command in the shell, waits for it to complete,
then returns the output
getstatusoutput(...): Runs a command in the shell, waits for it to complete,
then returns a (exitcode, output) tuple
"""
import builtins
import errno
import io
import locale
import os
import time
import signal
import sys
import threading
import warnings
import contextlib
from time import monotonic as _time
import types
try:
import fcntl
except ImportError:
fcntl = None
__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
"getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
"SubprocessError", "TimeoutExpired", "CompletedProcess"]
# NOTE: We intentionally exclude list2cmdline as it is
# considered an internal implementation detail. issue10838.
# use presence of msvcrt to detect Windows-like platforms (see bpo-8110)
try:
import msvcrt
except ModuleNotFoundError:
_mswindows = False
else:
_mswindows = True
# some platforms do not support subprocesses
_can_fork_exec = sys.platform not in {"emscripten", "wasi", "ios", "tvos", "watchos"}
if _mswindows:
import _winapi
from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401
STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
STD_ERROR_HANDLE, SW_HIDE,
STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW,
STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK,
ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS,
HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS,
NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS,
CREATE_NO_WINDOW, DETACHED_PROCESS,
CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB)
__all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
"STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
"STD_ERROR_HANDLE", "SW_HIDE",
"STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW",
"STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK",
"STARTUPINFO",
"ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS",
"HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS",
"NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS",
"CREATE_NO_WINDOW", "DETACHED_PROCESS",
"CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"])
else:
if _can_fork_exec:
from _posixsubprocess import fork_exec as _fork_exec
# used in methods that are called by __del__
class _del_safe:
waitpid = os.waitpid
waitstatus_to_exitcode = os.waitstatus_to_exitcode
WIFSTOPPED = os.WIFSTOPPED
WSTOPSIG = os.WSTOPSIG
WNOHANG = os.WNOHANG
ECHILD = errno.ECHILD
else:
class _del_safe:
waitpid = None
waitstatus_to_exitcode = None
WIFSTOPPED = None
WSTOPSIG = None
WNOHANG = None
ECHILD = errno.ECHILD
import select
import selectors
# Exception classes used by this module.
class SubprocessError(Exception): pass
class CalledProcessError(SubprocessError):
"""Raised when run() is called with check=True and the process
returns a non-zero exit status.
Attributes:
cmd, returncode, stdout, stderr, output
"""
def __init__(self, returncode, cmd, output=None, stderr=None):
self.returncode = returncode
self.cmd = cmd
self.output = output
self.stderr = stderr
def __str__(self):
if isinstance(self.returncode, int) and self.returncode < 0:
try:
return "Command '%s' died with %r." % (
self.cmd, signal.Signals(-self.returncode))
except ValueError:
return "Command '%s' died with unknown signal %d." % (
self.cmd, -self.returncode)
else:
return (f"Command {self.cmd!r} returned non-zero "
f"exit status {self.returncode}.")
@property
def stdout(self):
"""Alias for output attribute, to match stderr"""
return self.output
@stdout.setter
def stdout(self, value):
# There's no obvious reason to set this, but allow it anyway so
# .stdout is a transparent alias for .output
self.output = value
class TimeoutExpired(SubprocessError):
"""This exception is raised when the timeout expires while waiting for a
child process.
Attributes:
cmd, output, stdout, stderr, timeout
"""
def __init__(self, cmd, timeout, output=None, stderr=None):
self.cmd = cmd
self.timeout = timeout
self.output = output
self.stderr = stderr
def __str__(self):
return ("Command '%s' timed out after %s seconds" %
(self.cmd, self.timeout))
@property
def stdout(self):
return self.output
@stdout.setter
def stdout(self, value):
# There's no obvious reason to set this, but allow it anyway so
# .stdout is a transparent alias for .output
self.output = value
if _mswindows:
class STARTUPINFO:
def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None,
hStdError=None, wShowWindow=0, lpAttributeList=None):
self.dwFlags = dwFlags
self.hStdInput = hStdInput
self.hStdOutput = hStdOutput
self.hStdError = hStdError
self.wShowWindow = wShowWindow
self.lpAttributeList = lpAttributeList or {"handle_list": []}
def copy(self):
attr_list = self.lpAttributeList.copy()
if 'handle_list' in attr_list:
attr_list['handle_list'] = list(attr_list['handle_list'])
return STARTUPINFO(dwFlags=self.dwFlags,
hStdInput=self.hStdInput,
hStdOutput=self.hStdOutput,
hStdError=self.hStdError,
wShowWindow=self.wShowWindow,
lpAttributeList=attr_list)
class Handle(int):
closed = False
def Close(self, CloseHandle=_winapi.CloseHandle):
if not self.closed:
self.closed = True
CloseHandle(self)
def Detach(self):
if not self.closed:
self.closed = True
return int(self)
raise ValueError("already closed")
def __repr__(self):
return "%s(%d)" % (self.__class__.__name__, int(self))
__del__ = Close
else:
# When select or poll has indicated that the file is writable,
# we can write up to _PIPE_BUF bytes without risk of blocking.
# POSIX defines PIPE_BUF as >= 512.
_PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
# poll/select have the advantage of not requiring any extra file
# descriptor, contrarily to epoll/kqueue (also, they require a single
# syscall).
if hasattr(selectors, 'PollSelector'):
_PopenSelector = selectors.PollSelector
else:
_PopenSelector = selectors.SelectSelector
def _communicate_io_posix(selector, stdin, input_view, input_offset,
output_buffers, endtime, *, close_on_eof=False):
"""
Low-level POSIX I/O multiplexing loop used by Popen._communicate.
Handles the select loop for reading/writing but does not manage
stream lifecycle or raise timeout exceptions.
Args:
selector: A _PopenSelector with streams already registered
stdin: Writable file object for input, or None
input_view: memoryview of input bytes, or None
input_offset: Starting offset into input_view (for resume support)
output_buffers: Dict {file_object: list} to append read chunks to
endtime: Deadline timestamp, or None for no timeout
close_on_eof: If True, close output streams immediately when they
EOF rather than leaving them open for the caller to close.
Used by Popen._communicate() to match its historical behavior
of releasing fds as soon as the child closes the corresponding
pipe.
Returns:
(new_input_offset, completed)
- new_input_offset: How many bytes of input were written
- completed: True if all I/O finished, False if timed out
Note:
- Closes output streams on EOF only if close_on_eof=True
- Does NOT raise TimeoutExpired (caller handles)
- Appends to output_buffers lists in place
"""
stdin_fd = stdin.fileno() if stdin else None
while selector.get_map():
remaining = _deadline_remaining(endtime)
if remaining is not None and remaining <= 0:
return (input_offset, False) # Timed out
ready = selector.select(remaining)
# Check timeout after select (may have woken spuriously)
if endtime is not None and _time() > endtime:
return (input_offset, False) # Timed out
for key, events in ready:
if key.fd == stdin_fd:
chunk = input_view[input_offset:input_offset + _PIPE_BUF]
try:
input_offset += os.write(key.fd, chunk)
except BrokenPipeError:
selector.unregister(key.fd)
try:
stdin.close()
except BrokenPipeError:
pass
else:
if input_offset >= len(input_view):
selector.unregister(key.fd)
try:
stdin.close()
except BrokenPipeError:
pass
elif key.fileobj in output_buffers:
data = os.read(key.fd, 32768)
if not data:
selector.unregister(key.fileobj)
if close_on_eof:
try:
key.fileobj.close()
except OSError:
pass
else:
output_buffers[key.fileobj].append(data)
return (input_offset, True) # Completed
if _mswindows:
# On Windows we just need to close `Popen._handle` when we no longer need
# it, so that the kernel can free it. `Popen._handle` gets closed
# implicitly when the `Popen` instance is finalized (see `Handle.__del__`,
# which is calling `CloseHandle` as requested in [1]), so there is nothing
# for `_cleanup` to do.
#
# [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/
# creating-processes
_active = None
def _cleanup():
pass
else:
# This lists holds Popen instances for which the underlying process had not
# exited at the time its __del__ method got called: those processes are
# wait()ed for synchronously from _cleanup() when a new Popen object is
# created, to avoid zombie processes.
_active = []
def _cleanup():
if _active is None:
return
for inst in _active[:]:
res = inst._internal_poll(_deadstate=sys.maxsize)
if res is not None:
try:
_active.remove(inst)
except ValueError:
# This can happen if two threads create a new Popen instance.
# It's harmless that it was already removed, so ignore.
pass
PIPE = -1
STDOUT = -2
DEVNULL = -3
def _deadline_remaining(endtime):
"""Calculate remaining time until deadline."""
if endtime is None:
return None
return endtime - _time()
def _flush_stdin(stdin):
"""Flush stdin, ignoring BrokenPipeError and closed file ValueError."""
try:
stdin.flush()
except BrokenPipeError:
pass # communicate() must ignore BrokenPipeError.
except ValueError:
# Ignore ValueError: I/O operation on closed file.
if not stdin.closed:
raise
def _make_input_view(input_data):
"""Convert input data to a byte memoryview for writing.
Handles the case where input_data is already a memoryview with
non-byte elements (e.g., int32 array) by casting to a byte view.
This ensures len(view) returns the byte count, not element count.
"""
if not input_data:
return None
if isinstance(input_data, memoryview):
return input_data.cast("b") # ensure byte view for correct len()
return memoryview(input_data)
def _translate_newlines(data, encoding, errors):
"""Decode bytes to str and translate newlines to \n."""
data = data.decode(encoding, errors)
return data.replace("\r\n", "\n").replace("\r", "\n")
# XXX This function is only used by multiprocessing and the test suite,
# but it's here so that it can be imported when Python is compiled without
# threads.
def _optim_args_from_interpreter_flags():
"""Return a list of command-line arguments reproducing the current
optimization settings in sys.flags."""
args = []
value = sys.flags.optimize
if value > 0:
args.append('-' + 'O' * value)
return args
def _args_from_interpreter_flags():
"""Return a list of command-line arguments reproducing the current
settings in sys.flags, sys.warnoptions and sys._xoptions."""
flag_opt_map = {
'debug': 'd',
# 'inspect': 'i',
# 'interactive': 'i',
'dont_write_bytecode': 'B',
'no_site': 'S',
'verbose': 'v',
'bytes_warning': 'b',
'quiet': 'q',
# -O is handled in _optim_args_from_interpreter_flags()
}
args = _optim_args_from_interpreter_flags()
for flag, opt in flag_opt_map.items():
v = getattr(sys.flags, flag)
if v > 0:
args.append('-' + opt * v)
if sys.flags.isolated:
args.append('-I')
else:
if sys.flags.ignore_environment:
args.append('-E')
if sys.flags.no_user_site:
args.append('-s')
if sys.flags.safe_path:
args.append('-P')
# -W options
warnopts = sys.warnoptions[:]
xoptions = getattr(sys, '_xoptions', {})
bytes_warning = sys.flags.bytes_warning
dev_mode = sys.flags.dev_mode
if bytes_warning > 1:
warnopts.remove("error::BytesWarning")
elif bytes_warning:
warnopts.remove("default::BytesWarning")
if dev_mode:
warnopts.remove('default')
for opt in warnopts:
args.append('-W' + opt)
# -X options
if dev_mode:
args.extend(('-X', 'dev'))
for opt in sorted(xoptions):
if opt == 'dev':
# handled above via sys.flags.dev_mode
continue
value = xoptions[opt]
if value is True:
arg = opt
else:
arg = '%s=%s' % (opt, value)
args.extend(('-X', arg))
return args
def _text_encoding():
# Return default text encoding and emit EncodingWarning if
# sys.flags.warn_default_encoding is true.
if sys.flags.warn_default_encoding:
f = sys._getframe()
filename = f.f_code.co_filename
stacklevel = 2
while f := f.f_back:
if f.f_code.co_filename != filename:
break
stacklevel += 1
warnings.warn("'encoding' argument not specified.",
EncodingWarning, stacklevel)
if sys.flags.utf8_mode:
return "utf-8"
return locale.getencoding()
def call(*popenargs, timeout=None, **kwargs):
"""Run command with arguments. Wait for command to complete or
for timeout seconds, then return the returncode attribute.
The arguments are the same as for the Popen constructor. Example:
retcode = call(["ls", "-l"])
"""
with Popen(*popenargs, **kwargs) as p:
try:
return p.wait(timeout=timeout)
except: # Including KeyboardInterrupt, wait handled that.
p.kill()
# We don't call p.wait() again as p.__exit__ does that for us.
raise
def check_call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete. If
the exit code was zero then return, otherwise raise
CalledProcessError. The CalledProcessError object will have the
return code in the returncode attribute.
The arguments are the same as for the call function. Example:
check_call(["ls", "-l"])
"""
retcode = call(*popenargs, **kwargs)
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
raise CalledProcessError(retcode, cmd)
return 0
def check_output(*popenargs, timeout=None, **kwargs):
r"""Run command with arguments and return its output.
If the exit code was non-zero it raises a CalledProcessError. The
CalledProcessError object will have the return code in the returncode
attribute and output in the output attribute.
The arguments are the same as for the Popen constructor. Example:
>>> check_output(["ls", "-l", "/dev/null"])
b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
The stdout argument is not allowed as it is used internally.
To capture standard error in the result, use stderr=STDOUT.
>>> check_output(["/bin/sh", "-c",
... "ls -l non_existent_file ; exit 0"],
... stderr=STDOUT)
b'ls: non_existent_file: No such file or directory\n'
There is an additional optional argument, "input", allowing you to
pass a string to the subprocess's stdin. If you use this argument
you may not also use the Popen constructor's "stdin" argument, as
it too will be used internally. Example:
>>> check_output(["sed", "-e", "s/foo/bar/"],
... input=b"when in the course of fooman events\n")
b'when in the course of barman events\n'
By default, all communication is in bytes, and therefore any "input"
should be bytes, and the return value will be bytes. If in text mode,
any "input" should be a string, and the return value will be a string
decoded according to locale encoding, or by "encoding" if set. Text mode
is triggered by setting any of text, encoding, errors or universal_newlines.
"""
for kw in ('stdout', 'check'):
if kw in kwargs:
raise ValueError(f'{kw} argument not allowed, it will be overridden.')
if 'input' in kwargs and kwargs['input'] is None:
# Explicitly passing input=None was previously equivalent to passing an
# empty string. That is maintained here for backwards compatibility.
if kwargs.get('universal_newlines') or kwargs.get('text') or kwargs.get('encoding') \
or kwargs.get('errors'):
empty = ''
else:
empty = b''
kwargs['input'] = empty
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
**kwargs).stdout
class CompletedProcess(object):
"""A process that has finished running.
This is returned by run().
Attributes:
args: The list or str args passed to run().
returncode: The exit code of the process, negative for signals.
stdout: The standard output (None if not captured).
stderr: The standard error (None if not captured).
"""
def __init__(self, args, returncode, stdout=None, stderr=None):
self.args = args
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
def __repr__(self):
args = ['args={!r}'.format(self.args),
'returncode={!r}'.format(self.returncode)]
if self.stdout is not None:
args.append('stdout={!r}'.format(self.stdout))
if self.stderr is not None:
args.append('stderr={!r}'.format(self.stderr))
return "{}({})".format(type(self).__name__, ', '.join(args))
__class_getitem__ = classmethod(types.GenericAlias)
def check_returncode(self):
"""Raise CalledProcessError if the exit code is non-zero."""
if self.returncode:
raise CalledProcessError(self.returncode, self.args, self.stdout,
self.stderr)
def run(*popenargs,
input=None, capture_output=False, timeout=None, check=False, **kwargs):
"""Run command with arguments and return a CompletedProcess instance.
The returned instance will have attributes args, returncode, stdout and
stderr. By default, stdout and stderr are not captured, and those attributes
will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them,
or pass capture_output=True to capture both.
If check is True and the exit code was non-zero, it raises a
CalledProcessError. The CalledProcessError object will have the return code
in the returncode attribute, and output & stderr attributes if those streams
were captured.
If timeout (seconds) is given and the process takes too long,
a TimeoutExpired exception will be raised.
There is an optional argument "input", allowing you to
pass bytes or a string to the subprocess's stdin. If you use this argument
you may not also use the Popen constructor's "stdin" argument, as
it will be used internally.
By default, all communication is in bytes, and therefore any "input" should
be bytes, and the stdout and stderr will be bytes. If in text mode, any
"input" should be a string, and stdout and stderr will be strings decoded
according to locale encoding, or by "encoding" if set. Text mode is
triggered by setting any of text, encoding, errors or universal_newlines.
The other arguments are the same as for the Popen constructor.
"""
if input is not None:
if kwargs.get('stdin') is not None:
raise ValueError('stdin and input arguments may not both be used.')
kwargs['stdin'] = PIPE
if capture_output:
if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None:
raise ValueError('stdout and stderr arguments may not be used '
'with capture_output.')
kwargs['stdout'] = PIPE
kwargs['stderr'] = PIPE
with Popen(*popenargs, **kwargs) as process:
try:
stdout, stderr = process.communicate(input, timeout=timeout)
except TimeoutExpired as exc:
process.kill()
if _mswindows:
# Windows accumulates the output in a single blocking
# read() call run on child threads, with the timeout
# being done in a join() on those threads. communicate()
# _after_ kill() is required to collect that and add it
# to the exception.
exc.stdout, exc.stderr = process.communicate()
else:
# POSIX _communicate already populated the output so
# far into the TimeoutExpired exception.
process.wait()
raise
except: # Including KeyboardInterrupt, communicate handled that.
process.kill()
# We don't call process.wait() as .__exit__ does that for us.
raise
retcode = process.poll()
if check and retcode:
raise CalledProcessError(retcode, process.args,
output=stdout, stderr=stderr)
return CompletedProcess(process.args, retcode, stdout, stderr)
def list2cmdline(seq):
"""
Translate a sequence of arguments into a command line
string, using the same rules as the MS C runtime:
1) Arguments are delimited by white space, which is either a
space or a tab.
2) A string surrounded by double quotation marks is
interpreted as a single argument, regardless of white space
contained within. A quoted string can be embedded in an
argument.
3) A double quotation mark preceded by a backslash is
interpreted as a literal double quotation mark.
4) Backslashes are interpreted literally, unless they
immediately precede a double quotation mark.
5) If backslashes immediately precede a double quotation mark,
every pair of backslashes is interpreted as a literal
backslash. If the number of backslashes is odd, the last
backslash escapes the next double quotation mark as
described in rule 3.
"""
# See
# http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
# or search http://msdn.microsoft.com for
# "Parsing C++ Command-Line Arguments"
result = []
needquote = False
for arg in map(os.fsdecode, seq):
bs_buf = []
# Add a space to separate this argument from the others
if result:
result.append(' ')
needquote = (" " in arg) or ("\t" in arg) or not arg
if needquote:
result.append('"')
for c in arg:
if c == '\\':
# Don't know if we need to double yet.
bs_buf.append(c)
elif c == '"':
# Double backslashes.
result.append('\\' * len(bs_buf)*2)
bs_buf = []
result.append('\\"')
else:
# Normal char
if bs_buf:
result.extend(bs_buf)
bs_buf = []
result.append(c)
# Add remaining backslashes, if any.
if bs_buf:
result.extend(bs_buf)
if needquote:
result.extend(bs_buf)
result.append('"')
return ''.join(result)
# Various tools for executing commands and looking at their output and status.
#
def getstatusoutput(cmd, *, encoding=None, errors=None):
"""Return (exitcode, output) of executing cmd in a shell.
Execute the string 'cmd' in a shell with 'check_output' and
return a 2-tuple (status, output). The locale encoding is used
to decode the output and process newlines.
A trailing newline is stripped from the output.
The exit status for the command can be interpreted
according to the rules for the function 'wait'. Example:
>>> import subprocess
>>> subprocess.getstatusoutput('ls /bin/ls')
(0, '/bin/ls')
>>> subprocess.getstatusoutput('cat /bin/junk')
(1, 'cat: /bin/junk: No such file or directory')
>>> subprocess.getstatusoutput('/bin/junk')
(127, 'sh: /bin/junk: not found')
>>> subprocess.getstatusoutput('/bin/kill $$')
(-15, '')
"""
try:
data = check_output(cmd, shell=True, text=True, stderr=STDOUT,
encoding=encoding, errors=errors)
exitcode = 0
except CalledProcessError as ex:
data = ex.output
exitcode = ex.returncode
if data[-1:] == '\n':
data = data[:-1]
return exitcode, data
def getoutput(cmd, *, encoding=None, errors=None):
"""Return output (stdout or stderr) of executing cmd in a shell.
Like getstatusoutput(), except the exit status is ignored and the return
value is a string containing the command's output. Example:
>>> import subprocess
>>> subprocess.getoutput('ls /bin/ls')
'/bin/ls'
"""
return getstatusoutput(cmd, encoding=encoding, errors=errors)[1]
def _use_posix_spawn():
"""Check if posix_spawn() can be used for subprocess.
subprocess requires a posix_spawn() implementation that properly reports
errors to the parent process, & sets errno on the following failures:
* Process attribute actions failed.
* File actions failed.
* exec() failed.
Prefer an implementation which can use vfork() in some cases for best
performance.
"""
if _mswindows or not hasattr(os, 'posix_spawn'):
# os.posix_spawn() is not available
return False
if ((_env := os.environ.get('_PYTHON_SUBPROCESS_USE_POSIX_SPAWN')) in ('0', '1')):
return bool(int(_env))
if sys.platform in ('darwin', 'sunos5'):
# posix_spawn() is a syscall on both macOS and Solaris,
# and properly reports errors
return True
# Check libc name and runtime libc version
try:
ver = os.confstr('CS_GNU_LIBC_VERSION')
# parse 'glibc 2.28' as ('glibc', (2, 28))
parts = ver.split(maxsplit=1)
if len(parts) != 2:
# reject unknown format
raise ValueError
libc = parts[0]
version = tuple(map(int, parts[1].split('.')))
if sys.platform == 'linux' and libc == 'glibc' and version >= (2, 24):
# glibc 2.24 has a new Linux posix_spawn implementation using vfork
# which properly reports errors to the parent process.
return True
# Note: Don't use the implementation in earlier glibc because it doesn't
# use vfork (even if glibc 2.26 added a pipe to properly report errors
# to the parent process).
except (AttributeError, ValueError, OSError):
# os.confstr() or CS_GNU_LIBC_VERSION value not available
pass
# By default, assume that posix_spawn() does not properly report errors.
return False
def _can_use_pidfd_open():
# Availability: Linux >= 5.3
if not hasattr(os, "pidfd_open"):
return False
try:
pidfd = os.pidfd_open(os.getpid(), 0)
except OSError as err:
if err.errno in {errno.EMFILE, errno.ENFILE}:
# transitory 'too many open files'
return True
# likely blocked by security policy like SECCOMP (EPERM,
# EACCES, ENOSYS)
return False
else:
os.close(pidfd)
return True
def _can_use_kqueue():
# Availability: macOS, BSD
names = (
"kqueue",
"KQ_EV_ADD",
"KQ_EV_ONESHOT",
"KQ_FILTER_PROC",
"KQ_NOTE_EXIT",
)
if not all(hasattr(select, x) for x in names):
return False
kq = None
try:
kq = select.kqueue()
kev = select.kevent(
os.getpid(),
filter=select.KQ_FILTER_PROC,
flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT,
fflags=select.KQ_NOTE_EXIT,
)
kq.control([kev], 1, 0)
return True
except OSError as err:
if err.errno in {errno.EMFILE, errno.ENFILE}:
# transitory 'too many open files'
return True
return False
finally:
if kq is not None:
kq.close()
_CAN_USE_PIDFD_OPEN = not _mswindows and _can_use_pidfd_open()
_CAN_USE_KQUEUE = not _mswindows and _can_use_kqueue()
# Maximum timeout passed to poll() / kqueue.control() by Popen._wait():
# very large values (e.g. timeout=float('inf')) overflow the C timestamp
# conversion (gh-154836). Longer waits are performed in bounded slices
# until the deadline, like asyncio's MAXIMUM_SELECT_TIMEOUT.
_MAXIMUM_WAIT_TIMEOUT = 24 * 3600
# These are primarily fail-safe knobs for negatives. A True value does not
# guarantee the given libc/syscall API will be used.
_USE_POSIX_SPAWN = _use_posix_spawn()
_HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM')
class Popen:
""" Execute a child program in a new process.
For a complete description of the arguments see the Python documentation.
Arguments:
args: A string, or a sequence of program arguments.
bufsize: supplied as the buffering argument to the open() function when
creating the stdin/stdout/stderr pipe file objects
executable: A replacement program to execute.
stdin, stdout and stderr: These specify the executed programs' standard
input, standard output and standard error file handles, respectively.
preexec_fn: (POSIX only) An object to be called in the child process
just before the child is executed.
close_fds: Controls closing or inheriting of file descriptors.
shell: If true, the command will be executed through the shell.
cwd: Sets the current directory before the child is executed.
env: Defines the environment variables for the new process.
text: If true, decode stdin, stdout and stderr using the given encoding
(if set) or the system default otherwise.
universal_newlines: Alias of text, provided for backwards compatibility.
startupinfo and creationflags (Windows only)
restore_signals (POSIX only)
start_new_session (POSIX only)
process_group (POSIX only)
group (POSIX only)
extra_groups (POSIX only)
user (POSIX only)
umask (POSIX only)
pass_fds (POSIX only)
encoding and errors: Text mode encoding and error handling to use for
file objects stdin, stdout and stderr.
Attributes:
stdin, stdout, stderr, pid, returncode
"""
_child_created = False # Set here since __del__ checks it
def __init__(self, args, bufsize=-1, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=True,
shell=False, cwd=None, env=None, universal_newlines=None,
startupinfo=None, creationflags=0,
restore_signals=True, start_new_session=False,
pass_fds=(), *, user=None, group=None, extra_groups=None,
encoding=None, errors=None, text=None, umask=-1, pipesize=-1,
process_group=None):
"""Create new Popen instance."""
