"HTTPException", "NotConnected", "UnknownProtocol",

"UnknownTransferEncoding", "UnimplementedFileMode",

"IncompleteRead", "InvalidURL", "ImproperConnectionState",

"CannotSendRequest", "CannotSendHeader", "ResponseNotReady",

"BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",

"""Call data.encode("latin-1") but show a better error message."""

try:

return data.encode("latin-1")

except UnicodeEncodeError as err:

raise UnicodeEncodeError(

err.encoding,

err.object,

err.start,

err.end,

"%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') "

"if you want to send it encoded in UTF-8." %

# XXX The only usage of this method is in

# http.server.CGIHTTPRequestHandler. Maybe move the code there so

# that it doesn't need to be part of the public API. The API has

# never been defined so this could cause backwards compatibility

# issues.

def getallmatchingheaders(self, name):

"""Find all header lines matching a given header name.

name = name.lower() + ':'

n = len(name)

lst = []

hit = 0

for line in self.keys():

if line[:n].lower() == name:

hit = 1

elif not line[:1].isspace():

hit = 0

if hit:

lst.append(line)

"""Parses only RFC2822 headers from a file pointer.

headers = []

while True:

line = fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("header line")

headers.append(line)

if len(headers) > _MAXHEADERS:

raise HTTPException("got more than %d headers" % _MAXHEADERS)

if line in (b'\r\n', b'\n', b''):

break

hstring = b''.join(headers).decode('iso-8859-1')

# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.

# The bytes from the socket object are iso-8859-1 strings.

# See RFC 2616 sec 2.2 which notes an exception for MIME-encoded

# text following RFC 2047. The basic status line parsing only

# accepts iso-8859-1.

def __init__(self, sock, debuglevel=0, method=None, url=None):

# If the response includes a content-length header, we need to

# make sure that the client doesn't read more than the

# specified number of bytes. If it does, it will block until

# the server times out and closes the connection. This will

# happen if a self.fp.read() is done (without a size) whether

# self.fp is buffered or not. So, no self.fp.read() by

# clients unless they know what they are doing.

self.fp = sock.makefile("rb")

self.debuglevel = debuglevel

self._method = method

# The HTTPResponse object is returned via urllib. The clients

# of http and urllib expect different attributes for the

# headers. headers is used here and supports urllib. msg is

# provided as a backwards compatibility layer for http

# clients.

self.headers = self.msg = None

# from the Status-Line of the response

self.version = _UNKNOWN # HTTP-Version

self.status = _UNKNOWN # Status-Code

self.reason = _UNKNOWN # Reason-Phrase

self.chunked = _UNKNOWN # is "chunked" being used?

self.chunk_left = _UNKNOWN # bytes left to read in current chunk

self.length = _UNKNOWN # number of bytes left in response

self.will_close = _UNKNOWN # conn will close at end of response

def _read_status(self):

line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")

if len(line) > _MAXLINE:

raise LineTooLong("status line")

if self.debuglevel > 0:

print("reply:", repr(line))

if not line:

# Presumably, the server closed the connection before

# sending a valid response.

raise RemoteDisconnected("Remote end closed connection without"

" response")

try:

version, status, reason = line.split(None, 2)

except ValueError:

try:

version, status = line.split(None, 1)

reason = ""

except ValueError:

# empty version will cause next test to fail.

version = ""

if not version.startswith("HTTP/"):

self._close_conn()

raise BadStatusLine(line)

# The status code is a three-digit number

try:

status = int(status)

if status < 100 or status > 999:

raise BadStatusLine(line)

except ValueError:

raise BadStatusLine(line)

return version, status, reason

def begin(self):

if self.headers is not None:

# we've already started reading the response

return

# read until we get a non-100 response

while True:

version, status, reason = self._read_status()

if status != CONTINUE:

break

# skip the header from the 100 response

while True:

skip = self.fp.readline(_MAXLINE + 1)

if len(skip) > _MAXLINE:

raise LineTooLong("header line")

skip = skip.strip()

if not skip:

break

if self.debuglevel > 0:

print("header:", skip)

self.code = self.status = status

self.reason = reason.strip()

if version in ("HTTP/1.0", "HTTP/0.9"):

# Some servers might still return "0.9", treat it as 1.0 anyway

self.version = 10

elif version.startswith("HTTP/1."):

self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1

else:

raise UnknownProtocol(version)

self.headers = self.msg = parse_headers(self.fp)

if self.debuglevel > 0:

for hdr in self.headers:

print("header:", hdr, end=" ")

# are we using the chunked-style of transfer encoding?

tr_enc = self.headers.get("transfer-encoding")

if tr_enc and tr_enc.lower() == "chunked":

self.chunked = True

self.chunk_left = None

else:

self.chunked = False

# will the connection close at the end of the response?

self.will_close = self._check_close()

# do we have a Content-Length?

# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"

self.length = None

length = self.headers.get("content-length")

# are we using the chunked-style of transfer encoding?

tr_enc = self.headers.get("transfer-encoding")

if length and not self.chunked:

try:

self.length = int(length)

except ValueError:

self.length = None

else:

if self.length < 0: # ignore nonsensical negative lengths

self.length = None

else:

self.length = None

# does the body have a fixed length? (of zero)

if (status == NO_CONTENT or status == NOT_MODIFIED or

100 <= status < 200 or # 1xx codes

self._method == "HEAD"):

self.length = 0

# if the connection remains open, and we aren't using chunked, and

# a content-length was not provided, then assume that the connection

# WILL close.

if (not self.will_close and

not self.chunked and

self.length is None):

self.will_close = True

def _check_close(self):

conn = self.headers.get("connection")

if self.version == 11:

# An HTTP/1.1 proxy is assumed to stay open unless

# explicitly closed.

conn = self.headers.get("connection")

if conn and "close" in conn.lower():

return True

return False

# Some HTTP/1.0 implementations have support for persistent

# connections, using rules different than HTTP/1.1.

# For older HTTP, Keep-Alive indicates persistent connection.

if self.headers.get("keep-alive"):

return False

# At least Akamai returns a "Connection: Keep-Alive" header,

# which was supposed to be sent by the client.

if conn and "keep-alive" in conn.lower():

return False

# Proxy-Connection is a netscape hack.

pconn = self.headers.get("proxy-connection")

if pconn and "keep-alive" in pconn.lower():

return False

# otherwise, assume it will close

return True

def _close_conn(self):

fp = self.fp

self.fp = None

fp.close()

def close(self):

try:

super().close() # set "closed" flag

finally:

if self.fp:

self._close_conn()

# These implementations are for the benefit of io.BufferedReader.

# XXX This class should probably be revised to act more like

# the "raw stream" that BufferedReader expects.

def flush(self):

super().flush()

if self.fp:

self.fp.flush()

def readable(self):

return True

# End of "raw stream" methods

def isclosed(self):

"""True if the connection is closed."""

# NOTE: it is possible that we will not ever call self.close(). This

# case occurs when will_close is TRUE, length is None, and we

# read up to the last byte, but NOT past it.

#

# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be

# called, meaning self.isclosed() is meaningful.

return self.fp is None

def read(self, amt=None):

if self.fp is None:

return b""

if self._method == "HEAD":

self._close_conn()

return b""

if amt is not None:

# Amount is given, implement using readinto

b = bytearray(amt)

n = self.readinto(b)

return memoryview(b)[:n].tobytes()

else:

# Amount is not given (unbounded read) so we must check self.length

# and self.chunked

if self.chunked:

return self._readall_chunked()

if self.length is None:

s = self.fp.read()

else:

try:

s = self._safe_read(self.length)

except IncompleteRead:

self._close_conn()

raise

self.length = 0

self._close_conn() # we read everything

return s

def readinto(self, b):

if self.fp is None:

return 0

if self._method == "HEAD":

self._close_conn()

return 0

if self.chunked:

return self._readinto_chunked(b)

if self.length is not None:

if len(b) > self.length:

# clip the read to the "end of response"

b = memoryview(b)[0:self.length]

# we do not use _safe_read() here because this may be a .will_close

# connection, and the user is reading more bytes than will be provided

# (for example, reading in 1k chunks)

n = self.fp.readinto(b)

if not n and b:

# Ideally, we would raise IncompleteRead if the content-length

# wasn't satisfied, but it might break compatibility.

self._close_conn()

elif self.length is not None:

self.length -= n

if not self.length:

self._close_conn()

return n

def _read_next_chunk_size(self):

# Read the next chunk size from the file

line = self.fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("chunk size")

i = line.find(b";")

if i >= 0:

line = line[:i] # strip chunk-extensions

try:

return int(line, 16)

except ValueError:

# close the connection as protocol synchronisation is

# probably lost

self._close_conn()

raise

def _read_and_discard_trailer(self):

# read and discard trailer up to the CRLF terminator

### note: we shouldn't have any trailers!

while True:

line = self.fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("trailer line")

if not line:

# a vanishingly small number of sites EOF without

# sending the trailer

break

if line in (b'\r\n', b'\n', b''):

break

def _get_chunk_left(self):

# return self.chunk_left, reading a new chunk if necessary.

# chunk_left == 0: at the end of the current chunk, need to close it

# chunk_left == None: No current chunk, should read next.

# This function returns non-zero or None if the last chunk has

# been read.

chunk_left = self.chunk_left

if not chunk_left: # Can be 0 or None

if chunk_left is not None:

# We are at the end of chunk. dicard chunk end

self._safe_read(2) # toss the CRLF at the end of the chunk

try:

chunk_left = self._read_next_chunk_size()

except ValueError:

raise IncompleteRead(b'')

if chunk_left == 0:

# last chunk: 1*("0") [ chunk-extension ] CRLF

self._read_and_discard_trailer()

# we read everything; close the "file"

self._close_conn()

chunk_left = None

self.chunk_left = chunk_left

return chunk_left

def _readall_chunked(self):

assert self.chunked != _UNKNOWN

value = []

try:

while True:

chunk_left = self._get_chunk_left()

if chunk_left is None:

break

value.append(self._safe_read(chunk_left))

self.chunk_left = 0

return b''.join(value)

except IncompleteRead:

raise IncompleteRead(b''.join(value))

def _readinto_chunked(self, b):

assert self.chunked != _UNKNOWN

total_bytes = 0

mvb = memoryview(b)

try:

while True:

chunk_left = self._get_chunk_left()

if chunk_left is None:

return total_bytes

if len(mvb) <= chunk_left:

n = self._safe_readinto(mvb)

self.chunk_left = chunk_left - n

return total_bytes + n

temp_mvb = mvb[:chunk_left]

n = self._safe_readinto(temp_mvb)

mvb = mvb[n:]

total_bytes += n

self.chunk_left = 0

except IncompleteRead:

raise IncompleteRead(bytes(b[0:total_bytes]))

def _safe_read(self, amt):

"""Read the number of bytes requested, compensating for partial reads.

s = []

while amt > 0:

chunk = self.fp.read(min(amt, MAXAMOUNT))

if not chunk:

raise IncompleteRead(b''.join(s), amt)

s.append(chunk)

amt -= len(chunk)

return b"".join(s)

def _safe_readinto(self, b):

"""Same as _safe_read, but for reading into a buffer."""

total_bytes = 0

mvb = memoryview(b)

while total_bytes < len(b):

if MAXAMOUNT < len(mvb):

temp_mvb = mvb[0:MAXAMOUNT]

n = self.fp.readinto(temp_mvb)

else:

n = self.fp.readinto(mvb)

if not n:

raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b))

mvb = mvb[n:]

total_bytes += n

return total_bytes

def read1(self, n=-1):

"""Read with at most one underlying system call. If at least one

if self.fp is None or self._method == "HEAD":

return b""

if self.chunked:

return self._read1_chunked(n)

if self.length is not None and (n < 0 or n > self.length):

n = self.length

try:

result = self.fp.read1(n)

except ValueError:

if n >= 0:

raise

# some implementations, like BufferedReader, don't support -1

# Read an arbitrarily selected largeish chunk.

result = self.fp.read1(16*1024)

if not result and n:

self._close_conn()

elif self.length is not None:

self.length -= len(result)

return result

def peek(self, n=-1):

# Having this enables IOBase.readline() to read more than one

# byte at a time

if self.fp is None or self._method == "HEAD":

return b""

if self.chunked:

return self._peek_chunked(n)

return self.fp.peek(n)

def readline(self, limit=-1):

if self.fp is None or self._method == "HEAD":

return b""

if self.chunked:

# Fallback to IOBase readline which uses peek() and read()

return super().readline(limit)

if self.length is not None and (limit < 0 or limit > self.length):

limit = self.length

result = self.fp.readline(limit)

if not result and limit:

self._close_conn()

elif self.length is not None:

self.length -= len(result)

return result

def _read1_chunked(self, n):

# Strictly speaking, _get_chunk_left() may cause more than one read,

# but that is ok, since that is to satisfy the chunked protocol.

chunk_left = self._get_chunk_left()

if chunk_left is None or n == 0:

return b''

if not (0 <= n <= chunk_left):

n = chunk_left # if n is negative or larger than chunk_left

read = self.fp.read1(n)

self.chunk_left -= len(read)

if not read:

raise IncompleteRead(b"")

return read

def _peek_chunked(self, n):

# Strictly speaking, _get_chunk_left() may cause more than one read,

# but that is ok, since that is to satisfy the chunked protocol.

try:

chunk_left = self._get_chunk_left()

except IncompleteRead:

return b'' # peek doesn't worry about protocol

if chunk_left is None:

return b'' # eof

# peek is allowed to return more than requested. Just request the

# entire chunk, and truncate what we get.

return self.fp.peek(chunk_left)[:chunk_left]

def fileno(self):

return self.fp.fileno()

def getheader(self, name, default=None):

if self.headers is None:

raise ResponseNotReady()

headers = self.headers.get_all(name) or default

if isinstance(headers, str) or not hasattr(headers, '__iter__'):

return headers

else:

return ', '.join(headers)

def getheaders(self):

"""Return list of (header, value) tuples."""

if self.headers is None:

raise ResponseNotReady()

return list(self.headers.items())

# We override IOBase.__iter__ so that it doesn't check for closed-ness

def __iter__(self):

return self

# For compatibility with old-style urllib responses.

def info(self):

return self.headers

def geturl(self):

return self.url

def getcode(self):

_http_vsn = 11

_http_vsn_str = 'HTTP/1.1'

response_class = HTTPResponse

default_port = HTTP_PORT

auto_open = 1

debuglevel = 0

def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,

source_address=None):

self.timeout = timeout

self.source_address = source_address

self.sock = None

self._buffer = []

self.__response = None

self.__state = _CS_IDLE

self._method = None

self._tunnel_host = None

self._tunnel_port = None

self._tunnel_headers = {}

(self.host, self.port) = self._get_hostport(host, port)

self._validate_host(self.host)

# This is stored as an instance variable to allow unit

# tests to replace it with a suitable mockup

self._create_connection = socket.create_connection

def set_tunnel(self, host, port=None, headers=None):

"""Set up host and port for HTTP CONNECT tunnelling.

if self.sock:

raise RuntimeError("Can't set up tunnel for established connection")

self._tunnel_host, self._tunnel_port = self._get_hostport(host, port)

if headers:

self._tunnel_headers = headers

else:

self._tunnel_headers.clear()

def _get_hostport(self, host, port):

if port is None:

i = host.rfind(':')

j = host.rfind(']') # ipv6 addresses have [...]

if i > j:

try:

port = int(host[i+1:])

except ValueError:

if host[i+1:] == "": # http://foo.com:/ == http://foo.com/

port = self.default_port

else:

raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])

host = host[:i]

else:

port = self.default_port

if host and host[0] == '[' and host[-1] == ']':

host = host[1:-1]

return (host, port)

def set_debuglevel(self, level):

self.debuglevel = level

def _tunnel(self):

connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,

self._tunnel_port)

connect_bytes = connect_str.encode("ascii")

self.send(connect_bytes)

for header, value in self._tunnel_headers.items():

header_str = "%s: %s\r\n" % (header, value)

header_bytes = header_str.encode("latin-1")

self.send(header_bytes)

self.send(b'\r\n')

response = self.response_class(self.sock, method=self._method)

(version, code, message) = response._read_status()

if code != http.HTTPStatus.OK:

self.close()

raise OSError("Tunnel connection failed: %d %s" % (code,

message.strip()))

while True:

line = response.fp.readline(_MAXLINE + 1)

if len(line) > _MAXLINE:

raise LineTooLong("header line")

if not line:

# for sites which EOF without sending a trailer

break

if line in (b'\r\n', b'\n', b''):

break

if self.debuglevel > 0:

print('header:', line.decode())

def connect(self):

"""Connect to the host and port specified in __init__."""

self.sock = self._create_connection(

(self.host,self.port), self.timeout, self.source_address)

self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

if self._tunnel_host:

self._tunnel()

def close(self):

"""Close the connection to the HTTP server."""

self.__state = _CS_IDLE

try:

sock = self.sock

if sock:

self.sock = None

sock.close() # close it manually... there may be other refs

finally:

response = self.__response

if response:

self.__response = None

response.close()

def send(self, data):

"""Send `data' to the server.

if self.sock is None:

if self.auto_open:

self.connect()

else:

raise NotConnected()

if self.debuglevel > 0:

print("send:", repr(data))

blocksize = 8192

if hasattr(data, "read") :

if self.debuglevel > 0:

print("sendIng a read()able")

encode = False

try:

mode = data.mode

except AttributeError:

# io.BytesIO and other file-like objects don't have a `mode`

# attribute.

pass

else:

if "b" not in mode:

encode = True

if self.debuglevel > 0:

print("encoding file using iso-8859-1")

while 1:

datablock = data.read(blocksize)

if not datablock:

break

if encode:

datablock = datablock.encode("iso-8859-1")

self.sock.sendall(datablock)

return

try:

self.sock.sendall(data)

except TypeError:

if isinstance(data, collections.Iterable):

for d in data:

self.sock.sendall(d)

else:

raise TypeError("data should be a bytes-like object "

"or an iterable, got %r" % type(data))

def _output(self, s):

"""Add a line of output to the current request buffer.

self._buffer.append(s)

def _send_output(self, message_body=None):

"""Send the currently buffered request and clear the buffer.

self._buffer.extend((b"", b""))

msg = b"\r\n".join(self._buffer)

del self._buffer[:]

self.send(msg)

if message_body is not None:

self.send(message_body)

def putrequest(self, method, url, skip_host=False,

skip_accept_encoding=False):

"""Send a request to the server.

# if a prior response has been completed, then forget about it.

if self.__response and self.__response.isclosed():

self.__response = None

# in certain cases, we cannot issue another request on this connection.

# this occurs when:

# 1) we are in the process of sending a request. (_CS_REQ_STARTED)

# 2) a response to a previous request has signalled that it is going

# to close the connection upon completion.

# 3) the headers for the previous response have not been read, thus

# we cannot determine whether point (2) is true. (_CS_REQ_SENT)

#

# if there is no prior response, then we can request at will.

#

# if point (2) is true, then we will have passed the socket to the

# response (effectively meaning, "there is no prior response"), and

# will open a new one when a new request is made.

#

# Note: if a prior response exists, then we *can* start a new request.

# We are not allowed to begin fetching the response to this new

# request, however, until that prior response is complete.

#

if self.__state == _CS_IDLE:

self.__state = _CS_REQ_STARTED

else:

raise CannotSendRequest(self.__state)

self._validate_method(method)

# Save the method for use later in the response phase

self._method = method

url = url or '/'

self._validate_path(url)

request = '%s %s %s' % (method, url, self._http_vsn_str)