Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/urllib3/connectionpool.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 from __future__ import absolute_import | |
2 | |
3 import errno | |
4 import logging | |
5 import socket | |
6 import sys | |
7 import warnings | |
8 from socket import error as SocketError | |
9 from socket import timeout as SocketTimeout | |
10 | |
11 from .connection import ( | |
12 BaseSSLError, | |
13 BrokenPipeError, | |
14 DummyConnection, | |
15 HTTPConnection, | |
16 HTTPException, | |
17 HTTPSConnection, | |
18 VerifiedHTTPSConnection, | |
19 port_by_scheme, | |
20 ) | |
21 from .exceptions import ( | |
22 ClosedPoolError, | |
23 EmptyPoolError, | |
24 HeaderParsingError, | |
25 HostChangedError, | |
26 InsecureRequestWarning, | |
27 LocationValueError, | |
28 MaxRetryError, | |
29 NewConnectionError, | |
30 ProtocolError, | |
31 ProxyError, | |
32 ReadTimeoutError, | |
33 SSLError, | |
34 TimeoutError, | |
35 ) | |
36 from .packages import six | |
37 from .packages.six.moves import queue | |
38 from .packages.ssl_match_hostname import CertificateError | |
39 from .request import RequestMethods | |
40 from .response import HTTPResponse | |
41 from .util.connection import is_connection_dropped | |
42 from .util.proxy import connection_requires_http_tunnel | |
43 from .util.queue import LifoQueue | |
44 from .util.request import set_file_position | |
45 from .util.response import assert_header_parsing | |
46 from .util.retry import Retry | |
47 from .util.timeout import Timeout | |
48 from .util.url import Url, _encode_target | |
49 from .util.url import _normalize_host as normalize_host | |
50 from .util.url import get_host, parse_url | |
51 | |
52 xrange = six.moves.xrange | |
53 | |
54 log = logging.getLogger(__name__) | |
55 | |
56 _Default = object() | |
57 | |
58 | |
59 # Pool objects | |
60 class ConnectionPool(object): | |
61 """ | |
62 Base class for all connection pools, such as | |
63 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. | |
64 | |
65 .. note:: | |
66 ConnectionPool.urlopen() does not normalize or percent-encode target URIs | |
67 which is useful if your target server doesn't support percent-encoded | |
68 target URIs. | |
69 """ | |
70 | |
71 scheme = None | |
72 QueueCls = LifoQueue | |
73 | |
74 def __init__(self, host, port=None): | |
75 if not host: | |
76 raise LocationValueError("No host specified.") | |
77 | |
78 self.host = _normalize_host(host, scheme=self.scheme) | |
79 self._proxy_host = host.lower() | |
80 self.port = port | |
81 | |
82 def __str__(self): | |
83 return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) | |
84 | |
85 def __enter__(self): | |
86 return self | |
87 | |
88 def __exit__(self, exc_type, exc_val, exc_tb): | |
89 self.close() | |
90 # Return False to re-raise any potential exceptions | |
91 return False | |
92 | |
93 def close(self): | |
94 """ | |
95 Close all pooled connections and disable the pool. | |
96 """ | |
97 pass | |
98 | |
99 | |
100 # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 | |
101 _blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} | |
102 | |
103 | |
104 class HTTPConnectionPool(ConnectionPool, RequestMethods): | |
105 """ | |
106 Thread-safe connection pool for one host. | |
107 | |
108 :param host: | |
109 Host used for this HTTP Connection (e.g. "localhost"), passed into | |
110 :class:`http.client.HTTPConnection`. | |
111 | |
112 :param port: | |
113 Port used for this HTTP Connection (None is equivalent to 80), passed | |
114 into :class:`http.client.HTTPConnection`. | |
115 | |
116 :param strict: | |
117 Causes BadStatusLine to be raised if the status line can't be parsed | |
118 as a valid HTTP/1.0 or 1.1 status line, passed into | |
119 :class:`http.client.HTTPConnection`. | |
120 | |
121 .. note:: | |
122 Only works in Python 2. This parameter is ignored in Python 3. | |
123 | |
124 :param timeout: | |
125 Socket timeout in seconds for each individual connection. This can | |
126 be a float or integer, which sets the timeout for the HTTP request, | |
127 or an instance of :class:`urllib3.util.Timeout` which gives you more | |
128 fine-grained control over request timeouts. After the constructor has | |
129 been parsed, this is always a `urllib3.util.Timeout` object. | |
130 | |
131 :param maxsize: | |
132 Number of connections to save that can be reused. More than 1 is useful | |
133 in multithreaded situations. If ``block`` is set to False, more | |
134 connections will be created but they will not be saved once they've | |
135 been used. | |
136 | |
137 :param block: | |
138 If set to True, no more than ``maxsize`` connections will be used at | |
139 a time. When no free connections are available, the call will block | |
140 until a connection has been released. This is a useful side effect for | |
141 particular multithreaded situations where one does not want to use more | |
142 than maxsize connections per host to prevent flooding. | |
143 | |
144 :param headers: | |
145 Headers to include with all requests, unless other headers are given | |
146 explicitly. | |
147 | |
148 :param retries: | |
149 Retry configuration to use by default with requests in this pool. | |
150 | |
151 :param _proxy: | |
152 Parsed proxy URL, should not be used directly, instead, see | |
153 :class:`urllib3.ProxyManager` | |
154 | |
155 :param _proxy_headers: | |
156 A dictionary with proxy headers, should not be used directly, | |
157 instead, see :class:`urllib3.ProxyManager` | |
158 | |
159 :param \\**conn_kw: | |
160 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, | |
161 :class:`urllib3.connection.HTTPSConnection` instances. | |
162 """ | |
163 | |
164 scheme = "http" | |
165 ConnectionCls = HTTPConnection | |
166 ResponseCls = HTTPResponse | |
167 | |
168 def __init__( | |
169 self, | |
170 host, | |
171 port=None, | |
172 strict=False, | |
173 timeout=Timeout.DEFAULT_TIMEOUT, | |
174 maxsize=1, | |
175 block=False, | |
176 headers=None, | |
177 retries=None, | |
178 _proxy=None, | |
179 _proxy_headers=None, | |
180 _proxy_config=None, | |
181 **conn_kw | |
182 ): | |
183 ConnectionPool.__init__(self, host, port) | |
184 RequestMethods.__init__(self, headers) | |
185 | |
186 self.strict = strict | |
187 | |
188 if not isinstance(timeout, Timeout): | |
189 timeout = Timeout.from_float(timeout) | |
190 | |
191 if retries is None: | |
192 retries = Retry.DEFAULT | |
193 | |
194 self.timeout = timeout | |
195 self.retries = retries | |
196 | |
197 self.pool = self.QueueCls(maxsize) | |
198 self.block = block | |
199 | |
200 self.proxy = _proxy | |
201 self.proxy_headers = _proxy_headers or {} | |
202 self.proxy_config = _proxy_config | |
203 | |
204 # Fill the queue up so that doing get() on it will block properly | |
205 for _ in xrange(maxsize): | |
206 self.pool.put(None) | |
207 | |
208 # These are mostly for testing and debugging purposes. | |
209 self.num_connections = 0 | |
210 self.num_requests = 0 | |
211 self.conn_kw = conn_kw | |
212 | |
213 if self.proxy: | |
214 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. | |
215 # We cannot know if the user has added default socket options, so we cannot replace the | |
216 # list. | |
217 self.conn_kw.setdefault("socket_options", []) | |
218 | |
219 self.conn_kw["proxy"] = self.proxy | |
220 self.conn_kw["proxy_config"] = self.proxy_config | |
221 | |
222 def _new_conn(self): | |
223 """ | |
224 Return a fresh :class:`HTTPConnection`. | |
225 """ | |
226 self.num_connections += 1 | |
227 log.debug( | |
228 "Starting new HTTP connection (%d): %s:%s", | |
229 self.num_connections, | |
230 self.host, | |
231 self.port or "80", | |
232 ) | |
233 | |
234 conn = self.ConnectionCls( | |
235 host=self.host, | |
236 port=self.port, | |
237 timeout=self.timeout.connect_timeout, | |
238 strict=self.strict, | |
239 **self.conn_kw | |
240 ) | |
241 return conn | |
242 | |
243 def _get_conn(self, timeout=None): | |
244 """ | |
245 Get a connection. Will return a pooled connection if one is available. | |
246 | |
247 If no connections are available and :prop:`.block` is ``False``, then a | |
248 fresh connection is returned. | |
249 | |
250 :param timeout: | |
251 Seconds to wait before giving up and raising | |
252 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and | |
253 :prop:`.block` is ``True``. | |
254 """ | |
255 conn = None | |
256 try: | |
257 conn = self.pool.get(block=self.block, timeout=timeout) | |
258 | |
259 except AttributeError: # self.pool is None | |
260 raise ClosedPoolError(self, "Pool is closed.") | |
261 | |
262 except queue.Empty: | |
263 if self.block: | |
264 raise EmptyPoolError( | |
265 self, | |
266 "Pool reached maximum size and no more connections are allowed.", | |
267 ) | |
268 pass # Oh well, we'll create a new connection then | |
269 | |
270 # If this is a persistent connection, check if it got disconnected | |
271 if conn and is_connection_dropped(conn): | |
272 log.debug("Resetting dropped connection: %s", self.host) | |
273 conn.close() | |
274 if getattr(conn, "auto_open", 1) == 0: | |
275 # This is a proxied connection that has been mutated by | |
276 # http.client._tunnel() and cannot be reused (since it would | |
277 # attempt to bypass the proxy) | |
278 conn = None | |
279 | |
280 return conn or self._new_conn() | |
281 | |
282 def _put_conn(self, conn): | |
283 """ | |
284 Put a connection back into the pool. | |
285 | |
286 :param conn: | |
287 Connection object for the current host and port as returned by | |
288 :meth:`._new_conn` or :meth:`._get_conn`. | |
289 | |
290 If the pool is already full, the connection is closed and discarded | |
291 because we exceeded maxsize. If connections are discarded frequently, | |
292 then maxsize should be increased. | |
293 | |
294 If the pool is closed, then the connection will be closed and discarded. | |
295 """ | |
296 try: | |
297 self.pool.put(conn, block=False) | |
298 return # Everything is dandy, done. | |
299 except AttributeError: | |
300 # self.pool is None. | |
301 pass | |
302 except queue.Full: | |
303 # This should never happen if self.block == True | |
304 log.warning("Connection pool is full, discarding connection: %s", self.host) | |
305 | |
306 # Connection never got put back into the pool, close it. | |
307 if conn: | |
308 conn.close() | |
309 | |
310 def _validate_conn(self, conn): | |
311 """ | |
312 Called right before a request is made, after the socket is created. | |
313 """ | |
314 pass | |
315 | |
316 def _prepare_proxy(self, conn): | |
317 # Nothing to do for HTTP connections. | |
318 pass | |
319 | |
320 def _get_timeout(self, timeout): | |
321 """ Helper that always returns a :class:`urllib3.util.Timeout` """ | |
322 if timeout is _Default: | |
323 return self.timeout.clone() | |
324 | |
325 if isinstance(timeout, Timeout): | |
326 return timeout.clone() | |
327 else: | |
328 # User passed us an int/float. This is for backwards compatibility, | |
329 # can be removed later | |
330 return Timeout.from_float(timeout) | |
331 | |
332 def _raise_timeout(self, err, url, timeout_value): | |
333 """Is the error actually a timeout? Will raise a ReadTimeout or pass""" | |
334 | |
335 if isinstance(err, SocketTimeout): | |
336 raise ReadTimeoutError( | |
337 self, url, "Read timed out. (read timeout=%s)" % timeout_value | |
338 ) | |
339 | |
340 # See the above comment about EAGAIN in Python 3. In Python 2 we have | |
341 # to specifically catch it and throw the timeout error | |
342 if hasattr(err, "errno") and err.errno in _blocking_errnos: | |
343 raise ReadTimeoutError( | |
344 self, url, "Read timed out. (read timeout=%s)" % timeout_value | |
345 ) | |
346 | |
347 # Catch possible read timeouts thrown as SSL errors. If not the | |
348 # case, rethrow the original. We need to do this because of: | |
349 # http://bugs.python.org/issue10272 | |
350 if "timed out" in str(err) or "did not complete (read)" in str( | |
351 err | |
352 ): # Python < 2.7.4 | |
353 raise ReadTimeoutError( | |
354 self, url, "Read timed out. (read timeout=%s)" % timeout_value | |
355 ) | |
356 | |
357 def _make_request( | |
358 self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw | |
359 ): | |
360 """ | |
361 Perform a request on a given urllib connection object taken from our | |
362 pool. | |
363 | |
364 :param conn: | |
365 a connection from one of our connection pools | |
366 | |
367 :param timeout: | |
368 Socket timeout in seconds for the request. This can be a | |
369 float or integer, which will set the same timeout value for | |
370 the socket connect and the socket read, or an instance of | |
371 :class:`urllib3.util.Timeout`, which gives you more fine-grained | |
372 control over your timeouts. | |
373 """ | |
374 self.num_requests += 1 | |
375 | |
376 timeout_obj = self._get_timeout(timeout) | |
377 timeout_obj.start_connect() | |
378 conn.timeout = timeout_obj.connect_timeout | |
379 | |
380 # Trigger any extra validation we need to do. | |
381 try: | |
382 self._validate_conn(conn) | |
383 except (SocketTimeout, BaseSSLError) as e: | |
384 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. | |
385 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) | |
386 raise | |
387 | |
388 # conn.request() calls http.client.*.request, not the method in | |
389 # urllib3.request. It also calls makefile (recv) on the socket. | |
390 try: | |
391 if chunked: | |
392 conn.request_chunked(method, url, **httplib_request_kw) | |
393 else: | |
394 conn.request(method, url, **httplib_request_kw) | |
395 | |
396 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is | |
397 # legitimately able to close the connection after sending a valid response. | |
398 # With this behaviour, the received response is still readable. | |
399 except BrokenPipeError: | |
400 # Python 3 | |
401 pass | |
402 except IOError as e: | |
403 # Python 2 and macOS/Linux | |
404 # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS | |
405 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ | |
406 if e.errno not in { | |
407 errno.EPIPE, | |
408 errno.ESHUTDOWN, | |
409 errno.EPROTOTYPE, | |
410 }: | |
411 raise | |
412 | |
413 # Reset the timeout for the recv() on the socket | |
414 read_timeout = timeout_obj.read_timeout | |
415 | |
416 # App Engine doesn't have a sock attr | |
417 if getattr(conn, "sock", None): | |
418 # In Python 3 socket.py will catch EAGAIN and return None when you | |
419 # try and read into the file pointer created by http.client, which | |
420 # instead raises a BadStatusLine exception. Instead of catching | |
421 # the exception and assuming all BadStatusLine exceptions are read | |
422 # timeouts, check for a zero timeout before making the request. | |
423 if read_timeout == 0: | |
424 raise ReadTimeoutError( | |
425 self, url, "Read timed out. (read timeout=%s)" % read_timeout | |
426 ) | |
427 if read_timeout is Timeout.DEFAULT_TIMEOUT: | |
428 conn.sock.settimeout(socket.getdefaulttimeout()) | |
429 else: # None or a value | |
430 conn.sock.settimeout(read_timeout) | |
431 | |
432 # Receive the response from the server | |
433 try: | |
434 try: | |
435 # Python 2.7, use buffering of HTTP responses | |
436 httplib_response = conn.getresponse(buffering=True) | |
437 except TypeError: | |
438 # Python 3 | |
439 try: | |
440 httplib_response = conn.getresponse() | |
441 except BaseException as e: | |
442 # Remove the TypeError from the exception chain in | |
443 # Python 3 (including for exceptions like SystemExit). | |
444 # Otherwise it looks like a bug in the code. | |
445 six.raise_from(e, None) | |
446 except (SocketTimeout, BaseSSLError, SocketError) as e: | |
447 self._raise_timeout(err=e, url=url, timeout_value=read_timeout) | |
448 raise | |
449 | |
450 # AppEngine doesn't have a version attr. | |
451 http_version = getattr(conn, "_http_vsn_str", "HTTP/?") | |
452 log.debug( | |
453 '%s://%s:%s "%s %s %s" %s %s', | |
454 self.scheme, | |
455 self.host, | |
456 self.port, | |
457 method, | |
458 url, | |
459 http_version, | |
460 httplib_response.status, | |
461 httplib_response.length, | |
462 ) | |
463 | |
464 try: | |
465 assert_header_parsing(httplib_response.msg) | |
466 except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 | |
467 log.warning( | |
468 "Failed to parse headers (url=%s): %s", | |
469 self._absolute_url(url), | |
470 hpe, | |
471 exc_info=True, | |
472 ) | |
473 | |
474 return httplib_response | |
475 | |
476 def _absolute_url(self, path): | |
477 return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url | |
478 | |
479 def close(self): | |
480 """ | |
481 Close all pooled connections and disable the pool. | |
482 """ | |
483 if self.pool is None: | |
484 return | |
485 # Disable access to the pool | |
486 old_pool, self.pool = self.pool, None | |
487 | |
488 try: | |
489 while True: | |
490 conn = old_pool.get(block=False) | |
491 if conn: | |
492 conn.close() | |
493 | |
494 except queue.Empty: | |
495 pass # Done. | |
496 | |
497 def is_same_host(self, url): | |
498 """ | |
499 Check if the given ``url`` is a member of the same host as this | |
500 connection pool. | |
501 """ | |
502 if url.startswith("/"): | |
503 return True | |
504 | |
505 # TODO: Add optional support for socket.gethostbyname checking. | |
506 scheme, host, port = get_host(url) | |
507 if host is not None: | |
508 host = _normalize_host(host, scheme=scheme) | |
509 | |
510 # Use explicit default port for comparison when none is given | |
511 if self.port and not port: | |
512 port = port_by_scheme.get(scheme) | |
513 elif not self.port and port == port_by_scheme.get(scheme): | |
514 port = None | |
515 | |
516 return (scheme, host, port) == (self.scheme, self.host, self.port) | |
517 | |
518 def urlopen( | |
519 self, | |
520 method, | |
521 url, | |
522 body=None, | |
523 headers=None, | |
524 retries=None, | |
525 redirect=True, | |
526 assert_same_host=True, | |
527 timeout=_Default, | |
528 pool_timeout=None, | |
529 release_conn=None, | |
530 chunked=False, | |
531 body_pos=None, | |
532 **response_kw | |
533 ): | |
534 """ | |
535 Get a connection from the pool and perform an HTTP request. This is the | |
536 lowest level call for making a request, so you'll need to specify all | |
537 the raw details. | |
538 | |
539 .. note:: | |
540 | |
541 More commonly, it's appropriate to use a convenience method provided | |
542 by :class:`.RequestMethods`, such as :meth:`request`. | |
543 | |
544 .. note:: | |
545 | |
546 `release_conn` will only behave as expected if | |
547 `preload_content=False` because we want to make | |
548 `preload_content=False` the default behaviour someday soon without | |
549 breaking backwards compatibility. | |
550 | |
551 :param method: | |
552 HTTP request method (such as GET, POST, PUT, etc.) | |
553 | |
554 :param url: | |
555 The URL to perform the request on. | |
556 | |
557 :param body: | |
558 Data to send in the request body, either :class:`str`, :class:`bytes`, | |
559 an iterable of :class:`str`/:class:`bytes`, or a file-like object. | |
560 | |
561 :param headers: | |
562 Dictionary of custom headers to send, such as User-Agent, | |
563 If-None-Match, etc. If None, pool headers are used. If provided, | |
564 these headers completely replace any pool-specific headers. | |
565 | |
566 :param retries: | |
567 Configure the number of retries to allow before raising a | |
568 :class:`~urllib3.exceptions.MaxRetryError` exception. | |
569 | |
570 Pass ``None`` to retry until you receive a response. Pass a | |
571 :class:`~urllib3.util.retry.Retry` object for fine-grained control | |
572 over different types of retries. | |
573 Pass an integer number to retry connection errors that many times, | |
574 but no other types of errors. Pass zero to never retry. | |
575 | |
576 If ``False``, then retries are disabled and any exception is raised | |
577 immediately. Also, instead of raising a MaxRetryError on redirects, | |
578 the redirect response will be returned. | |
579 | |
580 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. | |
581 | |
582 :param redirect: | |
583 If True, automatically handle redirects (status codes 301, 302, | |
584 303, 307, 308). Each redirect counts as a retry. Disabling retries | |
585 will disable redirect, too. | |
586 | |
587 :param assert_same_host: | |
588 If ``True``, will make sure that the host of the pool requests is | |
589 consistent else will raise HostChangedError. When ``False``, you can | |
590 use the pool on an HTTP proxy and request foreign hosts. | |
591 | |
592 :param timeout: | |
593 If specified, overrides the default timeout for this one | |
594 request. It may be a float (in seconds) or an instance of | |
595 :class:`urllib3.util.Timeout`. | |
596 | |
597 :param pool_timeout: | |
598 If set and the pool is set to block=True, then this method will | |
599 block for ``pool_timeout`` seconds and raise EmptyPoolError if no | |
600 connection is available within the time period. | |
601 | |
602 :param release_conn: | |
603 If False, then the urlopen call will not release the connection | |
604 back into the pool once a response is received (but will release if | |
605 you read the entire contents of the response such as when | |
606 `preload_content=True`). This is useful if you're not preloading | |
607 the response's content immediately. You will need to call | |
608 ``r.release_conn()`` on the response ``r`` to return the connection | |
609 back into the pool. If None, it takes the value of | |
610 ``response_kw.get('preload_content', True)``. | |
611 | |
612 :param chunked: | |
613 If True, urllib3 will send the body using chunked transfer | |
614 encoding. Otherwise, urllib3 will send the body using the standard | |
615 content-length form. Defaults to False. | |
616 | |
617 :param int body_pos: | |
618 Position to seek to in file-like body in the event of a retry or | |
619 redirect. Typically this won't need to be set because urllib3 will | |
620 auto-populate the value when needed. | |
621 | |
622 :param \\**response_kw: | |
623 Additional parameters are passed to | |
624 :meth:`urllib3.response.HTTPResponse.from_httplib` | |
625 """ | |
626 | |
627 parsed_url = parse_url(url) | |
628 destination_scheme = parsed_url.scheme | |
629 | |
630 if headers is None: | |
631 headers = self.headers | |
632 | |
633 if not isinstance(retries, Retry): | |
634 retries = Retry.from_int(retries, redirect=redirect, default=self.retries) | |
635 | |
636 if release_conn is None: | |
637 release_conn = response_kw.get("preload_content", True) | |
638 | |
639 # Check host | |
640 if assert_same_host and not self.is_same_host(url): | |
641 raise HostChangedError(self, url, retries) | |
642 | |
643 # Ensure that the URL we're connecting to is properly encoded | |
644 if url.startswith("/"): | |
645 url = six.ensure_str(_encode_target(url)) | |
646 else: | |
647 url = six.ensure_str(parsed_url.url) | |
648 | |
649 conn = None | |
650 | |
651 # Track whether `conn` needs to be released before | |
652 # returning/raising/recursing. Update this variable if necessary, and | |
653 # leave `release_conn` constant throughout the function. That way, if | |
654 # the function recurses, the original value of `release_conn` will be | |
655 # passed down into the recursive call, and its value will be respected. | |
656 # | |
657 # See issue #651 [1] for details. | |
658 # | |
659 # [1] <https://github.com/urllib3/urllib3/issues/651> | |
660 release_this_conn = release_conn | |
661 | |
662 http_tunnel_required = connection_requires_http_tunnel( | |
663 self.proxy, self.proxy_config, destination_scheme | |
664 ) | |
665 | |
666 # Merge the proxy headers. Only done when not using HTTP CONNECT. We | |
667 # have to copy the headers dict so we can safely change it without those | |
668 # changes being reflected in anyone else's copy. | |
669 if not http_tunnel_required: | |
670 headers = headers.copy() | |
671 headers.update(self.proxy_headers) | |
672 | |
673 # Must keep the exception bound to a separate variable or else Python 3 | |
674 # complains about UnboundLocalError. | |
675 err = None | |
676 | |
677 # Keep track of whether we cleanly exited the except block. This | |
678 # ensures we do proper cleanup in finally. | |
679 clean_exit = False | |
680 | |
681 # Rewind body position, if needed. Record current position | |
682 # for future rewinds in the event of a redirect/retry. | |
683 body_pos = set_file_position(body, body_pos) | |
684 | |
685 try: | |
686 # Request a connection from the queue. | |
687 timeout_obj = self._get_timeout(timeout) | |
688 conn = self._get_conn(timeout=pool_timeout) | |
689 | |
690 conn.timeout = timeout_obj.connect_timeout | |
691 | |
692 is_new_proxy_conn = self.proxy is not None and not getattr( | |
693 conn, "sock", None | |
694 ) | |
695 if is_new_proxy_conn and http_tunnel_required: | |
696 self._prepare_proxy(conn) | |
697 | |
698 # Make the request on the httplib connection object. | |
699 httplib_response = self._make_request( | |
700 conn, | |
701 method, | |
702 url, | |
703 timeout=timeout_obj, | |
704 body=body, | |
705 headers=headers, | |
706 chunked=chunked, | |
707 ) | |
708 | |
709 # If we're going to release the connection in ``finally:``, then | |
710 # the response doesn't need to know about the connection. Otherwise | |
711 # it will also try to release it and we'll have a double-release | |
712 # mess. | |
713 response_conn = conn if not release_conn else None | |
714 | |
715 # Pass method to Response for length checking | |
716 response_kw["request_method"] = method | |
717 | |
718 # Import httplib's response into our own wrapper object | |
719 response = self.ResponseCls.from_httplib( | |
720 httplib_response, | |
721 pool=self, | |
722 connection=response_conn, | |
723 retries=retries, | |
724 **response_kw | |
725 ) | |
726 | |
727 # Everything went great! | |
728 clean_exit = True | |
729 | |
730 except EmptyPoolError: | |
731 # Didn't get a connection from the pool, no need to clean up | |
732 clean_exit = True | |
733 release_this_conn = False | |
734 raise | |
735 | |
736 except ( | |
737 TimeoutError, | |
738 HTTPException, | |
739 SocketError, | |
740 ProtocolError, | |
741 BaseSSLError, | |
742 SSLError, | |
743 CertificateError, | |
744 ) as e: | |
745 # Discard the connection for these exceptions. It will be | |
746 # replaced during the next _get_conn() call. | |
747 clean_exit = False | |
748 if isinstance(e, (BaseSSLError, CertificateError)): | |
749 e = SSLError(e) | |
750 elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: | |
751 e = ProxyError("Cannot connect to proxy.", e) | |
752 elif isinstance(e, (SocketError, HTTPException)): | |
753 e = ProtocolError("Connection aborted.", e) | |
754 | |
755 retries = retries.increment( | |
756 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] | |
757 ) | |
758 retries.sleep() | |
759 | |
760 # Keep track of the error for the retry warning. | |
761 err = e | |
762 | |
763 finally: | |
764 if not clean_exit: | |
765 # We hit some kind of exception, handled or otherwise. We need | |
766 # to throw the connection away unless explicitly told not to. | |
767 # Close the connection, set the variable to None, and make sure | |
768 # we put the None back in the pool to avoid leaking it. | |
769 conn = conn and conn.close() | |
770 release_this_conn = True | |
771 | |
772 if release_this_conn: | |
773 # Put the connection back to be reused. If the connection is | |
774 # expired then it will be None, which will get replaced with a | |
775 # fresh connection during _get_conn. | |
776 self._put_conn(conn) | |
777 | |
778 if not conn: | |
779 # Try again | |
780 log.warning( | |
781 "Retrying (%r) after connection broken by '%r': %s", retries, err, url | |
782 ) | |
783 return self.urlopen( | |
784 method, | |
785 url, | |
786 body, | |
787 headers, | |
788 retries, | |
789 redirect, | |
790 assert_same_host, | |
791 timeout=timeout, | |
792 pool_timeout=pool_timeout, | |
793 release_conn=release_conn, | |
794 chunked=chunked, | |
795 body_pos=body_pos, | |
796 **response_kw | |
797 ) | |
798 | |
799 # Handle redirect? | |
800 redirect_location = redirect and response.get_redirect_location() | |
801 if redirect_location: | |
802 if response.status == 303: | |
803 method = "GET" | |
804 | |
805 try: | |
806 retries = retries.increment(method, url, response=response, _pool=self) | |
807 except MaxRetryError: | |
808 if retries.raise_on_redirect: | |
809 response.drain_conn() | |
810 raise | |
811 return response | |
812 | |
813 response.drain_conn() | |
814 retries.sleep_for_retry(response) | |
815 log.debug("Redirecting %s -> %s", url, redirect_location) | |
816 return self.urlopen( | |
817 method, | |
818 redirect_location, | |
819 body, | |
820 headers, | |
821 retries=retries, | |
822 redirect=redirect, | |
823 assert_same_host=assert_same_host, | |
824 timeout=timeout, | |
825 pool_timeout=pool_timeout, | |
826 release_conn=release_conn, | |
827 chunked=chunked, | |
828 body_pos=body_pos, | |
829 **response_kw | |
830 ) | |
831 | |
832 # Check if we should retry the HTTP response. | |
833 has_retry_after = bool(response.getheader("Retry-After")) | |
834 if retries.is_retry(method, response.status, has_retry_after): | |
835 try: | |
836 retries = retries.increment(method, url, response=response, _pool=self) | |
837 except MaxRetryError: | |
838 if retries.raise_on_status: | |
839 response.drain_conn() | |
840 raise | |
841 return response | |
842 | |
843 response.drain_conn() | |
844 retries.sleep(response) | |
845 log.debug("Retry: %s", url) | |
846 return self.urlopen( | |
847 method, | |
848 url, | |
849 body, | |
850 headers, | |
851 retries=retries, | |
852 redirect=redirect, | |
853 assert_same_host=assert_same_host, | |
854 timeout=timeout, | |
855 pool_timeout=pool_timeout, | |
856 release_conn=release_conn, | |
857 chunked=chunked, | |
858 body_pos=body_pos, | |
859 **response_kw | |
860 ) | |
861 | |
862 return response | |
863 | |
864 | |
865 class HTTPSConnectionPool(HTTPConnectionPool): | |
866 """ | |
867 Same as :class:`.HTTPConnectionPool`, but HTTPS. | |
868 | |
869 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``, | |
870 ``assert_hostname`` and ``host`` in this order to verify connections. | |
871 If ``assert_hostname`` is False, no verification is done. | |
872 | |
873 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, | |
874 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl` | |
875 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade | |
876 the connection socket into an SSL socket. | |
877 """ | |
878 | |
879 scheme = "https" | |
880 ConnectionCls = HTTPSConnection | |
881 | |
882 def __init__( | |
883 self, | |
884 host, | |
885 port=None, | |
886 strict=False, | |
887 timeout=Timeout.DEFAULT_TIMEOUT, | |
888 maxsize=1, | |
889 block=False, | |
890 headers=None, | |
891 retries=None, | |
892 _proxy=None, | |
893 _proxy_headers=None, | |
894 key_file=None, | |
895 cert_file=None, | |
896 cert_reqs=None, | |
897 key_password=None, | |
898 ca_certs=None, | |
899 ssl_version=None, | |
900 assert_hostname=None, | |
901 assert_fingerprint=None, | |
902 ca_cert_dir=None, | |
903 **conn_kw | |
904 ): | |
905 | |
906 HTTPConnectionPool.__init__( | |
907 self, | |
908 host, | |
909 port, | |
910 strict, | |
911 timeout, | |
912 maxsize, | |
913 block, | |
914 headers, | |
915 retries, | |
916 _proxy, | |
917 _proxy_headers, | |
918 **conn_kw | |
919 ) | |
920 | |
921 self.key_file = key_file | |
922 self.cert_file = cert_file | |
923 self.cert_reqs = cert_reqs | |
924 self.key_password = key_password | |
925 self.ca_certs = ca_certs | |
926 self.ca_cert_dir = ca_cert_dir | |
927 self.ssl_version = ssl_version | |
928 self.assert_hostname = assert_hostname | |
929 self.assert_fingerprint = assert_fingerprint | |
930 | |
931 def _prepare_conn(self, conn): | |
932 """ | |
933 Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket` | |
934 and establish the tunnel if proxy is used. | |
935 """ | |
936 | |
937 if isinstance(conn, VerifiedHTTPSConnection): | |
938 conn.set_cert( | |
939 key_file=self.key_file, | |
940 key_password=self.key_password, | |
941 cert_file=self.cert_file, | |
942 cert_reqs=self.cert_reqs, | |
943 ca_certs=self.ca_certs, | |
944 ca_cert_dir=self.ca_cert_dir, | |
945 assert_hostname=self.assert_hostname, | |
946 assert_fingerprint=self.assert_fingerprint, | |
947 ) | |
948 conn.ssl_version = self.ssl_version | |
949 return conn | |
950 | |
951 def _prepare_proxy(self, conn): | |
952 """ | |
953 Establishes a tunnel connection through HTTP CONNECT. | |
954 | |
955 Tunnel connection is established early because otherwise httplib would | |
956 improperly set Host: header to proxy's IP:port. | |
957 """ | |
958 | |
959 conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) | |
960 | |
961 if self.proxy.scheme == "https": | |
962 conn.tls_in_tls_required = True | |
963 | |
964 conn.connect() | |
965 | |
966 def _new_conn(self): | |
967 """ | |
968 Return a fresh :class:`http.client.HTTPSConnection`. | |
969 """ | |
970 self.num_connections += 1 | |
971 log.debug( | |
972 "Starting new HTTPS connection (%d): %s:%s", | |
973 self.num_connections, | |
974 self.host, | |
975 self.port or "443", | |
976 ) | |
977 | |
978 if not self.ConnectionCls or self.ConnectionCls is DummyConnection: | |
979 raise SSLError( | |
980 "Can't connect to HTTPS URL because the SSL module is not available." | |
981 ) | |
982 | |
983 actual_host = self.host | |
984 actual_port = self.port | |
985 if self.proxy is not None: | |
986 actual_host = self.proxy.host | |
987 actual_port = self.proxy.port | |
988 | |
989 conn = self.ConnectionCls( | |
990 host=actual_host, | |
991 port=actual_port, | |
992 timeout=self.timeout.connect_timeout, | |
993 strict=self.strict, | |
994 cert_file=self.cert_file, | |
995 key_file=self.key_file, | |
996 key_password=self.key_password, | |
997 **self.conn_kw | |
998 ) | |
999 | |
1000 return self._prepare_conn(conn) | |
1001 | |
1002 def _validate_conn(self, conn): | |
1003 """ | |
1004 Called right before a request is made, after the socket is created. | |
1005 """ | |
1006 super(HTTPSConnectionPool, self)._validate_conn(conn) | |
1007 | |
1008 # Force connect early to allow us to validate the connection. | |
1009 if not getattr(conn, "sock", None): # AppEngine might not have `.sock` | |
1010 conn.connect() | |
1011 | |
1012 if not conn.is_verified: | |
1013 warnings.warn( | |
1014 ( | |
1015 "Unverified HTTPS request is being made to host '%s'. " | |
1016 "Adding certificate verification is strongly advised. See: " | |
1017 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" | |
1018 "#ssl-warnings" % conn.host | |
1019 ), | |
1020 InsecureRequestWarning, | |
1021 ) | |
1022 | |
1023 | |
1024 def connection_from_url(url, **kw): | |
1025 """ | |
1026 Given a url, return an :class:`.ConnectionPool` instance of its host. | |
1027 | |
1028 This is a shortcut for not having to parse out the scheme, host, and port | |
1029 of the url before creating an :class:`.ConnectionPool` instance. | |
1030 | |
1031 :param url: | |
1032 Absolute URL string that must include the scheme. Port is optional. | |
1033 | |
1034 :param \\**kw: | |
1035 Passes additional parameters to the constructor of the appropriate | |
1036 :class:`.ConnectionPool`. Useful for specifying things like | |
1037 timeout, maxsize, headers, etc. | |
1038 | |
1039 Example:: | |
1040 | |
1041 >>> conn = connection_from_url('http://google.com/') | |
1042 >>> r = conn.request('GET', '/') | |
1043 """ | |
1044 scheme, host, port = get_host(url) | |
1045 port = port or port_by_scheme.get(scheme, 80) | |
1046 if scheme == "https": | |
1047 return HTTPSConnectionPool(host, port=port, **kw) | |
1048 else: | |
1049 return HTTPConnectionPool(host, port=port, **kw) | |
1050 | |
1051 | |
1052 def _normalize_host(host, scheme): | |
1053 """ | |
1054 Normalize hosts for comparisons and use with sockets. | |
1055 """ | |
1056 | |
1057 host = normalize_host(host, scheme) | |
1058 | |
1059 # httplib doesn't like it when we include brackets in IPv6 addresses | |
1060 # Specifically, if we include brackets but also pass the port then | |
1061 # httplib crazily doubles up the square brackets on the Host header. | |
1062 # Instead, we need to make sure we never pass ``None`` as the port. | |
1063 # However, for backward compatibility reasons we can't actually | |
1064 # *assert* that. See http://bugs.python.org/issue28539 | |
1065 if host.startswith("[") and host.endswith("]"): | |
1066 host = host[1:-1] | |
1067 return host |