Disable default max headers limit and add --max-headers (closes #802)

CHANGELOG.rst

@@ -9,6 +9,9 @@ This project adheres to `Semantic Versioning <http://semver.org/>`_.
 `2.0.0-dev`_ (unreleased)
 -------------------------
 * Removed Python 2.7 support (`EOL Jan 2020 <https://www.python.org/dev/peps/pep-0373/>`_).
+* Removed Python’s default limit of 100 response headers.
+* Added ``--max-headers`` to allow setting the max header limit.
+
 
 `1.0.3`_ (2019-08-26)
 -------------------------

README.rst

@@ -646,6 +646,19 @@ To send a header with an empty value, use ``Header;``:
     $ http httpbin.org/headers 'Header;'
 
 
+Limiting response headers
+-------------------------
+
+The ``--max-headers=n`` options allows you to control the number of headers
+HTTPie tries reads before giving up (the default 0, i.e., there’s no limit).
+
+
+.. code-block:: bash
+
+    $ http --max-headers=100 httpbin.org/get
+
+
+
 Cookies
 =======
 

httpie/cli.py

@@ -512,6 +512,17 @@ network.add_argument(
     """
 )
 
+network.add_argument(
+    '--max-headers',
+    type=int,
+    default=0,
+    help="""
+    The maximum number of response headers to be read before giving up
+    (default 0, i.e., no limit).
+
+    """
+)
+
 network.add_argument(
     '--timeout',
     type=float,

httpie/client.py

@@ -1,7 +1,9 @@
 import json
 import sys
 
+import http.client
 import requests
+from decorator import contextmanager
 from requests.adapters import HTTPAdapter
 from requests.structures import CaseInsensitiveDict
 
@@ -30,6 +32,18 @@ JSON_ACCEPT = '{0}, */*'.format(JSON_CONTENT_TYPE)
 DEFAULT_UA = 'HTTPie/%s' % __version__
 
 
+# noinspection PyProtectedMember
+@contextmanager
+def max_headers(limit):
+    # <https://github.com/jakubroztocil/httpie/issues/802>
+    orig = http.client._MAXHEADERS
+    http.client._MAXHEADERS = limit or float('Inf')
+    try:
+        yield
+    finally:
+        http.client._MAXHEADERS = orig
+
+
 class HTTPieHTTPAdapter(HTTPAdapter):
 
     def __init__(self, ssl_version=None, **kwargs):
@@ -64,19 +78,20 @@ def get_response(args, config_dir):
     requests_session = get_requests_session(ssl_version)
     requests_session.max_redirects = args.max_redirects
 
-    if not args.session and not args.session_read_only:
-        kwargs = get_requests_kwargs(args)
-        if args.debug:
-            dump_request(kwargs)
-        response = requests_session.request(**kwargs)
-    else:
-        response = sessions.get_response(
-            requests_session=requests_session,
-            args=args,
-            config_dir=config_dir,
-            session_name=args.session or args.session_read_only,
-            read_only=bool(args.session_read_only),
-        )
+    with max_headers(args.max_headers):
+        if not args.session and not args.session_read_only:
+            kwargs = get_requests_kwargs(args)
+            if args.debug:
+                dump_request(kwargs)
+            response = requests_session.request(**kwargs)
+        else:
+            response = sessions.get_response(
+                requests_session=requests_session,
+                args=args,
+                config_dir=config_dir,
+                session_name=args.session or args.session_read_only,
+                read_only=bool(args.session_read_only),
+            )
 
     return response
 

tests/test_errors.py

@@ -5,6 +5,8 @@ from requests.exceptions import ConnectionError
 
 from httpie import ExitStatus
 from httpie.core import main
+from utils import http, HTTP_OK
+
 
 error_msg = None
 
@@ -47,3 +49,13 @@ def test_timeout(get_response):
     ret = main(['--ignore-stdin', 'www.google.com'], custom_log_error=error)
     assert ret == ExitStatus.ERROR_TIMEOUT
     assert error_msg == 'Request timed out (30s).'
+
+
+def test_max_headers_limit(httpbin_both):
+    with raises(ConnectionError) as e:
+        http('--max-headers=1', httpbin_both + '/get')
+    assert 'got more than 1 headers' in str(e.value)
+
+
+def test_max_headers_no_limit(httpbin_both):
+    assert HTTP_OK in http('--max-headers=0', httpbin_both + '/get')