Preserve individual headers with the same name on responses (#1208)

* Preserve individual headers with the same name on responses

* Rename RequestHeadersDict to HTTPHeadersDict

* Update tests/utils/http_server.py

* Update tests/utils/http_server.py

* Update httpie/adapters.py

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

httpie/adapters.py

@@ -0,0 +1,13 @@
+from httpie.cli.dicts import HTTPHeadersDict
+from requests.adapters import HTTPAdapter
+
+
+class HTTPieHTTPAdapter(HTTPAdapter):
+
+    def build_response(self, req, resp):
+        """Wrap the original headers with the `HTTPHeadersDict`
+        to preserve multiple headers that have the same name"""
+
+        response = super().build_response(req, resp)
+        response.headers = HTTPHeadersDict(getattr(resp, 'headers', {}))
+        return response

httpie/cli/dicts.py

@@ -9,7 +9,7 @@ class BaseMultiDict(MultiDict):
     """
 
 
-class RequestHeadersDict(CIMultiDict, BaseMultiDict):
+class HTTPHeadersDict(CIMultiDict, BaseMultiDict):
     """
     Headers are case-insensitive and multiple values are supported
     through the `add()` API.

httpie/cli/requestitems.py

@@ -11,7 +11,7 @@ from .constants import (
 )
 from .dicts import (
     BaseMultiDict, MultipartRequestDataDict, RequestDataDict,
-    RequestFilesDict, RequestHeadersDict, RequestJSONDataDict,
+    RequestFilesDict, HTTPHeadersDict, RequestJSONDataDict,
     RequestQueryParamsDict,
 )
 from .exceptions import ParseError
@@ -21,7 +21,7 @@ from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys
 class RequestItems:
 
     def __init__(self, as_form=False):
-        self.headers = RequestHeadersDict()
+        self.headers = HTTPHeadersDict()
         self.data = RequestDataDict() if as_form else RequestJSONDataDict()
         self.files = RequestFilesDict()
         self.params = RequestQueryParamsDict()

httpie/client.py

@@ -11,7 +11,8 @@ import requests
 # noinspection PyPackageRequirements
 import urllib3
 from . import __version__
-from .cli.dicts import RequestHeadersDict
+from .adapters import HTTPieHTTPAdapter
+from .cli.dicts import HTTPHeadersDict
 from .encoding import UTF8
 from .plugins.registry import plugin_manager
 from .sessions import get_httpie_session
@@ -153,6 +154,7 @@ def build_requests_session(
     requests_session = requests.Session()
 
     # Install our adapter.
+    http_adapter = HTTPieHTTPAdapter()
     https_adapter = HTTPieHTTPSAdapter(
         ciphers=ciphers,
         verify=verify,
@@ -161,6 +163,7 @@ def build_requests_session(
             if ssl_version else None
         ),
     )
+    requests_session.mount('http://', http_adapter)
     requests_session.mount('https://', https_adapter)
 
     # Install adapters from plugins.
@@ -179,8 +182,8 @@ def dump_request(kwargs: dict):
         f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n')
 
 
-def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict:
-    final_headers = RequestHeadersDict()
+def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict:
+    final_headers = HTTPHeadersDict()
     for name, value in headers.items():
         if value is not None:
             # “leading or trailing LWS MAY be removed without
@@ -197,13 +200,13 @@ def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict:
 
 def apply_missing_repeated_headers(
     prepared_request: requests.PreparedRequest,
-    original_headers: RequestHeadersDict
+    original_headers: HTTPHeadersDict
 ) -> None:
     """Update the given `prepared_request`'s headers with the original
     ones. This allows the requests to be prepared as usual, and then later
     merged with headers that are specified multiple times."""
 
-    new_headers = RequestHeadersDict(prepared_request.headers)
+    new_headers = HTTPHeadersDict(prepared_request.headers)
     for prepared_name, prepared_value in prepared_request.headers.items():
         if prepared_name not in original_headers:
             continue
@@ -225,8 +228,8 @@ def apply_missing_repeated_headers(
     prepared_request.headers = new_headers
 
 
-def make_default_headers(args: argparse.Namespace) -> RequestHeadersDict:
-    default_headers = RequestHeadersDict({
+def make_default_headers(args: argparse.Namespace) -> HTTPHeadersDict:
+    default_headers = HTTPHeadersDict({
         'User-Agent': DEFAULT_UA
     })
 
@@ -271,7 +274,7 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
 
 def make_request_kwargs(
     args: argparse.Namespace,
-    base_headers: RequestHeadersDict = None,
+    base_headers: HTTPHeadersDict = None,
     request_body_read_callback=lambda chunk: chunk
 ) -> dict:
     """

httpie/models.py

@@ -72,7 +72,9 @@ class HTTPResponse(HTTPMessage):
         )
         headers.extend(
             f'Set-Cookie: {cookie}'
-            for cookie in split_cookies(original.headers.get('Set-Cookie'))
+            for header, value in original.headers.items()
+            for cookie in split_cookies(value)
+            if header == 'Set-Cookie'
         )
         return '\r\n'.join(headers)
 

httpie/sessions.py

@@ -13,7 +13,7 @@ from urllib.parse import urlsplit
 from requests.auth import AuthBase
 from requests.cookies import RequestsCookieJar, create_cookie
 
-from .cli.dicts import RequestHeadersDict
+from .cli.dicts import HTTPHeadersDict
 from .config import BaseConfigDict, DEFAULT_CONFIG_DIR
 from .plugins.registry import plugin_manager
 
@@ -65,7 +65,7 @@ class Session(BaseConfigDict):
             'password': None
         }
 
-    def update_headers(self, request_headers: RequestHeadersDict):
+    def update_headers(self, request_headers: HTTPHeadersDict):
         """
         Update the session headers with the request ones while ignoring
         certain name prefixes.
@@ -98,8 +98,8 @@ class Session(BaseConfigDict):
         self['headers'] = dict(headers)
 
     @property
-    def headers(self) -> RequestHeadersDict:
-        return RequestHeadersDict(self['headers'])
+    def headers(self) -> HTTPHeadersDict:
+        return HTTPHeadersDict(self['headers'])
 
     @property
     def cookies(self) -> RequestsCookieJar:

httpie/ssl.py

@@ -1,6 +1,6 @@
 import ssl
 
-from requests.adapters import HTTPAdapter
+from httpie.adapters import HTTPAdapter
 # noinspection PyPackageRequirements
 from urllib3.util.ssl_ import (
     DEFAULT_CIPHERS, create_urllib3_context,