Support multiple headers sharing the same name (#1190)

* Support multiple headers sharing the same name

* Apply suggestions

* Don't normalize HTTP header names

* apply visual suggestions

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

* bump down multidict to 4.7.0

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

CHANGELOG.md

@@ -5,6 +5,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 
 ## [2.7.0.dev0](https://github.com/httpie/httpie/compare/2.6.0...master) (unreleased)
 
+- Added support for sending multiple HTTP headers with the same name. ([#130](https://github.com/httpie/httpie/issues/130))
+
 ## [2.6.0](https://github.com/httpie/httpie/compare/2.5.0...2.6.0) (2021-10-14)
 
 [What’s new in HTTPie 2.6.0 →](https://httpie.io/blog/httpie-2.6.0)

docs/README.md

@@ -869,6 +869,39 @@ To send a header with an empty value, use `Header;`, with a semicolon:
 
 ```bash
 http --offline example.org Numbers:one,two
+```
+
+```http
+GET / HTTP/1.1
+Numbers: one,two
+```
+
+Also be aware that if the current session contains any headers they will get overwriten
+by individual commands when sending a request instead of being joined together.
+
+### Limiting response headers
+
+The `--max-headers=n` options allows you to control the number of headers HTTPie reads before giving up (the default `0`, i.e., there’s no limit).
+
+```bash
+$ http --max-headers=100 pie.dev/get
+```
+
+## Offline mode
+
+Use `--offline` to construct HTTP requests without sending them anywhere.
+With `--offline`, HTTPie builds a request based on the specified options and arguments, prints it to `stdout`, and then exits. It works completely offline; no network connection is ever made. This has a number of use cases, including:
+
+Generating API documentation examples that you can copy & paste without sending a request:
+
+```bash
+$ http --offline POST server.chess/api/games API-Key:ZZZ w=magnus b=hikaru t=180 i=2
+```
+
+```bash
+$ http --offline MOVE server.chess/api/games/123 API-Key:ZZZ p=b a=R1a3 t=77
+```
+
 Generating raw requests that can be sent with any other client:
 
 ```bash

httpie/cli/dicts.py

@@ -1,15 +1,41 @@
 from collections import OrderedDict
 
-from requests.structures import CaseInsensitiveDict
+from multidict import MultiDict, CIMultiDict
 
 
-class RequestHeadersDict(CaseInsensitiveDict):
+class BaseMultiDict(MultiDict):
     """
-    Headers are case-insensitive and multiple values are currently not supported.
-
+    Base class for all MultiDicts.
     """
 
 
+class RequestHeadersDict(CIMultiDict, BaseMultiDict):
+    """
+    Headers are case-insensitive and multiple values are supported
+    through the `add()` API.
+    """
+
+    def add(self, key, value):
+        """
+        Add or update a new header.
+
+        If the given `value` is `None`, then all the previous
+        values will be overwritten and the value will be set
+        to `None`.
+        """
+        if value is None:
+            self[key] = value
+            return None
+
+        # If the previous value for the given header is `None`
+        # then discard it since we are explicitly giving a new
+        # value for it.
+        if key in self and self.getone(key) is None:
+            self.popone(key)
+
+        super().add(key, value)
+
+
 class RequestJSONDataDict(OrderedDict):
     pass
 

httpie/cli/requestitems.py

@@ -10,8 +10,8 @@ from .constants import (
     SEPARATOR_QUERY_PARAM,
 )
 from .dicts import (
-    MultipartRequestDataDict, RequestDataDict, RequestFilesDict,
-    RequestHeadersDict, RequestJSONDataDict,
+    BaseMultiDict, MultipartRequestDataDict, RequestDataDict,
+    RequestFilesDict, RequestHeadersDict, RequestJSONDataDict,
     RequestQueryParamsDict,
 )
 from .exceptions import ParseError
@@ -73,11 +73,15 @@ class RequestItems:
         for arg in request_item_args:
             processor_func, target_dict = rules[arg.sep]
             value = processor_func(arg)
-            target_dict[arg.key] = value
 
             if arg.sep in SEPARATORS_GROUP_MULTIPART:
                 instance.multipart_data[arg.key] = value
 
+            if isinstance(target_dict, BaseMultiDict):
+                target_dict.add(arg.key, value)
+            else:
+                target_dict[arg.key] = value
+
         return instance
 
 

httpie/client.py

@@ -79,6 +79,7 @@ def collect_messages(
 
     request = requests.Request(**request_kwargs)
     prepared_request = requests_session.prepare_request(request)
+    apply_missing_repeated_headers(prepared_request, request.headers)
     if args.path_as_is:
         prepared_request.url = ensure_path_as_is(
             orig_url=args.url,
@@ -190,10 +191,40 @@ def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict:
             if isinstance(value, str):
                 # See <https://github.com/httpie/httpie/issues/212>
                 value = value.encode()
-        final_headers[name] = value
+        final_headers.add(name, value)
     return final_headers
 
 
+def apply_missing_repeated_headers(
+    prepared_request: requests.PreparedRequest,
+    original_headers: RequestHeadersDict
+) -> None:
+    """Update the given `prepared_request`'s headers with the original
+    ones. This allows the requests to be prepared as usual, and then later
+    merged with headers that are specified multiple times."""
+
+    new_headers = RequestHeadersDict(prepared_request.headers)
+    for prepared_name, prepared_value in prepared_request.headers.items():
+        if prepared_name not in original_headers:
+            continue
+
+        original_keys, original_values = zip(*filter(
+            lambda item: item[0].casefold() == prepared_name.casefold(),
+            original_headers.items()
+        ))
+
+        if prepared_value not in original_values:
+            # If the current value is not among the initial values
+            # set for this field, then it means that this field got
+            # overridden on the way, and we should preserve it.
+            continue
+
+        new_headers.popone(prepared_name)
+        new_headers.update(zip(original_keys, original_values))
+
+    prepared_request.headers = new_headers
+
+
 def make_default_headers(args: argparse.Namespace) -> RequestHeadersDict:
     default_headers = RequestHeadersDict({
         'User-Agent': DEFAULT_UA

httpie/models.py

@@ -96,7 +96,7 @@ class HTTPRequest(HTTPMessage):
             query=f'?{url.query}' if url.query else ''
         )
 
-        headers = dict(self._orig.headers)
+        headers = self._orig.headers.copy()
         if 'Host' not in self._orig.headers:
             headers['Host'] = url.netloc.split('@')[-1]
 

setup.py

@@ -33,6 +33,7 @@ install_requires = [
     'requests[socks]>=2.22.0',
     'Pygments>=2.5.2',
     'requests-toolbelt>=0.9.1',
+    'multidict>=4.7.0',
     'setuptools',
 ]
 install_requires_win_only = [

tests/test_cli.py

@@ -39,8 +39,8 @@ class TestItemParsing:
             # files
             self.key_value_arg(fr'bar\@baz@{FILE_PATH_ARG}'),
         ])
-        # `requests.structures.CaseInsensitiveDict` => `dict`
-        headers = dict(items.headers._store.values())
+        # `RequestHeadersDict` => `dict`
+        headers = dict(items.headers)
 
         assert headers == {
             'foo:bar': 'baz',
@@ -88,8 +88,8 @@ class TestItemParsing:
         ])
 
         # Parsed headers
-        # `requests.structures.CaseInsensitiveDict` => `dict`
-        headers = dict(items.headers._store.values())
+        # `RequestHeadersDict` => `dict`
+        headers = dict(items.headers)
         assert headers == {
             'Header': 'value',
             'Unset-Header': None,

tests/test_httpie.py

@@ -209,6 +209,88 @@ def test_headers_empty_value_with_value_gives_error(httpbin):
         http('GET', httpbin + '/headers', 'Accept;SYNTAX_ERROR')
 
 
+def test_headers_omit(httpbin_both):
+    r = http('GET', httpbin_both + '/headers', 'Accept:')
+    assert 'Accept' not in r.json['headers']
+
+
+def test_headers_multiple_omit(httpbin_both):
+    r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Bar:baz',
+             'Foo:', 'Baz:quux')
+    assert 'Foo' not in r.json['headers']
+    assert r.json['headers']['Bar'] == 'baz'
+    assert r.json['headers']['Baz'] == 'quux'
+
+
+def test_headers_same_after_omit(httpbin_both):
+    r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:',
+             'Foo:quux')
+    assert r.json['headers']['Foo'] == 'quux'
+
+
+def test_headers_fully_omit(httpbin_both):
+    r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:baz',
+             'Foo:')
+    assert 'Foo' not in r.json['headers']
+
+
+def test_headers_multiple_values(httpbin_both):
+    r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:baz')
+    assert r.json['headers']['Foo'] == 'bar,baz'
+
+
+def test_headers_multiple_values_repeated(httpbin_both):
+    r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:baz',
+             'Foo:bar')
+    assert r.json['headers']['Foo'] == 'bar,baz,bar'
+
+
+@pytest.mark.parametrize("headers, expected", [
+    (
+        ["Foo;", "Foo:bar"],
+        ",bar"
+    ),
+    (
+        ["Foo:bar", "Foo;"],
+        "bar,"
+    ),
+    (
+        ["Foo:bar", "Foo;", "Foo:baz"],
+        "bar,,baz"
+    ),
+])
+def test_headers_multiple_values_with_empty(httpbin_both, headers, expected):
+    r = http('GET', httpbin_both + '/headers', *headers)
+    assert r.json['headers']['Foo'] == expected
+
+
+def test_headers_multiple_values_mixed(httpbin_both):
+    r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Vary:XXX',
+             'Foo:baz', 'Vary:YYY', 'Foo:quux')
+    assert r.json['headers']['Vary'] == 'XXX,YYY'
+    assert r.json['headers']['Foo'] == 'bar,baz,quux'
+
+
+def test_headers_preserve_prepared_headers(httpbin_both):
+    r = http('POST', httpbin_both + '/post', 'Content-Length:0',
+             '--raw', 'foo')
+    assert r.json['headers']['Content-Length'] == '3'
+
+
+@pytest.mark.parametrize('pretty', ['format', 'none'])
+def test_headers_multiple_headers_representation(httpbin_both, pretty):
+    r = http('--offline', '--pretty', pretty, 'example.org',
+             'A:A', 'A:B', 'A:C', 'B:A', 'B:B', 'C:C', 'c:c')
+
+    assert 'A: A' in r
+    assert 'A: B' in r
+    assert 'A: C' in r
+    assert 'B: A' in r
+    assert 'B: B' in r
+    assert 'C: C' in r
+    assert 'c: c' in r
+
+
 def test_json_input_preserve_order(httpbin_both):
     r = http('PATCH', httpbin_both + '/patch',
              'order:={"map":{"1":"first","2":"second"}}')

tests/test_redirects.py

@@ -73,12 +73,17 @@ def test_follow_redirect_with_repost(httpbin, status_code):
     r = http(
         '--follow',
         httpbin.url + '/redirect-to',
+        'A:A',
+        'A:B',
+        'B:B',
         f'url=={httpbin.url}/post',
         f'status_code=={status_code}',
         '@' + FILE_PATH_ARG,
     )
     assert HTTP_OK in r
     assert FILE_CONTENT in r
+    assert r.json['headers']['A'] == 'A,B'
+    assert r.json['headers']['B'] == 'B'
 
 
 @pytest.mark.skipif(is_windows, reason='occasionally fails w/ ConnectionError for no apparent reason')
@@ -88,11 +93,17 @@ def test_verbose_follow_redirect_with_repost(httpbin, status_code):
         '--follow',
         '--verbose',
         httpbin.url + '/redirect-to',
+        'A:A',
+        'A:B',
+        'B:B',
         f'url=={httpbin.url}/post',
         f'status_code=={status_code}',
         '@' + FILE_PATH_ARG,
     )
     assert f'HTTP/1.1 {status_code}' in r
+    assert 'A: A' in r
+    assert 'A: B' in r
+    assert 'B: B' in r
     assert r.count('POST /redirect-to') == 1
     assert r.count('POST /post') == 1
     assert r.count(FILE_CONTENT) == 3  # two requests + final response contain it

tests/test_sessions.py

@@ -143,6 +143,24 @@ class TestSessionFlow(SessionTestBase):
         # Should be the same as before r3.
         assert r2.json == r4.json
 
+    def test_session_overwrite_header(self, httpbin):
+        self.start_session(httpbin)
+
+        r2 = http('--session=test', 'GET', httpbin.url + '/get',
+                  'Hello:World2', env=self.env())
+        assert HTTP_OK in r2
+        assert r2.json['headers']['Hello'] == 'World2'
+
+        r3 = http('--session=test', 'GET', httpbin.url + '/get',
+                  'Hello:World2', 'Hello:World3', env=self.env())
+        assert HTTP_OK in r3
+        assert r3.json['headers']['Hello'] == 'World2,World3'
+
+        r3 = http('--session=test', 'GET', httpbin.url + '/get',
+                  'Hello:', 'Hello:World3', env=self.env())
+        assert HTTP_OK in r3
+        assert 'Hello' not in r3.json['headers']['Hello']
+
 
 class TestSession(SessionTestBase):
     """Stand-alone session tests."""