diff --git a/CHANGELOG.md b/CHANGELOG.md index d5ae7f12..6d2a73c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [2.7.0.dev0](https://github.com/httpie/httpie/compare/2.6.0...master) (unreleased) +- Added support for sending multiple HTTP headers with the same name. ([#130](https://github.com/httpie/httpie/issues/130)) + ## [2.6.0](https://github.com/httpie/httpie/compare/2.5.0...2.6.0) (2021-10-14) [What’s new in HTTPie 2.6.0 →](https://httpie.io/blog/httpie-2.6.0) diff --git a/docs/README.md b/docs/README.md index 494e7131..ff6fbad1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -869,6 +869,39 @@ To send a header with an empty value, use `Header;`, with a semicolon: $ http pie.dev/headers 'Header;' ``` +Please note that some internal headers, such as `Content-Length`, can't be unset if +they are automatically added by the client itself. + +### Multiple header values with the same name + +If the request is sent with multiple headers that are sharing the same name, then +the HTTPie will send them individually. + +```bash +http --offline example.org Cookie:one Cookie:two +``` + +```http +GET / HTTP/1.1 +Cookie: one +Cookie: two +``` + +It is also possible to pass a single header value pair, where the value is a comma +separated list of header values. Then the client will send it as a single header. + +```bash +http --offline example.org Numbers:one,two +``` + +```http +GET / HTTP/1.1 +Numbers: one,two +``` + +Also be aware that if the current session contains any headers they will get overwriten +by individual commands when sending a request instead of being joined together. + ### Limiting response headers The `--max-headers=n` options allows you to control the number of headers HTTPie reads before giving up (the default `0`, i.e., there’s no limit). diff --git a/httpie/cli/dicts.py b/httpie/cli/dicts.py index 84178236..43a624ee 100644 --- a/httpie/cli/dicts.py +++ b/httpie/cli/dicts.py @@ -1,15 +1,41 @@ from collections import OrderedDict -from requests.structures import CaseInsensitiveDict +from multidict import MultiDict, CIMultiDict -class RequestHeadersDict(CaseInsensitiveDict): +class BaseMultiDict(MultiDict): """ - Headers are case-insensitive and multiple values are currently not supported. - + Base class for all MultiDicts. """ +class RequestHeadersDict(CIMultiDict, BaseMultiDict): + """ + Headers are case-insensitive and multiple values are supported + through the `add()` API. + """ + + def add(self, key, value): + """ + Add or update a new header. + + If the given `value` is `None`, then all the previous + values will be overwritten and the value will be set + to `None`. + """ + if value is None: + self[key] = value + return None + + # If the previous value for the given header is `None` + # then discard it since we are explicitly giving a new + # value for it. + if key in self and self.getone(key) is None: + self.popone(key) + + super().add(key, value) + + class RequestJSONDataDict(OrderedDict): pass diff --git a/httpie/cli/requestitems.py b/httpie/cli/requestitems.py index 1dd6594c..73d5e4d0 100644 --- a/httpie/cli/requestitems.py +++ b/httpie/cli/requestitems.py @@ -10,8 +10,8 @@ from .constants import ( SEPARATOR_QUERY_PARAM, ) from .dicts import ( - MultipartRequestDataDict, RequestDataDict, RequestFilesDict, - RequestHeadersDict, RequestJSONDataDict, + BaseMultiDict, MultipartRequestDataDict, RequestDataDict, + RequestFilesDict, RequestHeadersDict, RequestJSONDataDict, RequestQueryParamsDict, ) from .exceptions import ParseError @@ -73,11 +73,15 @@ class RequestItems: for arg in request_item_args: processor_func, target_dict = rules[arg.sep] value = processor_func(arg) - target_dict[arg.key] = value if arg.sep in SEPARATORS_GROUP_MULTIPART: instance.multipart_data[arg.key] = value + if isinstance(target_dict, BaseMultiDict): + target_dict.add(arg.key, value) + else: + target_dict[arg.key] = value + return instance diff --git a/httpie/client.py b/httpie/client.py index 5feaf483..45b43276 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -79,6 +79,7 @@ def collect_messages( request = requests.Request(**request_kwargs) prepared_request = requests_session.prepare_request(request) + apply_missing_repeated_headers(prepared_request, request.headers) if args.path_as_is: prepared_request.url = ensure_path_as_is( orig_url=args.url, @@ -190,10 +191,40 @@ def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict: if isinstance(value, str): # See value = value.encode() - final_headers[name] = value + final_headers.add(name, value) return final_headers +def apply_missing_repeated_headers( + prepared_request: requests.PreparedRequest, + original_headers: RequestHeadersDict +) -> None: + """Update the given `prepared_request`'s headers with the original + ones. This allows the requests to be prepared as usual, and then later + merged with headers that are specified multiple times.""" + + new_headers = RequestHeadersDict(prepared_request.headers) + for prepared_name, prepared_value in prepared_request.headers.items(): + if prepared_name not in original_headers: + continue + + original_keys, original_values = zip(*filter( + lambda item: item[0].casefold() == prepared_name.casefold(), + original_headers.items() + )) + + if prepared_value not in original_values: + # If the current value is not among the initial values + # set for this field, then it means that this field got + # overridden on the way, and we should preserve it. + continue + + new_headers.popone(prepared_name) + new_headers.update(zip(original_keys, original_values)) + + prepared_request.headers = new_headers + + def make_default_headers(args: argparse.Namespace) -> RequestHeadersDict: default_headers = RequestHeadersDict({ 'User-Agent': DEFAULT_UA diff --git a/httpie/models.py b/httpie/models.py index c554dca9..9c72518a 100644 --- a/httpie/models.py +++ b/httpie/models.py @@ -96,7 +96,7 @@ class HTTPRequest(HTTPMessage): query=f'?{url.query}' if url.query else '' ) - headers = dict(self._orig.headers) + headers = self._orig.headers.copy() if 'Host' not in self._orig.headers: headers['Host'] = url.netloc.split('@')[-1] diff --git a/setup.py b/setup.py index ee6a4f41..1238bfd3 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires = [ 'requests[socks]>=2.22.0', 'Pygments>=2.5.2', 'requests-toolbelt>=0.9.1', + 'multidict>=4.7.0', 'setuptools', ] install_requires_win_only = [ diff --git a/tests/test_cli.py b/tests/test_cli.py index 4562deb6..e0f52147 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -39,8 +39,8 @@ class TestItemParsing: # files self.key_value_arg(fr'bar\@baz@{FILE_PATH_ARG}'), ]) - # `requests.structures.CaseInsensitiveDict` => `dict` - headers = dict(items.headers._store.values()) + # `RequestHeadersDict` => `dict` + headers = dict(items.headers) assert headers == { 'foo:bar': 'baz', @@ -88,8 +88,8 @@ class TestItemParsing: ]) # Parsed headers - # `requests.structures.CaseInsensitiveDict` => `dict` - headers = dict(items.headers._store.values()) + # `RequestHeadersDict` => `dict` + headers = dict(items.headers) assert headers == { 'Header': 'value', 'Unset-Header': None, diff --git a/tests/test_httpie.py b/tests/test_httpie.py index a6cda1c3..748e8769 100644 --- a/tests/test_httpie.py +++ b/tests/test_httpie.py @@ -209,6 +209,88 @@ def test_headers_empty_value_with_value_gives_error(httpbin): http('GET', httpbin + '/headers', 'Accept;SYNTAX_ERROR') +def test_headers_omit(httpbin_both): + r = http('GET', httpbin_both + '/headers', 'Accept:') + assert 'Accept' not in r.json['headers'] + + +def test_headers_multiple_omit(httpbin_both): + r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Bar:baz', + 'Foo:', 'Baz:quux') + assert 'Foo' not in r.json['headers'] + assert r.json['headers']['Bar'] == 'baz' + assert r.json['headers']['Baz'] == 'quux' + + +def test_headers_same_after_omit(httpbin_both): + r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:', + 'Foo:quux') + assert r.json['headers']['Foo'] == 'quux' + + +def test_headers_fully_omit(httpbin_both): + r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:baz', + 'Foo:') + assert 'Foo' not in r.json['headers'] + + +def test_headers_multiple_values(httpbin_both): + r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:baz') + assert r.json['headers']['Foo'] == 'bar,baz' + + +def test_headers_multiple_values_repeated(httpbin_both): + r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Foo:baz', + 'Foo:bar') + assert r.json['headers']['Foo'] == 'bar,baz,bar' + + +@pytest.mark.parametrize("headers, expected", [ + ( + ["Foo;", "Foo:bar"], + ",bar" + ), + ( + ["Foo:bar", "Foo;"], + "bar," + ), + ( + ["Foo:bar", "Foo;", "Foo:baz"], + "bar,,baz" + ), +]) +def test_headers_multiple_values_with_empty(httpbin_both, headers, expected): + r = http('GET', httpbin_both + '/headers', *headers) + assert r.json['headers']['Foo'] == expected + + +def test_headers_multiple_values_mixed(httpbin_both): + r = http('GET', httpbin_both + '/headers', 'Foo:bar', 'Vary:XXX', + 'Foo:baz', 'Vary:YYY', 'Foo:quux') + assert r.json['headers']['Vary'] == 'XXX,YYY' + assert r.json['headers']['Foo'] == 'bar,baz,quux' + + +def test_headers_preserve_prepared_headers(httpbin_both): + r = http('POST', httpbin_both + '/post', 'Content-Length:0', + '--raw', 'foo') + assert r.json['headers']['Content-Length'] == '3' + + +@pytest.mark.parametrize('pretty', ['format', 'none']) +def test_headers_multiple_headers_representation(httpbin_both, pretty): + r = http('--offline', '--pretty', pretty, 'example.org', + 'A:A', 'A:B', 'A:C', 'B:A', 'B:B', 'C:C', 'c:c') + + assert 'A: A' in r + assert 'A: B' in r + assert 'A: C' in r + assert 'B: A' in r + assert 'B: B' in r + assert 'C: C' in r + assert 'c: c' in r + + def test_json_input_preserve_order(httpbin_both): r = http('PATCH', httpbin_both + '/patch', 'order:={"map":{"1":"first","2":"second"}}') diff --git a/tests/test_redirects.py b/tests/test_redirects.py index 9aa6e1ce..81dcb2be 100644 --- a/tests/test_redirects.py +++ b/tests/test_redirects.py @@ -73,12 +73,17 @@ def test_follow_redirect_with_repost(httpbin, status_code): r = http( '--follow', httpbin.url + '/redirect-to', + 'A:A', + 'A:B', + 'B:B', f'url=={httpbin.url}/post', f'status_code=={status_code}', '@' + FILE_PATH_ARG, ) assert HTTP_OK in r assert FILE_CONTENT in r + assert r.json['headers']['A'] == 'A,B' + assert r.json['headers']['B'] == 'B' @pytest.mark.skipif(is_windows, reason='occasionally fails w/ ConnectionError for no apparent reason') @@ -88,11 +93,17 @@ def test_verbose_follow_redirect_with_repost(httpbin, status_code): '--follow', '--verbose', httpbin.url + '/redirect-to', + 'A:A', + 'A:B', + 'B:B', f'url=={httpbin.url}/post', f'status_code=={status_code}', '@' + FILE_PATH_ARG, ) assert f'HTTP/1.1 {status_code}' in r + assert 'A: A' in r + assert 'A: B' in r + assert 'B: B' in r assert r.count('POST /redirect-to') == 1 assert r.count('POST /post') == 1 assert r.count(FILE_CONTENT) == 3 # two requests + final response contain it diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 7f6af2ea..5615c08f 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -143,6 +143,24 @@ class TestSessionFlow(SessionTestBase): # Should be the same as before r3. assert r2.json == r4.json + def test_session_overwrite_header(self, httpbin): + self.start_session(httpbin) + + r2 = http('--session=test', 'GET', httpbin.url + '/get', + 'Hello:World2', env=self.env()) + assert HTTP_OK in r2 + assert r2.json['headers']['Hello'] == 'World2' + + r3 = http('--session=test', 'GET', httpbin.url + '/get', + 'Hello:World2', 'Hello:World3', env=self.env()) + assert HTTP_OK in r3 + assert r3.json['headers']['Hello'] == 'World2,World3' + + r3 = http('--session=test', 'GET', httpbin.url + '/get', + 'Hello:', 'Hello:World3', env=self.env()) + assert HTTP_OK in r3 + assert 'Hello' not in r3.json['headers']['Hello'] + class TestSession(SessionTestBase): """Stand-alone session tests."""