Implement support for multiple headers with the same name in sessions (#1335)

* Properly remove duplicate Cookie headers

* Implement support for multiple headers with the same name in sessions

* More testing

* Cleanup

* Remove duplicated test, cleanup

* Fix pycodestyle

* CHANGELOG

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

tests/fixtures/session_data/new/cookies_dict.json

@@ -27,5 +27,5 @@
             "value": "bar"
         }
     ],
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/new/cookies_dict_dev_version.json

@@ -27,5 +27,5 @@
             "value": "bar"
         }
     ],
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/new/cookies_dict_with_extras.json

@@ -26,8 +26,14 @@
             "value": "bar"
         }
     ],
-    "headers": {
-        "X-Data": "value",
-        "X-Foo": "bar"
-    }
+    "headers": [
+        {
+            "name": "X-Data",
+            "value": "value"
+        },
+        {
+            "name": "X-Foo",
+            "value": "bar"
+        }
+    ]
 }

tests/fixtures/session_data/new/empty_cookies_dict.json

@@ -10,5 +10,5 @@
         "username": null
     },
     "cookies": [],
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/new/empty_cookies_list.json

@@ -10,5 +10,5 @@
         "username": null
     },
     "cookies": [],
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/new/empty_headers_dict.json

@@ -0,0 +1,14 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "__version__"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": []
+}

tests/fixtures/session_data/new/empty_headers_list.json

@@ -0,0 +1,14 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "__version__"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": []
+}

tests/fixtures/session_data/new/headers_cookies_dict_mixed.json

@@ -0,0 +1,40 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "__version__"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [
+        {
+            "domain": __host__,
+            "expires": null,
+            "name": "baz",
+            "path": "/",
+            "secure": false,
+            "value": "quux"
+        },
+        {
+            "domain": __host__,
+            "expires": null,
+            "name": "foo",
+            "path": "/",
+            "secure": false,
+            "value": "bar"
+        }
+    ],
+    "headers": [
+        {
+            "name": "X-Data",
+            "value": "value"
+        },
+        {
+            "name": "X-Foo",
+            "value": "bar"
+        }
+    ]
+}

tests/fixtures/session_data/new/headers_dict.json

@@ -0,0 +1,23 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "__version__"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": [
+        {
+            "name": "foo",
+            "value": "bar"
+        },
+        {
+            "name": "baz",
+            "value": "quux"
+        }
+    ]
+}

tests/fixtures/session_data/new/headers_dict_extras.json

@@ -0,0 +1,39 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "__version__"
+    },
+    "auth": {
+        "raw_auth": "foo:bar",
+        "type": "basic"
+    },
+    "cookies": [
+        {
+            "domain": null,
+            "expires": null,
+            "name": "baz",
+            "path": "/",
+            "secure": false,
+            "value": "quux"
+        },
+        {
+            "domain": null,
+            "expires": null,
+            "name": "foo",
+            "path": "/",
+            "secure": false,
+            "value": "bar"
+        }
+    ],
+    "headers": [
+        {
+            "name": "X-Data",
+            "value": "value"
+        },
+        {
+            "name": "X-Foo",
+            "value": "bar"
+        }
+    ]
+}

tests/fixtures/session_data/new/headers_list.json

@@ -0,0 +1,23 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "3.2.0"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": [
+        {
+            "name": "X-Data",
+            "value": "value"
+        },
+        {
+            "name": "X-Foo",
+            "value": "bar"
+        }
+    ]
+}

tests/fixtures/session_data/old/cookies_dict.json

@@ -23,5 +23,5 @@
             "value": "bar"
         }
     },
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/old/cookies_dict_dev_version.json

@@ -23,5 +23,5 @@
             "value": "bar"
         }
     },
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/old/cookies_dict_with_extras.json

@@ -22,8 +22,14 @@
             "value": "bar"
         }
     },
-    "headers": {
-        "X-Data": "value",
-        "X-Foo": "bar"
-    }
+    "headers": [
+        {
+            "name": "X-Data",
+            "value": "value"
+        },
+        {
+            "name": "X-Foo",
+            "value": "bar"
+        }
+    ]
 }

tests/fixtures/session_data/old/empty_cookies_dict.json

@@ -10,5 +10,5 @@
         "username": null
     },
     "cookies": {},
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/old/empty_cookies_list.json

@@ -10,5 +10,5 @@
         "username": null
     },
     "cookies": [],
-    "headers": {}
+    "headers": []
 }

tests/fixtures/session_data/old/empty_headers_dict.json

@@ -0,0 +1,14 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "3.0.2"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": {}
+}

tests/fixtures/session_data/old/empty_headers_list.json

@@ -0,0 +1,14 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "3.0.2"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": []
+}

tests/fixtures/session_data/old/headers_cookies_dict_mixed.json

@@ -0,0 +1,30 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "3.0.2"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": {
+        "baz": {
+            "expires": null,
+            "path": "/",
+            "secure": false,
+            "value": "quux"
+        },
+        "foo": {
+            "expires": null,
+            "path": "/",
+            "secure": false,
+            "value": "bar"
+        }
+    },
+    "headers": {
+        "X-Data": "value",
+        "X-Foo": "bar"
+    }
+}

tests/fixtures/session_data/old/headers_dict.json

@@ -0,0 +1,17 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "3.1.0"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": {
+        "foo": "bar",
+        "baz": "quux"
+    }
+}

tests/fixtures/session_data/old/headers_dict_extras.json

@@ -0,0 +1,33 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "3.1.0"
+    },
+    "auth": {
+        "raw_auth": "foo:bar",
+        "type": "basic"
+    },
+    "cookies": [
+        {
+            "domain": null,
+            "name": "baz",
+            "expires": null,
+            "path": "/",
+            "secure": false,
+            "value": "quux"
+        },
+        {
+            "domain": null,
+            "name": "foo",
+            "expires": null,
+            "path": "/",
+            "secure": false,
+            "value": "bar"
+        }
+    ],
+    "headers": {
+        "X-Data": "value",
+        "X-Foo": "bar"
+    }
+}

tests/fixtures/session_data/old/headers_list.json

@@ -0,0 +1,23 @@
+{
+    "__meta__": {
+        "about": "HTTPie session file",
+        "help": "https://httpie.io/docs#sessions",
+        "httpie": "3.2.0"
+    },
+    "auth": {
+        "password": null,
+        "type": null,
+        "username": null
+    },
+    "cookies": [],
+    "headers": [
+        {
+            "name": "X-Data",
+            "value": "value"
+        },
+        {
+            "name": "X-Foo",
+            "value": "bar"
+        }
+    ]
+}

tests/test_sessions.py

@@ -664,7 +664,7 @@ def test_old_session_cookie_layout_loading(basic_session, httpbin, mock_env):
 @pytest.mark.parametrize('layout_type', [
     dict, list
 ])
-def test_session_cookie_layout_preservance(basic_session, mock_env, layout_type):
+def test_session_cookie_layout_preservation(basic_session, mock_env, layout_type):
     with open_session(basic_session, mock_env) as session:
         session['cookies'] = layout_type()
         session.cookies.set('foo', 'bar')
@@ -677,7 +677,7 @@ def test_session_cookie_layout_preservance(basic_session, mock_env, layout_type)
 @pytest.mark.parametrize('layout_type', [
     dict, list
 ])
-def test_session_cookie_layout_preservance_on_new_cookies(basic_session, httpbin, mock_env, layout_type):
+def test_session_cookie_layout_preservation_on_new_cookies(basic_session, httpbin, mock_env, layout_type):
     with open_session(basic_session, mock_env) as session:
         session['cookies'] = layout_type()
         session.cookies.set('foo', 'bar')
@@ -690,3 +690,113 @@ def test_session_cookie_layout_preservance_on_new_cookies(basic_session, httpbin
 
     with open_session(basic_session, mock_env, read_only=True) as session:
         assert isinstance(session['cookies'], layout_type)
+
+
+@pytest.mark.parametrize('headers, expect_warning', [
+    # Old-style header format
+    (
+        {},
+        False
+    ),
+    (
+        {'Foo': 'bar'},
+        True
+    ),
+    # New style header format
+    (
+        [],
+        False
+    ),
+    (
+        [{'name': 'Foo', 'value': 'Bar'}],
+        False
+    ),
+])
+def test_headers_old_layout_warning(basic_session, mock_env, headers, expect_warning):
+    with open_raw_session(basic_session) as raw_session:
+        raw_session['headers'] = headers
+
+    with open_session(basic_session, mock_env, read_only=True):
+        warning = b'Outdated layout detected'
+        stderr = read_stderr(mock_env)
+
+        if expect_warning:
+            assert warning in stderr
+        else:
+            assert warning not in stderr
+
+
+def test_outdated_layout_mixed(basic_session, mock_env):
+    with open_raw_session(basic_session) as raw_session:
+        raw_session['headers'] = {'Foo': 'Bar'}
+        raw_session['cookies'] = {
+            'cookie': {
+                'value': 'value'
+            }
+        }
+
+    with open_session(basic_session, mock_env, read_only=True):
+        stderr = read_stderr(mock_env)
+        # We should only see 1 warning.
+        assert stderr.count(b'Outdated layout') == 1
+
+
+def test_old_session_header_layout_loading(basic_session, httpbin, mock_env):
+    with open_session(basic_session, mock_env) as session:
+        # Use the old layout & set a header
+        session['headers'] = {}
+        session._headers.add('Foo', 'Bar')
+
+    response = http(
+        '--session', str(basic_session),
+        httpbin + '/get'
+    )
+    assert response.json['headers']['Foo'] == 'Bar'
+
+
+@pytest.mark.parametrize('layout_type', [
+    dict, list
+])
+def test_session_header_layout_preservation(basic_session, mock_env, layout_type):
+    with open_session(basic_session, mock_env) as session:
+        session['headers'] = layout_type()
+        session._headers.add('Foo', 'Bar')
+
+    with open_session(basic_session, mock_env, read_only=True) as session:
+        assert isinstance(session['headers'], layout_type)
+
+
+@pytest.mark.parametrize('layout_type', [
+    dict, list
+])
+def test_session_header_layout_preservation_on_new_headers(basic_session, httpbin, mock_env, layout_type):
+    with open_session(basic_session, mock_env) as session:
+        session['headers'] = layout_type()
+        session._headers.add('Foo', 'Bar')
+
+    http(
+        '--session', str(basic_session),
+        httpbin + '/get',
+        'Baz:Quux'
+    )
+
+    with open_session(basic_session, mock_env, read_only=True) as session:
+        assert isinstance(session['headers'], layout_type)
+
+
+def test_session_multiple_headers_with_same_name(basic_session, httpbin):
+    http(
+        '--session', str(basic_session),
+        httpbin + '/get',
+        'Foo:bar',
+        'Foo:baz',
+        'Foo:bar'
+    )
+
+    r = http(
+        '--offline',
+        '--session', str(basic_session),
+        httpbin + '/get',
+    )
+    assert r.count('Foo: bar') == 2
+    assert 'Foo: baz' in r