Fix duplicate keys preservation of JSON data (#1163)

* Fix duplicate keys preservation of JSON data

* Update issue number

* Fix type annotations

* Changes after review

* Rewording

tests/fixtures/__init__.py

@@ -16,6 +16,7 @@ def patharg(path):
 FIXTURES_ROOT = Path(__file__).parent
 FILE_PATH = FIXTURES_ROOT / 'test.txt'
 JSON_FILE_PATH = FIXTURES_ROOT / 'test.json'
+JSON_WITH_DUPE_KEYS_FILE_PATH = FIXTURES_ROOT / 'test_with_dupe_keys.json'
 BIN_FILE_PATH = FIXTURES_ROOT / 'test.bin'
 XML_FILES_PATH = FIXTURES_ROOT / 'xmldata'
 XML_FILES_VALID = list((XML_FILES_PATH / 'valid').glob('*_raw.xml'))

tests/fixtures/test_with_dupe_keys.json

@@ -0,0 +1 @@
+{"key":15,"key":15,"key":3,"key":7}

tests/test_cli.py

@@ -1,20 +1,21 @@
 """CLI argument parsing related tests."""
 import argparse
-import json
 
 import pytest
 from requests.exceptions import InvalidSchema
 
 import httpie.cli.argparser
-from .fixtures import (
-    FILE_CONTENT, FILE_PATH, FILE_PATH_ARG, JSON_FILE_CONTENT,
-    JSON_FILE_PATH_ARG,
-)
-from httpie.status import ExitStatus
 from httpie.cli import constants
 from httpie.cli.definition import parser
 from httpie.cli.argtypes import KeyValueArg, KeyValueArgType
 from httpie.cli.requestitems import RequestItems
+from httpie.status import ExitStatus
+from httpie.utils import load_json_preserve_order_and_dupe_keys
+
+from .fixtures import (
+    FILE_CONTENT, FILE_PATH, FILE_PATH_ARG, JSON_FILE_CONTENT,
+    JSON_FILE_PATH_ARG,
+)
 from .utils import HTTP_OK, MockEnvironment, StdinBytesIO, http
 
 
@@ -97,17 +98,15 @@ class TestItemParsing:
 
         # Parsed data
         raw_json_embed = items.data.pop('raw-json-embed')
-        assert raw_json_embed == json.loads(JSON_FILE_CONTENT)
+        assert raw_json_embed == load_json_preserve_order_and_dupe_keys(JSON_FILE_CONTENT)
         items.data['string-embed'] = items.data['string-embed'].strip()
         assert dict(items.data) == {
-            "ed": "",
-            "string": "value",
-            "bool": True,
-            "list": ["a", 1, {}, False],
-            "obj": {
-                "a": "b"
-            },
-            "string-embed": FILE_CONTENT,
+            'ed': '',
+            'string': 'value',
+            'bool': True,
+            'list': ['a', 1, {}, False],
+            'obj': load_json_preserve_order_and_dupe_keys('{"a": "b"}'),
+            'string-embed': FILE_CONTENT,
         }
 
         # Parsed query string parameters

tests/test_json.py

@@ -6,13 +6,29 @@ import responses
 from httpie.cli.constants import PRETTY_MAP
 from httpie.compat import is_windows
 from httpie.output.formatters.colors import ColorFormatter
+from httpie.utils import JsonDictPreservingDuplicateKeys
 
+from .fixtures import JSON_WITH_DUPE_KEYS_FILE_PATH
 from .utils import MockEnvironment, http, URL_EXAMPLE
 
 TEST_JSON_XXSI_PREFIXES = (r")]}',\n", ")]}',", 'while(1);', 'for(;;)', ')', ']', '}')
 TEST_JSON_VALUES = ({}, {'a': 0, 'b': 0}, [], ['a', 'b'], 'foo', True, False, None)  # FIX: missing int & float
 TEST_PREFIX_TOKEN_COLOR = '\x1b[38;5;15m' if is_windows else '\x1b[04m\x1b[91m'
 
+JSON_WITH_DUPES_RAW = '{"key": 15, "key": 15, "key": 3, "key": 7}'
+JSON_WITH_DUPES_FORMATTED_SORTED = '''{
+    "key": 3,
+    "key": 7,
+    "key": 15,
+    "key": 15
+}'''
+JSON_WITH_DUPES_FORMATTED_UNSORTED = '''{
+    "key": 15,
+    "key": 15,
+    "key": 3,
+    "key": 7
+}'''
+
 
 @pytest.mark.parametrize('data_prefix', TEST_JSON_XXSI_PREFIXES)
 @pytest.mark.parametrize('json_data', TEST_JSON_VALUES)
@@ -38,3 +54,35 @@ def test_json_formatter_with_body_preceded_by_non_json_data(data_prefix, json_da
         # meaning it was correctly handled as a whole.
         assert TEST_PREFIX_TOKEN_COLOR + data_prefix in expected_body, expected_body
     assert expected_body in r
+
+
+@responses.activate
+def test_duplicate_keys_support_from_response():
+    """JSON with duplicate keys should be handled correctly."""
+    responses.add(responses.GET, URL_EXAMPLE, body=JSON_WITH_DUPES_RAW,
+                  content_type='application/json')
+    args = ('--pretty', 'format', URL_EXAMPLE)
+
+    # Check implicit --sorted
+    if JsonDictPreservingDuplicateKeys.SUPPORTS_SORTING:
+        r = http(*args)
+        assert JSON_WITH_DUPES_FORMATTED_SORTED in r
+
+    # Check --unsorted
+    r = http(*args, '--unsorted')
+    assert JSON_WITH_DUPES_FORMATTED_UNSORTED in r
+
+
+def test_duplicate_keys_support_from_input_file():
+    """JSON file with duplicate keys should be handled correctly."""
+    args = ('--verbose', '--offline', URL_EXAMPLE,
+            f'@{JSON_WITH_DUPE_KEYS_FILE_PATH}')
+
+    # Check implicit --sorted
+    if JsonDictPreservingDuplicateKeys.SUPPORTS_SORTING:
+        r = http(*args)
+        assert JSON_WITH_DUPES_FORMATTED_SORTED in r
+
+    # Check --unsorted
+    r = http(*args, '--unsorted')
+    assert JSON_WITH_DUPES_FORMATTED_UNSORTED in r