Add nested JSON syntax to the HTTPie DSL (#1224)

* Add support for nested JSON syntax (#1169)

Co-authored-by: Batuhan Taskaya <isidentical@gmail.com>
Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

* minor improvements

* unpack top level lists

* Write more docs

* doc style changes

* fix double quotes

Co-authored-by: Mickaël Schoentgen <contact@tiger-222.fr>
Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

httpie/cli/constants.py

@@ -46,10 +46,10 @@ SEPARATOR_GROUP_DATA_EMBED_ITEMS = frozenset({
     SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
 })
 
-# Separators for raw JSON items
-SEPARATOR_GROUP_RAW_JSON_ITEMS = frozenset([
+# Separators for nested JSON items
+SEPARATOR_GROUP_NESTED_JSON_ITEMS = frozenset([
+    SEPARATOR_DATA_STRING,
     SEPARATOR_DATA_RAW_JSON,
-    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
 ])
 
 # Separators allowed in ITEM arguments

httpie/cli/definition.py

@@ -120,7 +120,7 @@ positional.add_argument(
 
     '=@' A data field like '=', but takes a file path and embeds its content:
 
-         essay=@Documents/essay.txt
+        essay=@Documents/essay.txt
 
     ':=@' A raw JSON field like ':=', but takes a file path and embeds its content:
 

httpie/cli/json_form.py

@@ -0,0 +1,150 @@
+"""
+Routines for JSON form syntax, used to support nested JSON request items.
+
+Highly inspired from the great jarg project <https://github.com/jdp/jarg/blob/master/jarg/jsonform.py>.
+"""
+import re
+import operator
+from typing import Optional
+
+
+def step(value: str, is_escaped: bool) -> str:
+    if is_escaped:
+        value = value.replace(r'\[', '[').replace(r'\]', ']')
+    return value
+
+
+def find_opening_bracket(
+    value: str,
+    search=re.compile(r'(?<!\\)\[').search
+) -> Optional[int]:
+    match = search(value)
+    if not match:
+        return None
+    return match.start()
+
+
+def find_closing_bracket(
+    value: str,
+    search=re.compile(r'(?<!\\)\]').search
+) -> Optional[int]:
+    match = search(value)
+    if not match:
+        return None
+    return match.start()
+
+
+def parse_path(path):
+    """
+    Parse a string as a JSON path.
+
+    An implementation of 'steps to parse a JSON encoding path'.
+    <https://www.w3.org/TR/html-json-forms/#dfn-steps-to-parse-a-json-encoding-path>
+
+    """
+    original = path
+    is_escaped = r'\[' in original
+
+    opening_bracket = find_opening_bracket(original)
+    last_step = [(step(path, is_escaped), {'last': True, 'type': 'object'})]
+    if opening_bracket is None:
+        return last_step
+
+    steps = [(step(original[:opening_bracket], is_escaped), {'type': 'object'})]
+    path = original[opening_bracket:]
+    while path:
+        if path.startswith('[]'):
+            steps[-1][1]['append'] = True
+            path = path[2:]
+            if path:
+                return last_step
+        elif path[0] == '[':
+            path = path[1:]
+            closing_bracket = find_closing_bracket(path)
+            if closing_bracket is None:
+                return last_step
+            key = path[:closing_bracket]
+            path = path[closing_bracket + 1:]
+            try:
+                steps.append((int(key), {'type': 'array'}))
+            except ValueError:
+                steps.append((key, {'type': 'object'}))
+        elif path[:2] == r'\[':
+            key = step(path[1:path.index(r'\]') + 2], is_escaped)
+            path = path[path.index(r'\]') + 2:]
+            steps.append((key, {'type': 'object'}))
+        else:
+            return last_step
+
+    for i in range(len(steps) - 1):
+        steps[i][1]['type'] = steps[i + 1][1]['type']
+    steps[-1][1]['last'] = True
+    return steps
+
+
+def set_value(context, step, current_value, entry_value):
+    """Apply a JSON value to a context object.
+
+    An implementation of 'steps to set a JSON encoding value'.
+    <https://www.w3.org/TR/html-json-forms/#dfn-steps-to-set-a-json-encoding-value>
+
+    """
+    key, flags = step
+    if flags.get('last', False):
+        if current_value is None:
+            if flags.get('append', False):
+                context[key] = [entry_value]
+            else:
+                if isinstance(context, list) and len(context) <= key:
+                    context.extend([None] * (key - len(context) + 1))
+                context[key] = entry_value
+        elif isinstance(current_value, list):
+            context[key].append(entry_value)
+        else:
+            context[key] = [current_value, entry_value]
+        return context
+
+    if current_value is None:
+        if flags.get('type') == 'array':
+            context[key] = []
+        else:
+            if isinstance(context, list) and len(context) <= key:
+                context.extend([None] * (key - len(context) + 1))
+            context[key] = {}
+        return context[key]
+    elif isinstance(current_value, dict):
+        return context[key]
+    elif isinstance(current_value, list):
+        if flags.get('type') == 'array':
+            return current_value
+
+        obj = {}
+        for i, item in enumerate(current_value):
+            if item is not None:
+                obj[i] = item
+        else:
+            context[key] = obj
+        return obj
+    else:
+        obj = {'': current_value}
+        context[key] = obj
+        return obj
+
+
+def interpret_json_form(pairs):
+    """The application/json form encoding algorithm.
+
+    <https://www.w3.org/TR/html-json-forms/#dfn-application-json-encoding-algorithm>
+
+    """
+    result = {}
+    for key, value in pairs:
+        steps = parse_path(key)
+        context = result
+        for step in steps:
+            try:
+                current_value = operator.getitem(context, step[0])
+            except LookupError:
+                current_value = None
+            context = set_value(context, step, current_value, value)
+    return result

httpie/cli/requestitems.py

@@ -5,7 +5,7 @@ from typing import Callable, Dict, IO, List, Optional, Tuple, Union
 from .argtypes import KeyValueArg
 from .constants import (
     SEPARATORS_GROUP_MULTIPART, SEPARATOR_DATA_EMBED_FILE_CONTENTS,
-    SEPARATOR_DATA_EMBED_RAW_JSON_FILE,
+    SEPARATOR_DATA_EMBED_RAW_JSON_FILE, SEPARATOR_GROUP_NESTED_JSON_ITEMS,
     SEPARATOR_DATA_RAW_JSON, SEPARATOR_DATA_STRING, SEPARATOR_FILE_UPLOAD,
     SEPARATOR_FILE_UPLOAD_TYPE, SEPARATOR_HEADER, SEPARATOR_HEADER_EMPTY,
     SEPARATOR_QUERY_PARAM, SEPARATOR_QUERY_EMBED_FILE, RequestType
@@ -16,7 +16,8 @@ from .dicts import (
     RequestQueryParamsDict,
 )
 from .exceptions import ParseError
-from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys
+from .json_form import interpret_json_form
+from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split
 
 
 class RequestItems:
@@ -67,6 +68,10 @@ class RequestItems:
                 process_data_embed_file_contents_arg,
                 instance.data,
             ),
+            SEPARATOR_GROUP_NESTED_JSON_ITEMS: (
+                process_data_nested_json_embed_args,
+                instance.data,
+            ),
             SEPARATOR_DATA_RAW_JSON: (
                 json_only(instance, process_data_raw_json_embed_arg),
                 instance.data,
@@ -77,6 +82,21 @@ class RequestItems:
             ),
         }
 
+        if instance.is_json:
+            json_item_args, request_item_args = split(
+                request_item_args,
+                lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS
+            )
+            if json_item_args:
+                pairs = [
+                    (arg.key, rules[arg.sep][0](arg))
+                    for arg in json_item_args
+                ]
+                processor_func, target_dict = rules[SEPARATOR_GROUP_NESTED_JSON_ITEMS]
+                value = processor_func(pairs)
+                target_dict.update(value)
+
+        # Then handle all other items.
         for arg in request_item_args:
             processor_func, target_dict = rules[arg.sep]
             value = processor_func(arg)
@@ -172,6 +192,10 @@ def process_data_raw_json_embed_arg(arg: KeyValueArg) -> JSONType:
     return value
 
 
+def process_data_nested_json_embed_args(pairs) -> Dict[str, JSONType]:
+    return interpret_json_form(pairs)
+
+
 def load_text_file(item: KeyValueArg) -> str:
     path = item.value
     try: