Fix encoding error with non-prettified encoded responses (#1168)

* Fix encoding error with non-prettified encoded responses

Removed `--format-option response.as` an promote `--response-as`: using
the format option would be misleading as it is now also used by non-prettified
responses.

* Encoding refactoring

* split --response-as into --response-mime and --response-charset
* add support for Content-Type charset for requests printed to terminal
* add support charset detection for requests printed to terminal without a Content-Type charset
* etc.

* `test_unicode.py` → `test_encoding.py`

* Drop sequence length check

* Clean-up tests

* [skip ci] Tweaks

* Use the compatible release clause for `charset_normalizer` requirement

Cf. https://www.python.org/dev/peps/pep-0440/#version-specifiers

* Clean-up

* Partially revert d52a4833e4

* Changelog

* Tweak tests

* [skip ci] Better test name

* Cleanup tests and add request body charset detection

* More test suite cleanups

* Cleanup

* Fix code style in test

* Improve detect_encoding() docstring

* Uniformize pytest.mark.parametrize() calls

* [skip ci] Comment out TODOs (will be tackled in a specific PR)

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

httpie/cli/argparser.py

@@ -458,8 +458,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
 
     def _process_format_options(self):
         format_options = self.args.format_options or []
-        if self.args.response_as is not None:
-            format_options.append('response.as:' + self.args.response_as)
         parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS
         for options_group in format_options:
             parsed_options = parse_format_options(options_group, defaults=parsed_options)

httpie/cli/argtypes.py

@@ -242,3 +242,19 @@ PARSED_DEFAULT_FORMAT_OPTIONS = parse_format_options(
     s=','.join(DEFAULT_FORMAT_OPTIONS),
     defaults=None,
 )
+
+
+def response_charset_type(encoding: str) -> str:
+    try:
+        ''.encode(encoding)
+    except LookupError:
+        raise argparse.ArgumentTypeError(
+            f'{encoding!r} is not a supported encoding')
+    return encoding
+
+
+def response_mime_type(mime_type: str) -> str:
+    if mime_type.count('/') != 1:
+        raise argparse.ArgumentTypeError(
+            f'{mime_type!r} doesn’t look like a mime type; use type/subtype')
+    return mime_type

httpie/cli/constants.py

@@ -85,13 +85,11 @@ PRETTY_MAP = {
 PRETTY_STDOUT_TTY_ONLY = object()
 
 
-EMPTY_FORMAT_OPTION = "''"
 DEFAULT_FORMAT_OPTIONS = [
     'headers.sort:true',
     'json.format:true',
     'json.indent:4',
     'json.sort_keys:true',
-    'response.as:' + EMPTY_FORMAT_OPTION,
     'xml.format:true',
     'xml.indent:2',
 ]

httpie/cli/definition.py

@@ -9,7 +9,7 @@ from .. import __doc__, __version__
 from .argparser import HTTPieArgumentParser
 from .argtypes import (
     KeyValueArgType, SessionNameValidator,
-    readable_file_arg,
+    readable_file_arg, response_charset_type, response_mime_type,
 )
 from .constants import (
     DEFAULT_FORMAT_OPTIONS, OUTPUT_OPTIONS,
@@ -310,21 +310,30 @@ output_processing.add_argument(
 )
 
 output_processing.add_argument(
-    '--response-as',
-    metavar='CONTENT_TYPE',
+    '--response-charset',
+    metavar='ENCODING',
+    type=response_charset_type,
     help='''
-    Override the response Content-Type for formatting purposes, e.g.:
+    Override the response encoding for terminal display purposes, e.g.:
 
-        --response-as=application/xml
-        --response-as=charset=utf-8
-        --response-as='application/xml; charset=utf-8'
+        --response-charset=utf8
+        --response-charset=big5
 
-    It is a shortcut for:
-
-        --format-options=response.as:CONTENT_TYPE
     '''
 )
 
+output_processing.add_argument(
+    '--response-mime',
+    metavar='MIME_TYPE',
+    type=response_mime_type,
+    help='''
+    Override the response mime type for coloring and formatting for the terminal, e.g.:
+
+        --response-mime=application/json
+        --response-mime=text/xml
+
+    '''
+)
 
 output_processing.add_argument(
     '--format-options',