Optimize encoding detection (#1243)

* Optimize encoding detection

* Use a threshold based system

tests/fixtures/__init__.py

@@ -32,6 +32,8 @@ JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH)
 # line would be escaped).
 FILE_CONTENT = FILE_PATH.read_text(encoding=UTF8).strip()
 
+ASCII_FILE_CONTENT = "random text" * 10
+
 
 JSON_FILE_CONTENT = JSON_FILE_PATH.read_text(encoding=UTF8)
 BIN_FILE_CONTENT = BIN_FILE_PATH.read_bytes()

tests/test_stream.py

@@ -11,7 +11,12 @@ from httpie.plugins import ConverterPlugin
 from httpie.plugins.registry import plugin_manager
 
 from .utils import StdinBytesIO, http, MockEnvironment, DUMMY_URL
-from .fixtures import BIN_FILE_CONTENT, BIN_FILE_PATH
+from .fixtures import (
+    ASCII_FILE_CONTENT,
+    BIN_FILE_CONTENT,
+    BIN_FILE_PATH,
+    FILE_CONTENT as UNICODE_FILE_CONTENT
+)
 
 PRETTY_OPTIONS = list(PRETTY_MAP.keys())
 
@@ -133,3 +138,9 @@ def test_auto_streaming(http_server, extras, expected):
         for call_arg in env.stdout.write.call_args_list
         if b'test' in call_arg[0][0]
     ]) == expected
+
+
+def test_streaming_encoding_detection(http_server):
+    r = http('--stream', http_server + '/stream/encoding/random')
+    assert ASCII_FILE_CONTENT in r
+    assert UNICODE_FILE_CONTENT in r

tests/utils/http_server.py

@@ -52,6 +52,27 @@ def chunked_drip(handler):
     handler.wfile.write('0\r\n\r\n'.encode('utf-8'))
 
 
+@TestHandler.handler('GET', '/stream/encoding/random')
+def random_encoding(handler):
+    from tests.fixtures import ASCII_FILE_CONTENT, FILE_CONTENT as UNICODE_FILE_CONTENT
+
+    handler.send_response(200)
+    handler.send_header('Transfer-Encoding', 'chunked')
+    handler.end_headers()
+
+    for body in [
+        ASCII_FILE_CONTENT,
+        ASCII_FILE_CONTENT,
+        UNICODE_FILE_CONTENT,
+        UNICODE_FILE_CONTENT,
+        UNICODE_FILE_CONTENT,
+    ]:
+        body += "\n"
+        handler.wfile.write(f'{len(body.encode()):X}\r\n{body}\r\n'.encode())
+
+    handler.wfile.write('0\r\n\r\n'.encode('utf-8'))
+
+
 @pytest.fixture(scope="function")
 def http_server():
     """A custom HTTP server implementation for our tests, that is