Implemented more robust unicode handling.

* Immediatelly convert all args from `bytes` to `str`.
* Added `Environment.stdin_encoding` and `Environment.stdout_encoding`
* Allow unicode characters in HTTP headers and basic auth credentials
  by encoding them using UTF8 instead of latin1 (#212).

tests/__init__.py

@@ -94,7 +94,12 @@ def http(*args, **kwargs):
         `exit_status`: the exit status
         `json`: decoded JSON (if possible) or `None`
 
-    Exceptions are propagated except for SystemExit.
+    Exceptions are propagated.
+
+    If you pass ``error_exit_ok=True``, then error exit statuses
+    won't result into an exception.
+
+    Example:
 
     $ http --auth=user:password GET httpbin.org/basic-auth/user/password
 
@@ -112,6 +117,7 @@ def http(*args, **kwargs):
 
 
     """
+    error_exit_ok = kwargs.pop('error_exit_ok', False)
     env = kwargs.get('env')
     if not env:
         env = kwargs['env'] = TestEnvironment()
@@ -123,21 +129,35 @@ def http(*args, **kwargs):
     if '--debug' not in args and '--traceback' not in args:
         args = ['--traceback'] + args
 
+    def dump_stderr():
+        stderr.seek(0)
+        sys.stderr.write(stderr.read())
+
     try:
         try:
             exit_status = main(args=args, **kwargs)
             if '--download' in args:
                 # Let the progress reporter thread finish.
                 time.sleep(.5)
+        except SystemExit:
+            if error_exit_ok:
+                exit_status = httpie.ExitStatus.ERROR
+            else:
+                dump_stderr()
+                raise
         except Exception:
+            stderr.seek(0)
             sys.stderr.write(stderr.read())
             raise
-        except SystemExit:
-            exit_status = httpie.ExitStatus.ERROR
+        else:
+            if exit_status != httpie.ExitStatus.OK and not error_exit_ok:
+                dump_stderr()
+                raise Exception('Unexpected exit status: %s', exit_status)
 
         stdout.seek(0)
         stderr.seek(0)
         output = stdout.read()
+
         try:
             output = output.decode('utf8')
         except UnicodeDecodeError:
@@ -149,6 +169,9 @@ def http(*args, **kwargs):
         r.stderr = stderr.read()
         r.exit_status = exit_status
 
+        if r.exit_status != httpie.ExitStatus.OK:
+            sys.stderr.write(r.stderr)
+
         return r
 
     finally:

tests/test_auth.py

@@ -1,4 +1,6 @@
 """HTTP authentication-related tests."""
+from unittest import TestCase
+
 import requests
 import pytest
 
@@ -6,38 +8,45 @@ from tests import http, httpbin, HTTP_OK
 import httpie.input
 
 
-class TestAuth:
+class AuthTest(TestCase):
     def test_basic_auth(self):
-        r = http('--auth=user:password',
-                 'GET', httpbin('/basic-auth/user/password'))
+        r = http('--auth=user:password', 'GET',
+                 httpbin('/basic-auth/user/password'))
         assert HTTP_OK in r
-        assert r.json == {'authenticated': True, 'user': 'user'}
+        assert '"authenticated": true' in r
+        assert '"user": "user"' in r
 
     @pytest.mark.skipif(
         requests.__version__ == '0.13.6',
         reason='Redirects with prefetch=False are broken in Requests 0.13.6')
     def test_digest_auth(self):
-        r = http('--auth-type=digest', '--auth=user:password',
-                 'GET', httpbin('/digest-auth/auth/user/password'))
+        r = http('--auth-type=digest', '--auth=user:password', 'GET',
+                 httpbin('/digest-auth/auth/user/password'))
         assert HTTP_OK in r
-        assert r.json == {'authenticated': True, 'user': 'user'}
+        assert r'"authenticated": true' in r
+        assert r'"user": "user"', r
 
     def test_password_prompt(self):
         httpie.input.AuthCredentials._getpass = lambda self, prompt: 'password'
         r = http('--auth', 'user', 'GET', httpbin('/basic-auth/user/password'))
         assert HTTP_OK in r
-        assert r.json == {'authenticated': True, 'user': 'user'}
+        assert '"authenticated": true' in r
+        assert '"user": "user"' in r
 
     def test_credentials_in_url(self):
-        r = http('GET', httpbin('/basic-auth/user/password',
-                                auth='user:password'))
+        url = httpbin('/basic-auth/user/password')
+        url = 'http://user:password@' + url.split('http://', 1)[1]
+        r = http('GET', url)
         assert HTTP_OK in r
-        assert r.json == {'authenticated': True, 'user': 'user'}
+        assert '"authenticated": true' in r
+        assert '"user": "user"' in r
 
     def test_credentials_in_url_auth_flag_has_priority(self):
         """When credentials are passed in URL and via -a at the same time,
          then the ones from -a are used."""
-        r = http('--auth=user:password', 'GET',
-                 httpbin('/basic-auth/user/password', auth='user:wrong'))
+        url = httpbin('/basic-auth/user/password')
+        url = 'http://user:wrong_password@' + url.split('http://', 1)[1]
+        r = http('--auth=user:password', 'GET', url)
         assert HTTP_OK in r
-        assert r.json == {'authenticated': True, 'user': 'user'}
+        assert '"authenticated": true' in r
+        assert '"user": "user"' in r

tests/test_cli.py

@@ -263,7 +263,8 @@ class TestNoOptions:
         assert 'GET /get HTTP/1.1' not in r
 
     def test_invalid_no_options(self):
-        r = http('--no-war', 'GET', httpbin('/get'))
+        r = http('--no-war', 'GET', httpbin('/get'),
+                 error_exit_ok=True)
         assert r.exit_status == 1
         assert 'unrecognized arguments: --no-war' in r.stderr
         assert 'GET /get HTTP/1.1' not in r
@@ -279,6 +280,7 @@ class TestIgnoreStdin:
         assert FILE_CONTENT not in r, "Don't send stdin data."
 
     def test_ignore_stdin_cannot_prompt_password(self):
-        r = http('--ignore-stdin', '--auth=no-password', httpbin('/get'))
+        r = http('--ignore-stdin', '--auth=no-password', httpbin('/get'),
+                 error_exit_ok=True)
         assert r.exit_status == ExitStatus.ERROR
         assert 'because --ignore-stdin' in r.stderr

tests/test_exit_status.py

@@ -22,14 +22,15 @@ class TestExitStatus:
         reason='timeout broken in requests'
                ' (https://github.com/jkbr/httpie/issues/185)')
     def test_timeout_exit_status(self):
-        r = http('--timeout=0.5', 'GET', httpbin('/delay/1'))
+        r = http('--timeout=0.5', 'GET', httpbin('/delay/1'),
+                 error_exit_ok=True)
         assert HTTP_OK in r
         assert r.exit_status == ExitStatus.ERROR_TIMEOUT
 
     def test_3xx_check_status_exits_3_and_stderr_when_stdout_redirected(self):
         env = TestEnvironment(stdout_isatty=False)
         r = http('--check-status', '--headers', 'GET', httpbin('/status/301'),
-                 env=env)
+                 env=env, error_exit_ok=True)
         assert 'HTTP/1.1 301' in r
         assert r.exit_status == ExitStatus.ERROR_HTTP_3XX
         assert '301 moved permanently' in r.stderr.lower()
@@ -38,19 +39,22 @@ class TestExitStatus:
         requests.__version__ == '0.13.6',
         reason='Redirects with prefetch=False are broken in Requests 0.13.6')
     def test_3xx_check_status_redirects_allowed_exits_0(self):
-        r = http('--check-status', '--follow', 'GET', httpbin('/status/301'))
+        r = http('--check-status', '--follow', 'GET', httpbin('/status/301'),
+                 error_exit_ok=True)
         # The redirect will be followed so 200 is expected.
         assert 'HTTP/1.1 200 OK' in r
         assert r.exit_status == ExitStatus.OK
 
     def test_4xx_check_status_exits_4(self):
-        r = http('--check-status', 'GET', httpbin('/status/401'))
+        r = http('--check-status', 'GET', httpbin('/status/401'),
+                 error_exit_ok=True)
         assert 'HTTP/1.1 401' in r
         assert r.exit_status == ExitStatus.ERROR_HTTP_4XX
         # Also stderr should be empty since stdout isn't redirected.
         assert not r.stderr
 
     def test_5xx_check_status_exits_5(self):
-        r = http('--check-status', 'GET', httpbin('/status/500'))
+        r = http('--check-status', 'GET', httpbin('/status/500'),
+                 error_exit_ok=True)
         assert 'HTTP/1.1 500' in r
         assert r.exit_status == ExitStatus.ERROR_HTTP_5XX

tests/test_httpie.py

@@ -13,12 +13,12 @@ class TestHTTPie:
         assert 'HTTPie data:' in r.stderr
 
     def test_help(self):
-        r = http('--help')
+        r = http('--help', error_exit_ok=True)
         assert r.exit_status == httpie.ExitStatus.ERROR
         assert 'https://github.com/jkbr/httpie/issues' in r
 
     def test_version(self):
-        r = http('--version')
+        r = http('--version', error_exit_ok=True)
         assert r.exit_status == httpie.ExitStatus.ERROR
         # FIXME: py3 has version in stdout, py2 in stderr
         assert httpie.__version__ == r.stderr.strip() + r.strip()

tests/test_unicode.py

@@ -0,0 +1,45 @@
+# coding=utf-8
+"""
+Various unicode handling related tests.
+
+"""
+from tests import http, httpbin
+
+
+JP_SUN = u'太陽'
+
+
+class TestUnicode:
+
+    def test_unicode_headers(self):
+        r = http('GET', httpbin('/headers'), u'Test:%s' % JP_SUN)
+        assert r.json['headers']['Test'] == JP_SUN
+
+    def test_unicode_form_item(self):
+        r = http('--form', 'POST', httpbin('/post'), u'test=%s' % JP_SUN)
+        assert r.json['form']['test'] == JP_SUN
+
+    def test_unicode_json_item(self):
+        r = http('--json', 'POST', httpbin('/post'), u'test=%s' % JP_SUN)
+        assert r.json['json']['test'] == JP_SUN
+
+    def test_unicode_raw_json_item(self):
+        r = http('--json', 'POST', httpbin('/post'), u'test:=["%s"]' % JP_SUN)
+        assert r.json['json']['test'] == [JP_SUN]
+
+    def test_unicode_url(self):
+        r = http(httpbin(u'/get?test=' + JP_SUN))
+        assert r.json['args']['test'] == JP_SUN
+
+    def test_unicode_basic_auth(self):
+        # it doesn't really authenticate us because httpbin
+        # doesn't interpret the utf8-encoded auth
+        http('--verbose', '--auth', u'test:%s' % JP_SUN,
+             httpbin(u'/basic-auth/test/' + JP_SUN))
+
+    def test_unicode_digest_auth(self):
+        # it doesn't really authenticate us because httpbin
+        # doesn't interpret the utf8-encoded auth
+        http('--auth-type=digest',
+             '--auth', u'test:%s' % JP_SUN,
+             httpbin(u'/digest-auth/auth/test/' + JP_SUN))

tests/test_uploads.py

@@ -45,11 +45,11 @@ class TestRequestBodyFromFilePath:
     def test_request_body_from_file_by_path_no_field_name_allowed(self):
         env = TestEnvironment(stdin_isatty=True)
         r = http('POST', httpbin('/post'), 'field-name@' + FILE_PATH_ARG,
-                 env=env)
+                 env=env, error_exit_ok=True)
         assert 'perhaps you meant --form?' in r.stderr
 
     def test_request_body_from_file_by_path_no_data_items_allowed(self):
         env = TestEnvironment(stdin_isatty=False)
         r = http('POST', httpbin('/post'), '@' + FILE_PATH_ARG, 'foo=bar',
-                 env=env)
+                 env=env, error_exit_ok=True)
         assert 'cannot be mixed' in r.stderr

tests/test_windows.py

@@ -23,5 +23,6 @@ class TestFakeWindows:
         output_file = os.path.join(
             tempfile.gettempdir(), '__httpie_test_output__')
         r = http('--output', output_file,
-                 '--pretty=all', 'GET', httpbin('/get'), env=env)
+                 '--pretty=all', 'GET', httpbin('/get'),
+                 env=env, error_exit_ok=True)
         assert 'Only terminal output can be colorized on Windows' in r.stderr