Fixed multipart requests output; binary support.

* Bodies of multipart requests are correctly printed (closes #30).
* Binary requests and responses should always work (they are also suppressed
  for terminal output). So things like this work::

     http www.google.com/favicon.ico > favicon.ico

README.rst

@@ -164,7 +164,11 @@ the second one has via ``stdin``::
 Note that when the **output is redirected** (like the examples above), HTTPie
 applies a different set of defaults than for a console output. Namely, colors
 aren't used (unless ``--pretty`` is set) and only the response body
-is printed (unless ``--print`` options specified).
+is printed (unless ``--print`` options specified). It is a convenience
+that allows for things like the one above or downloading (smallish) binary
+files without having to set any flags::
+
+    http www.google.com/favicon.ico > favicon.ico
 
 An alternative to ``stdin`` is to pass a filename whose content will be used
 as the request body. It has the advantage that the ``Content-Type`` header
@@ -330,6 +334,9 @@ Changelog
 =========
 
 * `0.2.7dev`_
+    * Proper handling of binary requests and responses.
+    * Fixed printing of ``multipart/form-data`` requests.
+    * Renamed ``--traceback`` to ``--debug``.
 * `0.2.6`_ (2012-07-26)
     * The short option for ``--headers`` is now ``-h`` (``-t`` has been
       removed, for usage use ``--help``).

httpie/cli.py

@@ -51,9 +51,10 @@ group_type.add_argument(
 #############################################
 
 parser.add_argument(
-    '--traceback', action='store_true', default=False,
+    '--debug', action='store_true', default=False,
     help=_('''
-        Print exception traceback should one occur.
+        Prints exception traceback should one occur and other
+        information useful for debugging HTTPie itself.
     ''')
 )
 

httpie/core.py

@@ -8,7 +8,6 @@ Invocation flow:
     4. Write to `stdout` and exit.
 
 """
-import os
 import sys
 import json
 
@@ -17,9 +16,8 @@ import requests.auth
 from requests.compat import str
 
 from .models import HTTPMessage, Environment
-from .output import OutputProcessor
+from .output import OutputProcessor, format
-from .input import (PRETTIFY_STDOUT_TTY_ONLY,
+from .input import (OUT_REQ_BODY, OUT_REQ_HEAD,
-                    OUT_REQ_BODY, OUT_REQ_HEAD,
                     OUT_RESP_HEAD, OUT_RESP_BODY)
 from .cli import parser
 
@@ -85,7 +83,7 @@ def get_response(args, env):
         env.stderr.write('\n')
         sys.exit(1)
     except Exception as e:
-        if args.traceback:
+        if args.debug:
             raise
         env.stderr.write(str(e.message) + '\n')
         sys.exit(1)
@@ -93,49 +91,35 @@ def get_response(args, env):
 
 def get_output(args, env, request, response):
     """Format parts of the `request`-`response` exchange
-     according to `args` and `env` and return a `unicode`.
+     according to `args` and `env` and return `bytes`.
 
     """
-    do_prettify = (args.prettify is True
-                   or (args.prettify == PRETTIFY_STDOUT_TTY_ONLY
-                       and env.stdout_isatty))
 
-    do_output_request = (OUT_REQ_HEAD in args.output_options
+    exchange = []
-                         or OUT_REQ_BODY in args.output_options)
+    prettifier = (OutputProcessor(env, pygments_style=args.style)
+                  if args.prettify else None)
 
-    do_output_response = (OUT_RESP_HEAD in args.output_options
+    if (OUT_REQ_HEAD in args.output_options
-                          or OUT_RESP_BODY in args.output_options)
+        or OUT_REQ_BODY in args.output_options):
-
+        exchange.append(format(
-    prettifier = None
+            HTTPMessage.from_request(request),
-    if do_prettify:
+            env=env,
-        prettifier = OutputProcessor(
-            env, pygments_style=args.style)
-
-    buf = []
-
-    if do_output_request:
-        req_msg = HTTPMessage.from_request(request)
-        req = req_msg.format(
             prettifier=prettifier,
             with_headers=OUT_REQ_HEAD in args.output_options,
             with_body=OUT_REQ_BODY in args.output_options
-        )
+        ))
-        buf.append(req)
-        buf.append('\n')
-        if do_output_response:
-            buf.append('\n')
 
-    if do_output_response:
+    if (OUT_RESP_HEAD in args.output_options
-        resp_msg = HTTPMessage.from_response(response)
+        or OUT_RESP_BODY in args.output_options):
-        resp = resp_msg.format(
+        exchange.append(format(
+            HTTPMessage.from_response(response),
+            env=env,
             prettifier=prettifier,
             with_headers=OUT_RESP_HEAD in args.output_options,
-            with_body=OUT_RESP_BODY in args.output_options
+            with_body=OUT_RESP_BODY in args.output_options)
         )
-        buf.append(resp)
-        buf.append('\n')
 
-    return ''.join(buf)
+    return b''.join(exchange)
 
 
 def get_exist_status(code, allow_redirects=False):
@@ -172,8 +156,9 @@ def main(args=sys.argv[1:], env=Environment()):
                 response.raw.status, response.raw.reason)
             env.stderr.write(err.encode('utf8'))
 
-    output = get_output(args, env, response.request, response)
+    output_bytes = get_output(args, env, response.request, response)
-    output_bytes = output.encode('utf8')
+
+    # output_bytes = output.encode('utf8')
     f = getattr(env.stdout, 'buffer', env.stdout)
     f.write(output_bytes)
 

httpie/input.py

@@ -103,9 +103,17 @@ class Parser(argparse.ArgumentParser):
             # Stdin already read (if not a tty) so it's save to prompt.
             args.auth.prompt_password()
 
+        if args.files:
+            # Will be read multiple times.
+            for name in args.files:
+                args.files[name] = args.files[name].read()
+
+        if args.prettify == PRETTIFY_STDOUT_TTY_ONLY:
+            args.prettify = env.stdout_isatty
+
         return args
 
-    def _body_from_file(self, args, f):
+    def _body_from_file(self, args, data):
         """Use the content of `f` as the `request.data`.
 
         There can only be one source of request data.
@@ -114,7 +122,7 @@ class Parser(argparse.ArgumentParser):
         if args.data:
             self.error('Request body (from stdin or a file) and request '
                        'data (key=value) cannot be mixed.')
-        args.data = f.read()
+        args.data = data
 
     def _guess_method(self, args, env):
         """Set `args.method` if not specified to either POST or GET
@@ -139,7 +147,7 @@ class Parser(argparse.ArgumentParser):
                     0, KeyValueArgType(*SEP_GROUP_ITEMS).__call__(args.url))
 
             except argparse.ArgumentTypeError as e:
-                if args.traceback:
+                if args.debug:
                     raise
                 self.error(e.message)
 
@@ -169,7 +177,7 @@ class Parser(argparse.ArgumentParser):
                         files=args.files,
                         params=args.params)
         except ParseError as e:
-            if args.traceback:
+            if args.debug:
                 raise
             self.error(e.message)
 
@@ -406,12 +414,12 @@ def parse_items(items, data=None, headers=None, files=None, params=None):
             target = params
         elif item.sep == SEP_FILES:
             try:
-                value = open(os.path.expanduser(item.value), 'r')
+                value = open(os.path.expanduser(value), 'r')
             except IOError as e:
                 raise ParseError(
                     'Invalid argument "%s": %s' % (item.orig, e))
             if not key:
-                key = os.path.basename(value.name)
+                key = os.path.basename(item.value)
             target = files
 
         elif item.sep in [SEP_DATA, SEP_DATA_RAW_JSON]:

httpie/models.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from requests.compat import urlparse, is_windows
+from requests.compat import urlparse, is_windows, bytes, str
 
 
 class Environment(object):
@@ -39,39 +39,40 @@ class Environment(object):
 class HTTPMessage(object):
     """Model representing an HTTP message."""
 
-    def __init__(self, line, headers, body, content_type=None):
+    def __init__(self, line, headers, body, encoding=None, content_type=None):
-        # {Request,Status}-Line
+        """All args are a `str` except for `body` which is a `bytes`."""
-        self.line = line
+
+        assert isinstance(line, str)
+        assert content_type is None or isinstance(content_type, str)
+        assert isinstance(body, bytes)
+
+        self.line = line  # {Request,Status}-Line
         self.headers = headers
         self.body = body
+        self.encoding = encoding
         self.content_type = content_type
 
-    def format(self, prettifier=None, with_headers=True, with_body=True):
+    @classmethod
-        """Return a `unicode` representation of `self`. """
+    def from_response(cls, response):
-        pretty = prettifier is not None
+        """Make an `HTTPMessage` from `requests.models.Response`."""
-        bits = []
+        encoding = response.encoding or None
+        original = response.raw._original_response
+        response_headers = response.headers
+        status_line = str('HTTP/{version} {status} {reason}'.format(
+            version='.'.join(str(original.version)),
+            status=original.status,
+            reason=original.reason
+        ))
+        body = response.content
 
-        if with_headers:
+        return cls(line=status_line,
-            bits.append(self.line)
+                   headers=str(original.msg),
-            bits.append(self.headers)
+                   body=body,
-            if pretty:
+                   encoding=encoding,
-                bits = [
+                   content_type=str(response_headers.get('Content-Type', '')))
-                    prettifier.process_headers('\n'.join(bits))
-                ]
-            if with_body and self.body:
-                bits.append('\n')
 
-        if with_body and self.body:
+    @classmethod
-            if pretty and self.content_type:
+    def from_request(cls, request):
-                bits.append(prettifier.process_body(
-                    self.body, self.content_type))
-            else:
-                bits.append(self.body)
-
-        return '\n'.join(bit.strip() for bit in bits)
-
-    @staticmethod
-    def from_request(request):
         """Make an `HTTPMessage` from `requests.models.Request`."""
 
         url = urlparse(request.url)
@@ -90,54 +91,41 @@ class HTTPMessage(object):
                 qs += type(request)._encode_params(request.params)
 
         # Request-Line
-        request_line = '{method} {path}{query} HTTP/1.1'.format(
+        request_line = str('{method} {path}{query} HTTP/1.1'.format(
             method=request.method,
             path=url.path or '/',
             query=qs
-        )
+        ))
 
         # Headers
         headers = dict(request.headers)
         content_type = headers.get('Content-Type')
+
+        if isinstance(content_type, bytes):
+            # Happens when uploading files.
+            # TODO: submit a bug report for Requests
+            content_type = headers['Content-Type'] = content_type.decode('utf8')
+
         if 'Host' not in headers:
             headers['Host'] = url.netloc
-        headers = '\n'.join(
+        headers = '\n'.join('%s: %s' % (name, value)
-            str('%s: %s') % (name, value)
+                            for name, value in headers.items())
-            for name, value
-            in headers.items()
-        )
 
         # Body
-        try:
+        if request.files:
-            body = request.data
+            body, _ = request._encode_files(request.files)
-        except AttributeError:
+        else:
-            # requests < 0.12.1
+            try:
-            body = request._enc_data
+                body = request.data
-        if isinstance(body, dict):
+            except AttributeError:
-            #noinspection PyUnresolvedReferences
+                # requests < 0.12.1
-            body = type(request)._encode_params(body)
+                body = request._enc_data
+            if isinstance(body, dict):
+                #noinspection PyUnresolvedReferences
+                body = type(request)._encode_params(body)
+            body = body.encode('utf8')
 
-        return HTTPMessage(
+        return cls(line=request_line,
-            line=request_line,
+                   headers=headers,
-            headers=headers,
+                   body=body,
-            body=body,
+                   content_type=content_type)
-            content_type=content_type
-        )
-
-    @classmethod
-    def from_response(cls, response):
-        """Make an `HTTPMessage` from `requests.models.Response`."""
-        encoding = response.encoding or 'ISO-8859-1'
-        original = response.raw._original_response
-        response_headers = response.headers
-        status_line = 'HTTP/{version} {status} {reason}'.format(
-            version='.'.join(str(original.version)),
-            status=original.status,
-            reason=original.reason
-        )
-        body = response.content.decode(encoding) if response.content else ''
-        return cls(
-            line=status_line,
-            headers=str(original.msg),
-            body=body,
-            content_type=response_headers.get('Content-Type'))

httpie/output.py

@@ -11,13 +11,88 @@ from pygments.lexers import get_lexer_for_mimetype
 from pygments.formatters.terminal import TerminalFormatter
 from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.util import ClassNotFound
-from requests.compat import is_windows
+from requests.compat import is_windows, bytes
 
 from . import solarized
+from .models import Environment
 
 
 DEFAULT_STYLE = 'solarized'
 AVAILABLE_STYLES = [DEFAULT_STYLE] + list(STYLE_MAP.keys())
+BINARY_SUPPRESSED_NOTICE = (
+    '+-----------------------------------------+\n'
+    '| NOTE: binary data not shown in terminal |\n'
+    '+-----------------------------------------+'
+)
+
+
+def format(msg, prettifier=None, with_headers=True, with_body=True,
+           env=Environment()):
+    """Return a UTF8-encoded representation of a `models.HTTPMessage`.
+
+    Sometimes it contains binary data so we always return `bytes`.
+
+    If `prettifier` is set or the output is terminal then a binary
+    body is not included in the output replaced with notice.
+
+    Generally, when the `stdout` is redirected, the output match the actual
+    message as match as possible. When we are `--pretty` set (or implied)
+    or when the output is a terminal, then we prefer readability over
+    precision.
+
+    """
+    chunks = []
+
+    if with_headers:
+        headers = '\n'.join([msg.line, msg.headers]).encode('utf8')
+
+        if prettifier:
+            # Prettifies work on unicode
+            headers = prettifier.process_headers(
+                headers.decode('utf8')).encode('utf8')
+
+        chunks.append(headers.strip())
+
+        if with_body and msg.body or env.stdout_isatty:
+            chunks.append(b'\n\n')
+
+    if with_body and msg.body:
+
+        body = msg.body
+        bin_suppressed = False
+
+        if prettifier or env.stdout_isatty:
+
+            # Convert body to UTF8.
+            try:
+                body = msg.body.decode(msg.encoding or 'utf8')
+            except UnicodeDecodeError:
+                # Assume binary. It could also be that `self.encoding`
+                # doesn't correspond to the actual encoding.
+                bin_suppressed = True
+                body = BINARY_SUPPRESSED_NOTICE.encode('utf8')
+                if not with_headers:
+                    body = b'\n' + body
+
+            else:
+                # Convert (possibly back) to UTF8.
+                body = body.encode('utf8')
+
+        if not bin_suppressed and prettifier and msg.content_type:
+            # Prettifies work on unicode.
+            body = (prettifier
+                    .process_body(body.decode('utf8'),
+                                  msg.content_type)
+                    .encode('utf8').strip())
+
+        chunks.append(body)
+
+        if env.stdout_isatty:
+            chunks.append(b'\n\n')
+
+    formatted = b''.join(chunks)
+
+    return formatted
 
 
 class HTTPLexer(lexer.RegexLexer):

tests/tests.py

@@ -25,9 +25,13 @@ import json
 import tempfile
 import unittest
 import argparse
-import requests
+try:
-from requests.compat import is_py26, is_py3, str
+    from urllib.request import urlopen
+except ImportError:
+    from urllib2 import urlopen
 
+import requests
+from requests.compat import is_py26, is_py3, bytes, str
 
 #################################################################
 # Utils/setup
@@ -40,6 +44,7 @@ sys.path.insert(0, os.path.realpath(os.path.join(TESTS_ROOT, '..')))
 from httpie import input
 from httpie.models import Environment
 from httpie.core import main, get_output
+from httpie.output import BINARY_SUPPRESSED_NOTICE
 
 
 HTTPBIN_URL = os.environ.get('HTTPBIN_URL',
@@ -55,17 +60,22 @@ def httpbin(path):
     return HTTPBIN_URL + path
 
 
-class Response(str):
+class BytesResponse(bytes):
-    """
-    A unicode subclass holding the output of `main()`, and also
-    the exit status, the contents of ``stderr``, and de-serialized
-    JSON response (if possible).
 
-    """
     exit_status = None
     stderr = None
     json = None
 
+    def __eq__(self, other):
+        return super(BytesResponse, self).__eq__(other)
+
+class StrResponse(str):
+    exit_status = None
+    stderr = None
+    json = None
+    def __eq__(self, other):
+        return super(StrResponse, self).__eq__(other)
+
 
 def http(*args, **kwargs):
     """
@@ -85,34 +95,40 @@ def http(*args, **kwargs):
     stdout = kwargs['env'].stdout = tempfile.TemporaryFile()
     stderr = kwargs['env'].stderr = tempfile.TemporaryFile()
 
-    exit_status = main(args=['--traceback'] + list(args), **kwargs)
+    exit_status = main(args=['--debug'] + list(args), **kwargs)
 
     stdout.seek(0)
     stderr.seek(0)
 
-    r = Response(stdout.read().decode('utf8'))
+    output = stdout.read()
+
+    try:
+        r = StrResponse(output.decode('utf8'))
+    except UnicodeDecodeError:
+        r = BytesResponse(output)
+    else:
+        if TERMINAL_COLOR_PRESENCE_CHECK not in r:
+            # De-serialize JSON body if possible.
+            if r.strip().startswith('{'):
+                #noinspection PyTypeChecker
+                r.json = json.loads(r)
+            elif r.count('Content-Type:') == 1 and 'application/json' in r:
+                try:
+                    j = r.strip()[r.strip().rindex('\n\n'):]
+                except ValueError:
+                    pass
+                else:
+                    try:
+                        r.json = json.loads(j)
+                    except ValueError:
+                        pass
+
     r.stderr = stderr.read().decode('utf8')
     r.exit_status = exit_status
 
     stdout.close()
     stderr.close()
 
-    if TERMINAL_COLOR_PRESENCE_CHECK not in r:
-        # De-serialize JSON body if possible.
-        if r.strip().startswith('{'):
-            #noinspection PyTypeChecker
-            r.json = json.loads(r)
-        elif r.count('Content-Type:') == 1 and 'application/json' in r:
-            try:
-                j = r.strip()[r.strip().rindex('\n\n'):]
-            except ValueError:
-                pass
-            else:
-                try:
-                    r.json = json.loads(j)
-                except ValueError:
-                    pass
-
     return r
 
 
@@ -506,7 +522,7 @@ class VerboseFlagTest(BaseTestCase):
 class MultipartFormDataFileUploadTest(BaseTestCase):
 
     def test_non_existent_file_raises_parse_error(self):
-        self.assertRaises(input.ParseError, http,
+        self.assertRaises(SystemExit, http,
             '--form',
             '--traceback',
             'POST',
@@ -517,16 +533,58 @@ class MultipartFormDataFileUploadTest(BaseTestCase):
     def test_upload_ok(self):
         r = http(
             '--form',
+            '--verbose',
             'POST',
             httpbin('/post'),
             'test-file@%s' % TEST_FILE_PATH,
             'foo=bar'
         )
+
         self.assertIn('HTTP/1.1 200', r)
-        self.assertIn('"test-file": "%s' % TEST_FILE_CONTENT, r)
+        self.assertIn('Content-Disposition: form-data; name="foo"', r)
+        self.assertIn('Content-Disposition: form-data; name="test-file";'
+                      ' filename="test-file"', r)
+        self.assertEqual(r.count(TEST_FILE_CONTENT), 2)
         self.assertIn('"foo": "bar"', r)
 
 
+class TestBinaryResponses(BaseTestCase):
+
+    url = 'http://www.google.com/favicon.ico'
+
+    @property
+    def bindata(self):
+        if not hasattr(self, '_bindata'):
+            self._bindata = urlopen(self.url).read()
+        return self._bindata
+
+    def test_binary_suppresses_when_terminal(self):
+        r = http(
+            'GET',
+            self.url
+        )
+        self.assertIn(BINARY_SUPPRESSED_NOTICE, r)
+
+    def test_binary_suppresses_when_not_terminal_but_pretty(self):
+        r = http(
+            '--pretty',
+            'GET',
+            self.url,
+            env=Environment(stdin_isatty=True,
+                            stdout_isatty=False)
+        )
+        self.assertIn(BINARY_SUPPRESSED_NOTICE, r)
+
+    def test_binary_included_and_correct_when_suitable(self):
+        r = http(
+            'GET',
+            self.url,
+            env=Environment(stdin_isatty=True,
+                            stdout_isatty=False)
+        )
+        self.assertEqual(r, self.bindata)
+
+
 class RequestBodyFromFilePathTest(BaseTestCase):
     """
     `http URL @file'
@@ -764,9 +822,9 @@ class ArgumentParserTestCase(unittest.TestCase):
 
         self.parser._guess_method(args, Environment())
 
-        self.assertEquals(args.method, 'GET')
+        self.assertEqual(args.method, 'GET')
-        self.assertEquals(args.url, 'http://example.com/')
+        self.assertEqual(args.url, 'http://example.com/')
-        self.assertEquals(args.items, [])
+        self.assertEqual(args.items, [])
 
     def test_guess_when_method_not_set(self):
         args = argparse.Namespace()
@@ -779,9 +837,9 @@ class ArgumentParserTestCase(unittest.TestCase):
             stdout_isatty=True,
         ))
 
-        self.assertEquals(args.method, 'GET')
+        self.assertEqual(args.method, 'GET')
-        self.assertEquals(args.url, 'http://example.com/')
+        self.assertEqual(args.url, 'http://example.com/')
-        self.assertEquals(args.items, [])
+        self.assertEqual(args.items, [])
 
     def test_guess_when_method_set_but_invalid_and_data_field(self):
         args = argparse.Namespace()
@@ -791,9 +849,9 @@ class ArgumentParserTestCase(unittest.TestCase):
 
         self.parser._guess_method(args, Environment())
 
-        self.assertEquals(args.method, 'POST')
+        self.assertEqual(args.method, 'POST')
-        self.assertEquals(args.url, 'http://example.com/')
+        self.assertEqual(args.url, 'http://example.com/')
-        self.assertEquals(
+        self.assertEqual(
             args.items,
             [input.KeyValue(
                 key='data', value='field', sep='=', orig='data=field')])
@@ -809,9 +867,9 @@ class ArgumentParserTestCase(unittest.TestCase):
             stdout_isatty=True,
         ))
 
-        self.assertEquals(args.method, 'GET')
+        self.assertEqual(args.method, 'GET')
-        self.assertEquals(args.url, 'http://example.com/')
+        self.assertEqual(args.url, 'http://example.com/')
-        self.assertEquals(
+        self.assertEqual(
             args.items,
             [input.KeyValue(
                 key='test', value='header', sep=':', orig='test:header')])
@@ -827,7 +885,7 @@ class ArgumentParserTestCase(unittest.TestCase):
 
         self.parser._guess_method(args, Environment())
 
-        self.assertEquals(args.items, [
+        self.assertEqual(args.items, [
             input.KeyValue(
                 key='new_item', value='a', sep='=', orig='new_item=a'),
             input.KeyValue(key
@@ -879,8 +937,7 @@ class UnicodeOutputTestCase(BaseTestCase):
         args.style = 'default'
 
         # colorized output contains escape sequences
-        output = get_output(args, Environment(), response.request, response)
+        output = get_output(args, Environment(), response.request, response).decode('utf8')
-
         for key, value in response_dict.items():
             self.assertIn(key, output)
             self.assertIn(value, output)