Take advantage of streaming.

It's now possible to download huge files with HTTPie, and it's often faster than curl and wget!

httpie/core.py

@@ -16,7 +16,7 @@ import requests.auth
 from requests.compat import str
 
 from .models import HTTPRequest, HTTPResponse, Environment
-from .output import OutputProcessor, format
+from .output import OutputProcessor, formatted_stream
 from .input import (OUT_REQ_BODY, OUT_REQ_HEAD,
                     OUT_RESP_HEAD, OUT_RESP_BODY)
 from .cli import parser
@@ -51,45 +51,35 @@ def get_response(args, env):
             # the `Content-Type` for us.
             args.headers['Content-Type'] = FORM
 
-    try:
-        credentials = None
-        if args.auth:
-            credentials = {
-                'basic': requests.auth.HTTPBasicAuth,
-                'digest': requests.auth.HTTPDigestAuth,
-            }[args.auth_type](args.auth.key, args.auth.value)
+    credentials = None
+    if args.auth:
+        credentials = {
+            'basic': requests.auth.HTTPBasicAuth,
+            'digest': requests.auth.HTTPDigestAuth,
+        }[args.auth_type](args.auth.key, args.auth.value)
 
-        if not (args.url.startswith(HTTP) or args.url.startswith(HTTPS)):
-            scheme = HTTPS if env.progname == 'https' else HTTP
-            url = scheme + args.url
-        else:
-            url = args.url
+    if not (args.url.startswith(HTTP) or args.url.startswith(HTTPS)):
+        scheme = HTTPS if env.progname == 'https' else HTTP
+        url = scheme + args.url
+    else:
+        url = args.url
 
-        return requests.request(
-            method=args.method.lower(),
-            url=url,
-            headers=args.headers,
-            data=args.data,
-            verify={'yes': True, 'no': False}.get(args.verify, args.verify),
-            timeout=args.timeout,
-            auth=credentials,
-            proxies=dict((p.key, p.value) for p in args.proxy),
-            files=args.files,
-            allow_redirects=args.allow_redirects,
-            params=args.params,
-        )
-
-    except (KeyboardInterrupt, SystemExit):
-        env.stderr.write('\n')
-        sys.exit(1)
-    except Exception as e:
-        if args.debug:
-            raise
-        env.stderr.write(str(repr(e) + '\n'))
-        sys.exit(1)
+    return requests.request(
+        method=args.method.lower(),
+        url=url,
+        headers=args.headers,
+        data=args.data,
+        verify={'yes': True, 'no': False}.get(args.verify, args.verify),
+        timeout=args.timeout,
+        auth=credentials,
+        proxies=dict((p.key, p.value) for p in args.proxy),
+        files=args.files,
+        allow_redirects=args.allow_redirects,
+        params=args.params,
+    )
 
 
-def get_output(args, env, request, response):
+def output_stream(args, env, request, response):
     """Format parts of the `request`-`response` exchange
      according to `args` and `env` and return `bytes`.
 
@@ -99,32 +89,38 @@ def get_output(args, env, request, response):
     prettifier = (OutputProcessor(env, pygments_style=args.style)
                   if args.prettify else None)
 
-    if (OUT_REQ_HEAD in args.output_options
-        or OUT_REQ_BODY in args.output_options):
-        exchange.append(format(
+    with_request = (OUT_REQ_HEAD in args.output_options
+                    or OUT_REQ_BODY in args.output_options)
+    with_response = (OUT_RESP_HEAD in args.output_options
+                     or OUT_RESP_BODY in args.output_options)
+
+    if with_request:
+        request_iter = formatted_stream(
             msg=HTTPRequest(request),
             env=env,
             prettifier=prettifier,
             with_headers=OUT_REQ_HEAD in args.output_options,
-            with_body=OUT_REQ_BODY in args.output_options
-        ))
+            with_body=OUT_REQ_BODY in args.output_options)
 
-    if (OUT_RESP_HEAD in args.output_options
-        or OUT_RESP_BODY in args.output_options):
-        exchange.append(format(
+        for chunk in request_iter:
+            yield chunk
+
+        if with_response:
+            yield b'\n\n\n'
+
+    if with_response:
+        response_iter = formatted_stream(
             msg=HTTPResponse(response),
             env=env,
             prettifier=prettifier,
             with_headers=OUT_RESP_HEAD in args.output_options,
             with_body=OUT_RESP_BODY in args.output_options)
-        )
 
-    output = b'\n\n\n'.join(exchange)
+        for chunk in response_iter:
+            yield chunk
 
     if env.stdout_isatty:
-        output += b'\n\n'
-
-    return output
+        yield b'\n\n'
 
 
 def get_exist_status(code, allow_redirects=False):
@@ -155,25 +151,37 @@ def main(args=sys.argv[1:], env=Environment()):
             ' Please use `--output FILE\' instead.\n')
         return 1
 
-    args = parser.parse_args(args=args, env=env)
-
-    response = get_response(args, env)
-
-    status = 0
-
-    if args.check_status:
-        status = get_exist_status(response.status_code,
-                                  args.allow_redirects)
-        if status and not env.stdout_isatty:
-            err = 'http error: %s %s\n' % (
-                response.raw.status, response.raw.reason)
-            env.stderr.write(err)
-
-    output = get_output(args, env, response.request, response)
-
     try:
-        env.stdout.buffer.write(output)
-    except AttributeError:
-        env.stdout.write(output)
+        args = parser.parse_args(args=args, env=env)
+        response = get_response(args, env)
+        status = 0
+
+        if args.check_status:
+            status = get_exist_status(response.status_code,
+                                      args.allow_redirects)
+            if status and not env.stdout_isatty:
+                err = 'http error: %s %s\n' % (
+                    response.raw.status, response.raw.reason)
+                env.stderr.write(err)
+
+        try:
+            # We are writing bytes so we use buffer on Python 3
+            buffer = env.stdout.buffer
+        except AttributeError:
+            buffer = env.stdout
+
+        for chunk in output_stream(args, env, response.request, response):
+            buffer.write(chunk)
+            if env.stdout_isatty:
+                env.stdout.flush()
+
+    except (KeyboardInterrupt, SystemExit):
+        env.stderr.write('\n')
+        return 1
+    except Exception as e:
+        if '--debug' in args:
+            raise
+        env.stderr.write(str(repr(e) + '\n'))
+        return 1
 
     return status