python/httpie-cli
1
0
Fork 0
mirror of https://github.com/httpie/cli.git synced 2024-11-24 08:22:22 +02:00
Code

Add one-by-one processing of each HTTP request or response and --offline

Browse Source
This commit is contained in:
Jakub Roztocil 2019-09-03 17:14:39 +02:00
parent c946b3d34f
commit bece3c77bb
22 changed files with 536 additions and 483 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.rst
View File

@ -11,6 +11,12 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
* Removed Python 2.7 support (`EOL Jan 2020 <https://www.python.org/dev/peps/pep-0373/>`_).
* Removed the default 30-second connection ``--timeout`` limit.
* Removed Python’s default limit of 100 response headers.
* Replaced the old collect-all-then-process handling of HTTP communication
with one-by-one processing of each HTTP request or response as they become
available. This means that you can see headers immediately,
see what is being send even when the request fails, etc.
* Added ``--offline`` to allow building an HTTP request and printing it but not
actually sending it over the network.
* Added ``--max-headers`` to allow setting the max header limit.
* Added ``--compress`` to allow request body compression.
* Added ``--ignore-netrc`` to allow bypassing credentials from ``.netrc``.

11
httpie/cli/argparser.py
View File

@ -13,6 +13,7 @@ from httpie.cli.constants import (
OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED, OUT_RESP_BODY, PRETTY_MAP,
PRETTY_STDOUT_TTY_ONLY, SEPARATOR_CREDENTIALS, SEPARATOR_GROUP_ALL_ITEMS,
SEPARATOR_GROUP_DATA_ITEMS, URL_SCHEME_RE,
OUTPUT_OPTIONS_DEFAULT_OFFLINE,
)
from httpie.cli.exceptions import ParseError
from httpie.cli.requestitems import RequestItems
@ -348,12 +349,12 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
if self.args.output_options is None:
if self.args.verbose:
self.args.output_options = ''.join(OUTPUT_OPTIONS)
elif self.args.offline:
self.args.output_options = OUTPUT_OPTIONS_DEFAULT_OFFLINE
elif not self.env.stdout_isatty:
self.args.output_options = OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED
else:
self.args.output_options = (
OUTPUT_OPTIONS_DEFAULT
if self.env.stdout_isatty
else OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED
)
self.args.output_options = OUTPUT_OPTIONS_DEFAULT
if self.args.output_options_history is None:
self.args.output_options_history = self.args.output_options

1
httpie/cli/constants.py
View File

@ -86,6 +86,7 @@ PRETTY_STDOUT_TTY_ONLY = object()
# Defaults
OUTPUT_OPTIONS_DEFAULT = OUT_RESP_HEAD + OUT_RESP_BODY
OUTPUT_OPTIONS_DEFAULT_STDOUT_REDIRECTED = OUT_RESP_BODY
OUTPUT_OPTIONS_DEFAULT_OFFLINE = OUT_REQ_HEAD + OUT_REQ_BODY
SSL_VERSION_ARG_MAPPING = {
'ssl2.3': 'PROTOCOL_SSLv23',

8
httpie/cli/definition.py
View File

@ -468,6 +468,14 @@ auth.add_argument(
network = parser.add_argument_group(title='Network')
network.add_argument(
'--offline',
default=False,
action='store_true',
help="""
Build the request and print it but don’t actually send it.
"""
)
network.add_argument(
'--proxy',
default=[],

171
httpie/client.py
View File

@ -5,14 +5,16 @@ import sys
import zlib
from contextlib import contextmanager
from pathlib import Path
from typing import Iterable, Union
import requests
from requests.adapters import HTTPAdapter
from httpie import __version__, sessions
from httpie import __version__
from httpie.cli.constants import SSL_VERSION_ARG_MAPPING
from httpie.cli.dicts import RequestHeadersDict
from httpie.plugins import plugin_manager
from httpie.sessions import get_httpie_session
from httpie.utils import repr_dict
@ -24,12 +26,90 @@ try:
except (ImportError, AttributeError):
pass
FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=utf-8'
JSON_CONTENT_TYPE = 'application/json'
JSON_ACCEPT = f'{JSON_CONTENT_TYPE}, */*'
DEFAULT_UA = f'HTTPie/{__version__}'
def collect_messages(
args: argparse.Namespace,
config_dir: Path
) -> Iterable[Union[requests.PreparedRequest, requests.Response]]:
httpie_session = None
httpie_session_headers = None
if args.session or args.session_read_only:
httpie_session = get_httpie_session(
config_dir=config_dir,
session_name=args.session or args.session_read_only,
host=args.headers.get('Host'),
url=args.url,
)
httpie_session_headers = httpie_session.headers
request_kwargs = make_request_kwargs(
args=args,
base_headers=httpie_session_headers,
)
send_kwargs = make_send_kwargs(args)
send_kwargs_mergeable_from_env = make_send_kwargs_mergeable_from_env(args)
requests_session = build_requests_session(
ssl_version=args.ssl_version,
compress_arg=args.compress,
)
if httpie_session:
httpie_session.update_headers(request_kwargs['headers'])
requests_session.cookies = httpie_session.cookies
if args.auth_plugin:
# Save auth from CLI to HTTPie session.
httpie_session.auth = {
'type': args.auth_plugin.auth_type,
'raw_auth': args.auth_plugin.raw_auth,
}
elif httpie_session.auth:
# Apply auth from HTTPie session
request_kwargs['auth'] = httpie_session.auth
if args.debug:
# TODO: reflect the split between request and send kwargs.
dump_request(request_kwargs)
request = requests.Request(**request_kwargs)
prepared_request = requests_session.prepare_request(request)
response_count = 0
while prepared_request:
yield prepared_request
if not args.offline:
send_kwargs_merged = requests_session.merge_environment_settings(
url=prepared_request.url,
**send_kwargs_mergeable_from_env,
)
with max_headers(args.max_headers):
response = requests_session.send(
request=prepared_request,
**send_kwargs_merged,
**send_kwargs,
)
response_count += 1
if response.next:
if args.max_redirects and response_count == args.max_redirects:
raise requests.TooManyRedirects
if args.follow:
prepared_request = response.next
if args.all:
yield response
continue
yield response
break
if httpie_session:
if httpie_session.is_new() or not args.session_read_only:
httpie_session.cookies = requests_session.cookies
httpie_session.save()
# noinspection PyProtectedMember
@contextmanager
def max_headers(limit):
@ -83,14 +163,14 @@ class HTTPieHTTPAdapter(HTTPAdapter):
def build_requests_session(
ssl_version: str,
compress_arg: int,
ssl_version: str = None,
) -> requests.Session:
requests_session = requests.Session()
# Install our adapter.
adapter = HTTPieHTTPAdapter(
ssl_version=ssl_version,
ssl_version=SSL_VERSION_ARG_MAPPING[ssl_version] if ssl_version else None,
compression_enabled=compress_arg > 0,
compress_always=compress_arg > 1,
)
@ -108,40 +188,6 @@ def build_requests_session(
return requests_session
def get_response(
args: argparse.Namespace,
config_dir: Path
) -> requests.Response:
"""Send the request and return a `request.Response`."""
ssl_version = None
if args.ssl_version:
ssl_version = SSL_VERSION_ARG_MAPPING[args.ssl_version]
requests_session = build_requests_session(
ssl_version=ssl_version,
compress_arg=args.compress
)
requests_session.max_redirects = args.max_redirects
with max_headers(args.max_headers):
if not args.session and not args.session_read_only:
requests_kwargs = make_requests_kwargs(args)
if args.debug:
dump_request(requests_kwargs)
response = requests_session.request(**requests_kwargs)
else:
response = sessions.get_response(
requests_session=requests_session,
args=args,
config_dir=config_dir,
session_name=args.session or args.session_read_only,
read_only=bool(args.session_read_only),
)
return response
def dump_request(kwargs: dict):
sys.stderr.write(
f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n')
@ -181,12 +227,40 @@ def make_default_headers(args: argparse.Namespace) -> RequestHeadersDict:
return default_headers
def make_requests_kwargs(
def make_send_kwargs(args: argparse.Namespace) -> dict:
kwargs = {
'timeout': args.timeout or None,
'allow_redirects': False,
}
return kwargs
def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
cert = None
if args.cert:
cert = args.cert
if args.cert_key:
cert = cert, args.cert_key
kwargs = {
'proxies': {p.key: p.value for p in args.proxy},
'stream': True,
'verify': {
'yes': True,
'true': True,
'no': False,
'false': False,
}.get(args.verify.lower(), args.verify),
'cert': cert,
}
return kwargs
def make_request_kwargs(
args: argparse.Namespace,
base_headers: RequestHeadersDict = None
) -> dict:
"""
Translate our `args` into `requests.request` keyword arguments.
Translate our `args` into `requests.Request` keyword arguments.
"""
# Serialize JSON data, if needed.
@ -207,31 +281,14 @@ def make_requests_kwargs(
headers.update(args.headers)
headers = finalize_headers(headers)
cert = None
if args.cert:
cert = args.cert
if args.cert_key:
cert = cert, args.cert_key
kwargs = {
'stream': True,
'method': args.method.lower(),
'url': args.url,
'headers': headers,
'data': data,
'verify': {
'yes': True,
'true': True,
'no': False,
'false': False,
}.get(args.verify.lower(), args.verify),
'cert': cert,
'timeout': args.timeout or None,
'auth': args.auth,
'proxies': {p.key: p.value for p in args.proxy},
'files': args.files,
'allow_redirects': args.follow,
'params': args.params,
'files': args.files,
}
return kwargs

310
httpie/core.py
View File

@ -1,17 +1,4 @@
"""This module provides the main functionality of HTTPie.
Invocation flow:
1. Read, validate and process the input (args, `stdin`).
2. Create and send a request.
3. Stream, and possibly process and format, the parts
of the request-response exchange selected by output options.
4. Simultaneously write to `stdout`
5. Exit.
"""
import argparse
import errno
import platform
import sys
from typing import Callable, List, Union
@ -21,162 +8,13 @@ from pygments import __version__ as pygments_version
from requests import __version__ as requests_version
from httpie import ExitStatus, __version__ as httpie_version
from httpie.client import get_response
from httpie.client import collect_messages
from httpie.context import Environment
from httpie.downloads import Downloader
from httpie.output.streams import (
build_output_stream,
write_stream,
write_stream_with_colors_win_py3,
)
from httpie.output.writer import write_message, write_stream
from httpie.plugins import plugin_manager
def get_exit_status(http_status: int, follow=False) -> ExitStatus:
"""Translate HTTP status code to exit status code."""
if 300 <= http_status <= 399 and not follow:
# Redirect
return ExitStatus.ERROR_HTTP_3XX
elif 400 <= http_status <= 499:
# Client Error
return ExitStatus.ERROR_HTTP_4XX
elif 500 <= http_status <= 599:
# Server Error
return ExitStatus.ERROR_HTTP_5XX
else:
return ExitStatus.SUCCESS
def print_debug_info(env: Environment):
env.stderr.writelines([
'HTTPie %s\n' % httpie_version,
'Requests %s\n' % requests_version,
'Pygments %s\n' % pygments_version,
'Python %s\n%s\n' % (sys.version, sys.executable),
'%s %s' % (platform.system(), platform.release()),
])
env.stderr.write('\n\n')
env.stderr.write(repr(env))
env.stderr.write('\n')
def decode_args(
args: List[Union[str, bytes]],
stdin_encoding: str
) -> List[str]:
"""
Convert all bytes args to str
by decoding them using stdin encoding.
"""
return [
arg.decode(stdin_encoding)
if type(arg) == bytes else arg
for arg in args
]
def program(
args: argparse.Namespace,
env: Environment,
log_error: Callable
) -> ExitStatus:
"""
The main program without error handling
:param args: parsed args (argparse.Namespace)
:type env: Environment
:param log_error: error log function
:return: status code
"""
exit_status = ExitStatus.SUCCESS
downloader = None
show_traceback = args.debug or args.traceback
try:
if args.download:
args.follow = True # --download implies --follow.
downloader = Downloader(
output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume
)
downloader.pre_request(args.headers)
final_response = get_response(args, config_dir=env.config.directory)
if args.all:
responses = final_response.history + [final_response]
else:
responses = [final_response]
for response in responses:
if args.check_status or downloader:
exit_status = get_exit_status(
http_status=response.status_code,
follow=args.follow
)
if not env.stdout_isatty and exit_status != ExitStatus.SUCCESS:
log_error(
'HTTP %s %s', response.raw.status, response.raw.reason,
level='warning'
)
write_stream_kwargs = {
'stream': build_output_stream(
args=args,
env=env,
request=response.request,
response=response,
output_options=(
args.output_options
if response is final_response
else args.output_options_history
)
),
# NOTE: `env.stdout` will in fact be `stderr` with `--download`
'outfile': env.stdout,
'flush': env.stdout_isatty or args.stream
}
try:
if env.is_windows and 'colors' in args.prettify:
write_stream_with_colors_win_py3(**write_stream_kwargs)
else:
write_stream(**write_stream_kwargs)
except IOError as e:
if not show_traceback and e.errno == errno.EPIPE:
# Ignore broken pipes unless --traceback.
env.stderr.write('\n')
else:
raise
if downloader and exit_status == ExitStatus.SUCCESS:
# Last response body download.
download_stream, download_to = downloader.start(final_response)
write_stream(
stream=download_stream,
outfile=download_to,
flush=False,
)
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
log_error('Incomplete download: size=%d; downloaded=%d' % (
downloader.status.total_size,
downloader.status.downloaded
))
return exit_status
finally:
if downloader and not downloader.finished:
downloader.failed()
if (not isinstance(args, list) and args.output_file
and args.output_file_specified):
args.output_file.close()
def main(
args: List[Union[str, bytes]] = sys.argv,
env=Environment(),
@ -191,15 +29,13 @@ def main(
Return exit status code.
"""
args = decode_args(args, env.stdin_encoding)
args = decode_raw_args(args, env.stdin_encoding)
program_name, *args = args
plugin_manager.load_installed_plugins()
def log_error(msg, *args, **kwargs):
msg = msg % args
level = kwargs.get('level', 'error')
def log_error(msg, level='error'):
assert level in ['error', 'warning']
env.stderr.write('\nhttp: %s: %s\n' % (level, msg))
env.stderr.write(f'\n{program_name}: {level}: {msg}\n')
from httpie.cli.definition import parser
@ -256,22 +92,146 @@ def main(
exit_status = ExitStatus.ERROR
except requests.Timeout:
exit_status = ExitStatus.ERROR_TIMEOUT
log_error('Request timed out (%ss).', parsed_args.timeout)
log_error(f'Request timed out ({parsed_args.timeout}s).')
except requests.TooManyRedirects:
exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
log_error('Too many redirects (--max-redirects=%s).',
parsed_args.max_redirects)
log_error(
f'Too many redirects'
f' (--max-redirects=parsed_args.max_redirects).'
)
except Exception as e:
# TODO: Further distinction between expected and unexpected errors.
msg = str(e)
if hasattr(e, 'request'):
request = e.request
if hasattr(request, 'url'):
msg += ' while doing %s request to URL: %s' % (
request.method, request.url)
log_error('%s: %s', type(e).__name__, msg)
msg = (
f'{msg} while doing a {request.method}'
f' request to URL: {request.url}'
)
log_error(f'{type(e).__name__}: {msg}')
if include_traceback:
raise
exit_status = ExitStatus.ERROR
return exit_status
def program(
args: argparse.Namespace,
env: Environment,
log_error: Callable
) -> ExitStatus:
"""
The main program without error handling.
"""
exit_status = ExitStatus.SUCCESS
downloader = None
try:
if args.download:
args.follow = True # --download implies --follow.
downloader = Downloader(
output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume
)
downloader.pre_request(args.headers)
initial_request = None
final_response = None
for message in collect_messages(args, env.config.directory):
write_message(
requests_message=message,
env=env,
args=args,
)
if isinstance(message, requests.PreparedRequest):
if not initial_request:
initial_request = message
else:
final_response = message
if args.check_status or downloader:
exit_status = get_exit_status(
http_status=message.status_code,
follow=args.follow
)
if not env.stdout_isatty and exit_status != ExitStatus.SUCCESS:
log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning'
)
if downloader and exit_status == ExitStatus.SUCCESS:
# Last response body download.
download_stream, download_to = downloader.start(
initial_url=initial_request.url,
final_response=final_response,
)
write_stream(
stream=download_stream,
outfile=download_to,
flush=False,
)
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
log_error('Incomplete download: size=%d; downloaded=%d' % (
downloader.status.total_size,
downloader.status.downloaded
))
return exit_status
finally:
if downloader and not downloader.finished:
downloader.failed()
if (not isinstance(args, list) and args.output_file
and args.output_file_specified):
args.output_file.close()
def get_exit_status(http_status: int, follow=False) -> ExitStatus:
"""Translate HTTP status code to exit status code."""
if 300 <= http_status <= 399 and not follow:
# Redirect
return ExitStatus.ERROR_HTTP_3XX
elif 400 <= http_status <= 499:
# Client Error
return ExitStatus.ERROR_HTTP_4XX
elif 500 <= http_status <= 599:
# Server Error
return ExitStatus.ERROR_HTTP_5XX
else:
return ExitStatus.SUCCESS
def print_debug_info(env: Environment):
env.stderr.writelines([
'HTTPie %s\n' % httpie_version,
'Requests %s\n' % requests_version,
'Pygments %s\n' % pygments_version,
'Python %s\n%s\n' % (sys.version, sys.executable),
'%s %s' % (platform.system(), platform.release()),
])
env.stderr.write('\n\n')
env.stderr.write(repr(env))
env.stderr.write('\n')
def decode_raw_args(
args: List[Union[str, bytes]],
stdin_encoding: str
) -> List[str]:
"""
Convert all bytes args to str
by decoding them using stdin encoding.
"""
return [
arg.decode(stdin_encoding)
if type(arg) == bytes else arg
for arg in args
]

22
httpie/downloads.py
View File

@ -121,7 +121,7 @@ def filename_from_content_disposition(
return filename
def filename_from_url(url: str, content_type: str) -> str:
def filename_from_url(url: str, content_type: Optional[str]) -> str:
fn = urlsplit(url).path.rstrip('/')
fn = os.path.basename(fn) if fn else 'index'
if '.' not in fn and content_type:
@ -230,11 +230,16 @@ class Downloader:
request_headers['Range'] = 'bytes=%d-' % bytes_have
self._resumed_from = bytes_have
def start(self, final_response: requests.Response) -> Tuple[RawStream, IO]:
def start(
self,
initial_url: str,
final_response: requests.Response
) -> Tuple[RawStream, IO]:
"""
Initiate and return a stream for `response` body with progress
callback attached. Can be called only once.
:param initial_url: The original requested URL
:param final_response: Initiated response object with headers already fetched
:return: RawStream, output_file
@ -251,7 +256,9 @@ class Downloader:
if not self._output_file:
self._output_file = self._get_output_file_from_response(
final_response)
initial_url=initial_url,
final_response=final_response,
)
else:
# `--output, -o` provided
if self._resume and final_response.status_code == PARTIAL_CONTENT:
@ -322,7 +329,8 @@ class Downloader:
@staticmethod
def _get_output_file_from_response(
final_response: requests.Response
initial_url: str,
final_response: requests.Response,
) -> IO:
# Output file not specified. Pick a name that doesn't exist yet.
filename = None
@ -330,12 +338,8 @@ class Downloader:
filename = filename_from_content_disposition(
final_response.headers['Content-Disposition'])
if not filename:
initial_response = (
final_response.history[0] if final_response.history
else final_response
)
filename = filename_from_url(
url=initial_response.url,
url=initial_url,
content_type=final_response.headers.get('Content-Type'),
)
unique_filename = get_unique_filename(filename)

143
httpie/output/streams.py
View File

@ -1,14 +1,8 @@
import argparse
from itertools import chain
from typing import Callable, IO, Iterable, TextIO, Tuple, Type, Union
from typing import Callable, Iterable, Union
import requests
from httpie.cli.constants import (
OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD,
)
from httpie.context import Environment
from httpie.models import HTTPMessage, HTTPRequest, HTTPResponse
from httpie.models import HTTPMessage
from httpie.output.processing import Conversion, Formatting
@ -27,143 +21,14 @@ class BinarySuppressedError(Exception):
message = BINARY_SUPPRESSED_NOTICE
def write_stream(
stream: 'BaseStream',
outfile: Union[IO, TextIO],
flush: bool
):
"""Write the output stream."""
try:
# Writing bytes so we use the buffer interface (Python 3).
buf = outfile.buffer
except AttributeError:
buf = outfile
for chunk in stream:
buf.write(chunk)
if flush:
outfile.flush()
def write_stream_with_colors_win_py3(
stream: 'BaseStream',
outfile: TextIO,
flush: bool
):
"""Like `write`, but colorized chunks are written as text
directly to `outfile` to ensure it gets processed by colorama.
Applies only to Windows with Python 3 and colorized terminal output.
"""
color = b'\x1b['
encoding = outfile.encoding
for chunk in stream:
if color in chunk:
outfile.write(chunk.decode(encoding))
else:
outfile.buffer.write(chunk)
if flush:
outfile.flush()
def build_output_stream(
args: argparse.Namespace,
env: Environment,
request: requests.Request,
response: requests.Response,
output_options: str
) -> Iterable[bytes]:
"""Build and return a chain of iterators over the `request`-`response`
exchange each of which yields `bytes` chunks.
"""
req_h = OUT_REQ_HEAD in output_options
req_b = OUT_REQ_BODY in output_options
resp_h = OUT_RESP_HEAD in output_options
resp_b = OUT_RESP_BODY in output_options
req = req_h or req_b
resp = resp_h or resp_b
output = []
stream_class, stream_kwargs = get_stream_type_and_kwargs(
env=env, args=args)
if req:
output.append(
stream_class(
msg=HTTPRequest(request),
with_headers=req_h,
with_body=req_b,
**stream_kwargs,
)
)
if req_b and resp:
# Request/Response separator.
output.append([b'\n\n'])
if resp:
output.append(
stream_class(
msg=HTTPResponse(response),
with_headers=resp_h,
with_body=resp_b,
**stream_kwargs,
)
)
if env.stdout_isatty and resp_b:
# Ensure a blank line after the response body.
# For terminal output only.
output.append([b'\n\n'])
return chain(*output)
def get_stream_type_and_kwargs(
env: Environment,
args: argparse.Namespace
) -> Tuple[Type['BaseStream'], dict]:
"""Pick the right stream type and kwargs for it based on `env` and `args`.
"""
if not env.stdout_isatty and not args.prettify:
stream_class = RawStream
stream_kwargs = {
'chunk_size': (
RawStream.CHUNK_SIZE_BY_LINE
if args.stream
else RawStream.CHUNK_SIZE
)
}
elif args.prettify:
stream_class = PrettyStream if args.stream else BufferedPrettyStream
stream_kwargs = {
'env': env,
'conversion': Conversion(),
'formatting': Formatting(
env=env,
groups=args.prettify,
color_scheme=args.style,
explicit_json=args.json,
)
}
else:
stream_class = EncodedStream
stream_kwargs = {
'env': env
}
return stream_class, stream_kwargs
class BaseStream:
"""Base HTTP message output stream class."""
def __init__(
self,
msg: HTTPMessage,
with_headers=True, with_body=True,
with_headers=True,
with_body=True,
on_body_chunk_downloaded: Callable[[bytes], None] = None
):
"""

163
httpie/output/writer.py Normal file
View File

@ -0,0 +1,163 @@
import argparse
import errno
from typing import Union, IO, TextIO, Tuple, Type
import requests
from httpie.context import Environment
from httpie.models import HTTPRequest, HTTPResponse
from httpie.output.processing import Conversion, Formatting
from httpie.output.streams import (
RawStream, PrettyStream,
BufferedPrettyStream, EncodedStream,
BaseStream,
)
from httpie.cli.constants import (
OUT_REQ_BODY, OUT_REQ_HEAD, OUT_RESP_BODY, OUT_RESP_HEAD,
)
def write_message(
requests_message: Union[requests.PreparedRequest, requests.Response],
env: Environment,
args: argparse.Namespace,
):
output_options_by_message_type = {
requests.PreparedRequest: {
'with_headers': OUT_REQ_HEAD in args.output_options,
'with_body': OUT_REQ_BODY in args.output_options,
},
requests.Response: {
'with_headers': OUT_RESP_HEAD in args.output_options,
'with_body': OUT_RESP_BODY in args.output_options,
},
}
output_options = output_options_by_message_type[type(requests_message)]
if not any(output_options.values()):
return
write_stream_kwargs = {
'stream': build_output_stream_for_message(
args=args,
env=env,
requests_message=requests_message,
**output_options,
),
# NOTE: `env.stdout` will in fact be `stderr` with `--download`
'outfile': env.stdout,
'flush': env.stdout_isatty or args.stream
}
try:
if env.is_windows and 'colors' in args.prettify:
write_stream_with_colors_win_py3(**write_stream_kwargs)
else:
write_stream(**write_stream_kwargs)
except IOError as e:
show_traceback = args.debug or args.traceback
if not show_traceback and e.errno == errno.EPIPE:
# Ignore broken pipes unless --traceback.
env.stderr.write('\n')
else:
raise
def write_stream(
stream: BaseStream,
outfile: Union[IO, TextIO],
flush: bool
):
"""Write the output stream."""
try:
# Writing bytes so we use the buffer interface (Python 3).
buf = outfile.buffer
except AttributeError:
buf = outfile
for chunk in stream:
buf.write(chunk)
if flush:
outfile.flush()
def write_stream_with_colors_win_py3(
stream: 'BaseStream',
outfile: TextIO,
flush: bool
):
"""Like `write`, but colorized chunks are written as text
directly to `outfile` to ensure it gets processed by colorama.
Applies only to Windows with Python 3 and colorized terminal output.
"""
color = b'\x1b['
encoding = outfile.encoding
for chunk in stream:
if color in chunk:
outfile.write(chunk.decode(encoding))
else:
outfile.buffer.write(chunk)
if flush:
outfile.flush()
def build_output_stream_for_message(
args: argparse.Namespace,
env: Environment,
requests_message: Union[requests.PreparedRequest, requests.Response],
with_headers: bool,
with_body: bool,
):
stream_class, stream_kwargs = get_stream_type_and_kwargs(
env=env,
args=args,
)
message_class = {
requests.PreparedRequest: HTTPRequest,
requests.Response: HTTPResponse,
}[type(requests_message)]
yield from stream_class(
msg=message_class(requests_message),
with_headers=with_headers,
with_body=with_body,
**stream_kwargs,
)
if env.stdout_isatty and with_body:
# Ensure a blank line after the response body.
# For terminal output only.
yield b'\n\n'
def get_stream_type_and_kwargs(
env: Environment,
args: argparse.Namespace
) -> Tuple[Type['BaseStream'], dict]:
"""Pick the right stream type and kwargs for it based on `env` and `args`.
"""
if not env.stdout_isatty and not args.prettify:
stream_class = RawStream
stream_kwargs = {
'chunk_size': (
RawStream.CHUNK_SIZE_BY_LINE
if args.stream
else RawStream.CHUNK_SIZE
)
}
elif args.prettify:
stream_class = PrettyStream if args.stream else BufferedPrettyStream
stream_kwargs = {
'env': env,
'conversion': Conversion(),
'formatting': Formatting(
env=env,
groups=args.prettify,
color_scheme=args.style,
explicit_json=args.json,
)
}
else:
stream_class = EncodedStream
stream_kwargs = {
'env': env
}
return stream_class, stream_kwargs

54
httpie/sessions.py
View File

@ -1,16 +1,14 @@
"""Persistent, JSON-serialized sessions.
"""
import argparse
import re
import os
import re
from pathlib import Path
from typing import Optional, Union
from urllib.parse import urlsplit
from requests.auth import AuthBase
from requests.cookies import RequestsCookieJar, create_cookie
import requests
from httpie.cli.dicts import RequestHeadersDict
from httpie.config import BaseConfigDict, DEFAULT_CONFIG_DIR
@ -26,23 +24,16 @@ VALID_SESSION_NAME_PATTERN = re.compile('^[a-zA-Z0-9_.-]+$')
SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-']
def get_response(
requests_session: requests.Session,
session_name: str,
def get_httpie_session(
config_dir: Path,
args: argparse.Namespace,
read_only=False,
) -> requests.Response:
"""Like `client.get_responses`, but applies permanent
aspects of the session to the request.
"""
from .client import make_requests_kwargs, dump_request
session_name: str,
host: Optional[str],
url: str,
) -> 'Session':
if os.path.sep in session_name:
path = os.path.expanduser(session_name)
else:
hostname = (args.headers.get('Host', None)
or urlsplit(args.url).netloc.split('@')[-1])
hostname = host or urlsplit(url).netloc.split('@')[-1]
if not hostname:
# HACK/FIXME: httpie-unixsocket's URLs have no hostname.
hostname = 'localhost'
@ -50,38 +41,11 @@ def get_response(
# host:port => host_port
hostname = hostname.replace(':', '_')
path = (
config_dir / SESSIONS_DIR_NAME / hostname
/ (session_name + '.json')
config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json'
)
session = Session(path)
session.load()
kwargs = make_requests_kwargs(args, base_headers=session.headers)
if args.debug:
dump_request(kwargs)
session.update_headers(kwargs['headers'])
if args.auth_plugin:
session.auth = {
'type': args.auth_plugin.auth_type,
'raw_auth': args.auth_plugin.raw_auth,
}
elif session.auth:
kwargs['auth'] = session.auth
requests_session.cookies = session.cookies
try:
response = requests_session.request(**kwargs)
except Exception:
raise
else:
# Existing sessions with `read_only=True` don't get updated.
if session.is_new() or not read_only:
session.cookies = requests_session.cookies
session.save()
return response
return session
class Session(BaseConfigDict):

2
tests/test_auth.py
View File

@ -71,7 +71,7 @@ def test_missing_auth(httpbin):
'--auth-type=basic',
'GET',
httpbin + '/basic-auth/user/password',
error_exit_ok=True
tolerate_error_exit_status=True
)
assert HTTP_OK not in r
assert '--auth required' in r.stderr

4
tests/test_cli.py
View File

@ -303,7 +303,7 @@ class TestNoOptions:
def test_invalid_no_options(self, httpbin):
r = http('--no-war', 'GET', httpbin.url + '/get',
error_exit_ok=True)
tolerate_error_exit_status=True)
assert r.exit_status == ExitStatus.ERROR
assert 'unrecognized arguments: --no-war' in r.stderr
assert 'GET /get HTTP/1.1' not in r
@ -322,7 +322,7 @@ class TestStdin:
def test_ignore_stdin_cannot_prompt_password(self, httpbin):
r = http('--ignore-stdin', '--auth=no-password', httpbin.url + '/get',
error_exit_ok=True)
tolerate_error_exit_status=True)
assert r.exit_status == ExitStatus.ERROR
assert 'because --ignore-stdin' in r.stderr

27
tests/test_downloads.py
View File

@ -135,10 +135,13 @@ class TestDownloads:
def test_download_with_Content_Length(self, httpbin_both):
with open(os.devnull, 'w') as devnull:
downloader = Downloader(output_file=devnull, progress_file=devnull)
downloader.start(Response(
url=httpbin_both.url + '/',
headers={'Content-Length': 10}
))
downloader.start(
initial_url='/',
final_response=Response(
url=httpbin_both.url + '/',
headers={'Content-Length': 10}
)
)
time.sleep(1.1)
downloader.chunk_downloaded(b'12345')
time.sleep(1.1)
@ -150,7 +153,10 @@ class TestDownloads:
def test_download_no_Content_Length(self, httpbin_both):
with open(os.devnull, 'w') as devnull:
downloader = Downloader(output_file=devnull, progress_file=devnull)
downloader.start(Response(url=httpbin_both.url + '/'))
downloader.start(
final_response=Response(url=httpbin_both.url + '/'),
initial_url='/'
)
time.sleep(1.1)
downloader.chunk_downloaded(b'12345')
downloader.finish()
@ -160,10 +166,13 @@ class TestDownloads:
def test_download_interrupted(self, httpbin_both):
with open(os.devnull, 'w') as devnull:
downloader = Downloader(output_file=devnull, progress_file=devnull)
downloader.start(Response(
url=httpbin_both.url + '/',
headers={'Content-Length': 5}
))
downloader.start(
final_response=Response(
url=httpbin_both.url + '/',
headers={'Content-Length': 5}
),
initial_url='/'
)
downloader.chunk_downloaded(b'1234')
downloader.finish()
assert downloader.interrupted

10
tests/test_errors.py
View File

@ -1,17 +1,17 @@
import mock
from pytest import raises
from requests import Request, Timeout
from requests import Request
from requests.exceptions import ConnectionError
from httpie import ExitStatus
from httpie.core import main
from utils import http, HTTP_OK
from utils import HTTP_OK, http
error_msg = None
@mock.patch('httpie.core.get_response')
@mock.patch('httpie.core.program')
def test_error(get_response):
def error(msg, *args, **kwargs):
global error_msg
@ -24,11 +24,11 @@ def test_error(get_response):
assert ret == ExitStatus.ERROR
assert error_msg == (
'ConnectionError: '
'Connection aborted while doing GET request to URL: '
'Connection aborted while doing a GET request to URL: '
'http://www.google.com')
@mock.patch('httpie.core.get_response')
@mock.patch('httpie.core.program')
def test_error_traceback(get_response):
exc = ConnectionError('Connection aborted')
exc.request = Request(method='GET', url='http://www.google.com')

14
tests/test_exit_status.py
View File

@ -7,14 +7,14 @@ from utils import MockEnvironment, http, HTTP_OK
def test_keyboard_interrupt_during_arg_parsing_exit_status(httpbin):
with mock.patch('httpie.cli.definition.parser.parse_args',
side_effect=KeyboardInterrupt()):
r = http('GET', httpbin.url + '/get', error_exit_ok=True)
r = http('GET', httpbin.url + '/get', tolerate_error_exit_status=True)
assert r.exit_status == ExitStatus.ERROR_CTRL_C
def test_keyboard_interrupt_in_program_exit_status(httpbin):
with mock.patch('httpie.core.program',
side_effect=KeyboardInterrupt()):
r = http('GET', httpbin.url + '/get', error_exit_ok=True)
r = http('GET', httpbin.url + '/get', tolerate_error_exit_status=True)
assert r.exit_status == ExitStatus.ERROR_CTRL_C
@ -34,7 +34,7 @@ def test_error_response_exits_0_without_check_status(httpbin):
def test_timeout_exit_status(httpbin):
r = http('--timeout=0.01', 'GET', httpbin.url + '/delay/0.5',
error_exit_ok=True)
tolerate_error_exit_status=True)
assert r.exit_status == ExitStatus.ERROR_TIMEOUT
@ -43,7 +43,7 @@ def test_3xx_check_status_exits_3_and_stderr_when_stdout_redirected(
env = MockEnvironment(stdout_isatty=False)
r = http('--check-status', '--headers',
'GET', httpbin.url + '/status/301',
env=env, error_exit_ok=True)
env=env, tolerate_error_exit_status=True)
assert '301 MOVED PERMANENTLY' in r
assert r.exit_status == ExitStatus.ERROR_HTTP_3XX
assert '301 moved permanently' in r.stderr.lower()
@ -52,7 +52,7 @@ def test_3xx_check_status_exits_3_and_stderr_when_stdout_redirected(
def test_3xx_check_status_redirects_allowed_exits_0(httpbin):
r = http('--check-status', '--follow',
'GET', httpbin.url + '/status/301',
error_exit_ok=True)
tolerate_error_exit_status=True)
# The redirect will be followed so 200 is expected.
assert HTTP_OK in r
assert r.exit_status == ExitStatus.SUCCESS
@ -60,7 +60,7 @@ def test_3xx_check_status_redirects_allowed_exits_0(httpbin):
def test_4xx_check_status_exits_4(httpbin):
r = http('--check-status', 'GET', httpbin.url + '/status/401',
error_exit_ok=True)
tolerate_error_exit_status=True)
assert '401 UNAUTHORIZED' in r
assert r.exit_status == ExitStatus.ERROR_HTTP_4XX
# Also stderr should be empty since stdout isn't redirected.
@ -69,6 +69,6 @@ def test_4xx_check_status_exits_4(httpbin):
def test_5xx_check_status_exits_5(httpbin):
r = http('--check-status', 'GET', httpbin.url + '/status/500',
error_exit_ok=True)
tolerate_error_exit_status=True)
assert '500 INTERNAL SERVER ERROR' in r
assert r.exit_status == ExitStatus.ERROR_HTTP_5XX

4
tests/test_httpie.py
View File

@ -15,13 +15,13 @@ def test_debug():
def test_help():
r = http('--help', error_exit_ok=True)
r = http('--help', tolerate_error_exit_status=True)
assert r.exit_status == httpie.ExitStatus.SUCCESS
assert 'https://github.com/jakubroztocil/httpie/issues' in r
def test_version():
r = http('--version', error_exit_ok=True)
r = http('--version', tolerate_error_exit_status=True)
assert r.exit_status == httpie.ExitStatus.SUCCESS
# FIXME: py3 has version in stdout, py2 in stderr
assert httpie.__version__ == r.strip()

31
tests/test_redirects.py
View File

@ -28,20 +28,25 @@ def test_follow_all_output_options_used_for_redirects(httpbin):
assert r.count('GET /') == 3
assert HTTP_OK not in r
def test_follow_redirect_output_options(httpbin):
r = http('--check-status',
'--follow',
'--all',
'--print=h',
'--history-print=H',
httpbin.url + '/redirect/2')
assert r.count('GET /') == 2
assert 'HTTP/1.1 302 FOUND' not in r
assert HTTP_OK in r
#
# def test_follow_redirect_output_options(httpbin):
# r = http('--check-status',
# '--follow',
# '--all',
# '--print=h',
# '--history-print=H',
# httpbin.url + '/redirect/2')
# assert r.count('GET /') == 2
# assert 'HTTP/1.1 302 FOUND' not in r
# assert HTTP_OK in r
#
def test_max_redirects(httpbin):
r = http('--max-redirects=1', '--follow', httpbin.url + '/redirect/3',
error_exit_ok=True)
r = http(
'--max-redirects=1',
'--follow',
httpbin.url + '/redirect/3',
tolerate_error_exit_status=True,
)
assert r.exit_status == ExitStatus.ERROR_TOO_MANY_REDIRECTS

13
tests/test_sessions.py
View File

@ -45,10 +45,15 @@ class TestSessionFlow(SessionTestBase):
"""
super().start_session(httpbin)
r1 = http('--follow', '--session=test', '--auth=username:password',
'GET', httpbin.url + '/cookies/set?hello=world',
'Hello:World',
env=self.env())
r1 = http(
'--follow',
'--session=test',
'--auth=username:password',
'GET',
httpbin.url + '/cookies/set?hello=world',
'Hello:World',
env=self.env()
)
assert HTTP_OK in r1
def test_session_created_and_reused(self, httpbin):

2
tests/test_ssl.py
View File

@ -66,7 +66,7 @@ class TestClientCert:
def test_cert_file_not_found(self, httpbin_secure):
r = http(httpbin_secure + '/get',
'--cert', '/__not_found__',
error_exit_ok=True)
tolerate_error_exit_status=True)
assert r.exit_status == ExitStatus.ERROR
assert 'No such file or directory' in r.stderr

11
tests/test_uploads.py
View File

@ -64,12 +64,17 @@ class TestRequestBodyFromFilePath:
self, httpbin):
env = MockEnvironment(stdin_isatty=True)
r = http('POST', httpbin.url + '/post', 'field-name@' + FILE_PATH_ARG,
env=env, error_exit_ok=True)
env=env, tolerate_error_exit_status=True)
assert 'perhaps you meant --form?' in r.stderr
def test_request_body_from_file_by_path_no_data_items_allowed(
self, httpbin):
env = MockEnvironment(stdin_isatty=False)
r = http('POST', httpbin.url + '/post', '@' + FILE_PATH_ARG, 'foo=bar',
env=env, error_exit_ok=True)
r = http(
'POST',
httpbin.url + '/post',
'@' + FILE_PATH_ARG, 'foo=bar',
env=env,
tolerate_error_exit_status=True,
)
assert 'cannot be mixed' in r.stderr

2
tests/test_windows.py
View File

@ -27,5 +27,5 @@ class TestFakeWindows:
)
r = http('--output', output_file,
'--pretty=all', 'GET', httpbin.url + '/get',
env=env, error_exit_ok=True)
env=env, tolerate_error_exit_status=True)
assert 'Only terminal output can be colorized on Windows' in r.stderr

10
tests/utils.py
View File

@ -167,7 +167,7 @@ def http(*args, program_name='http', **kwargs):
Exceptions are propagated.
If you pass ``error_exit_ok=True``, then error exit statuses
If you pass ``tolerate_error_exit_status=True``, then error exit statuses
won't result into an exception.
Example:
@ -188,7 +188,7 @@ def http(*args, program_name='http', **kwargs):
True
"""
error_exit_ok = kwargs.pop('error_exit_ok', False)
tolerate_error_exit_status = kwargs.pop('tolerate_error_exit_status', False)
env = kwargs.get('env')
if not env:
env = kwargs['env'] = MockEnvironment()
@ -200,7 +200,7 @@ def http(*args, program_name='http', **kwargs):
args_with_config_defaults = args + env.config.default_options
add_to_args = []
if '--debug' not in args_with_config_defaults:
if not error_exit_ok and '--traceback' not in args_with_config_defaults:
if not tolerate_error_exit_status and '--traceback' not in args_with_config_defaults:
add_to_args.append('--traceback')
if not any('--timeout' in arg for arg in args_with_config_defaults):
add_to_args.append('--timeout=3')
@ -218,7 +218,7 @@ def http(*args, program_name='http', **kwargs):
# Let the progress reporter thread finish.
time.sleep(.5)
except SystemExit:
if error_exit_ok:
if tolerate_error_exit_status:
exit_status = ExitStatus.ERROR
else:
dump_stderr()
@ -228,7 +228,7 @@ def http(*args, program_name='http', **kwargs):
sys.stderr.write(stderr.read())
raise
else:
if not error_exit_ok and exit_status != ExitStatus.SUCCESS:
if not tolerate_error_exit_status and exit_status != ExitStatus.SUCCESS:
dump_stderr()
raise ExitStatusError(
'httpie.core.main() unexpectedly returned'