python/httpie-cli
1
0
Fork 0
mirror of https://github.com/httpie/cli.git synced 2025-10-30 23:47:52 +02:00
Code Activity

Remove automatic config file creation to avoid concurrency issues.

Browse Source 
Close #788
Close #812
This commit is contained in:
Jakub Roztocil
2019-12-02 17:43:16 +01:00
parent f0058eeaee
commit f202f338a4
11 changed files with 178 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.rst
View File

@@ -11,6 +11,7 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
* Removed Python 2.7 support (`EOL Jan 2020 <https://www.python.org/doc/sunset-python-2/>`_).
* Removed the default 30-second connection ``--timeout`` limit.
* Removed Python’s default limit of 100 response headers.
* Removed automatic config file creation to avoid concurrency issues.
* Replaced the old collect-all-then-process handling of HTTP communication
with one-by-one processing of each HTTP request or response as they become
available. This means that you can see headers immediately,
@@ -26,7 +27,6 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
* Added ``tests/`` to the PyPi package for the convenience of
downstream package maintainers.
* Fixed an error when ``stdin`` was a closed fd.
* Fixed an error when the config directory was not writeable.
* Improved ``--debug`` output formatting.

56
README.rst
View File

@@ -1487,7 +1487,8 @@ To create or reuse a different session, simple specify a different name:
$ http --session=user2 -a user2:password example.org X-Bar:Foo
Named sessions' data is stored in JSON files in the directory
Named sessions’s data is stored in JSON files in the the ``sessions``
subdirectory of the `config`_ directory:
``~/.httpie/sessions/<host>/<name>.json``
(``%APPDATA%\httpie\sessions\<host>\<name>.json`` on Windows).
@@ -1517,46 +1518,61 @@ exchange once it is created, specify the session name via
Config
======
HTTPie uses a simple JSON config file.
HTTPie uses a simple ``config.json`` file. The file doesn’t exist by default
but you can create it manually.
Config file location
--------------------
Config file directory
---------------------
The default location of the configuration file is ``~/.httpie/config.json``
(or ``%APPDATA%\httpie\config.json`` on Windows). The config directory
location can be changed by setting the ``HTTPIE_CONFIG_DIR``
environment variable. To view the exact location run ``http --debug``.
(or ``%APPDATA%\httpie\config.json`` on Windows).
The config directory can be changed by setting the ``$HTTPIE_CONFIG_DIR``
environment variable:
.. code-block:: bash
$ export HTTPIE_CONFIG_DIR=/tmp/httpie
$ http example.org
To view the exact location run ``http --debug``.
Configurable options
--------------------
The JSON file contains an object with the following keys:
Currently HTTPie offers a single configurable option:
``default_options``
~~~~~~~~~~~~~~~~~~~
An ``Array`` (by default empty) of default options that should be applied to
every invocation of HTTPie.
For instance, you can use this option to change the default style and output
options: ``"default_options": ["--style=fruity", "--body"]`` Another useful
default option could be ``"--session=default"`` to make HTTPie always
use `sessions`_ (one named ``default`` will automatically be used).
Or you could change the implicit request content type from JSON to form by
adding ``--form`` to the list.
For instance, you can use this config option to change your default color theme:
``__meta__``
~~~~~~~~~~~~
.. code-block:: bash
HTTPie automatically stores some of its metadata here. Please do not change.
$ cat ~/.httpie/config.json
.. code-block:: json
{
"default_options": [
"--style=fruity"
]
}
Even though it is technically possible to include there any of HTTPie’s
options, it is not recommended to modify the default behaviour in a way
that would break your compatibility with the wider world as that can
generate a lot of confusion.
Un-setting previously specified options
---------------------------------------

3
httpie/cli/argparser.py
View File

@@ -61,7 +61,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
def parse_args(
self,
env: Environment,
program_name='http',
args=None,
namespace=None
) -> argparse.Namespace:
@@ -89,7 +88,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
if self.has_stdin_data:
self._body_from_file(self.env.stdin)
if not URL_SCHEME_RE.match(self.args.url):
if os.path.basename(program_name) == 'https':
if os.path.basename(env.program_name) == 'https':
scheme = 'https://'
else:
scheme = self.args.default_scheme + "://"

2
httpie/client.py
View File

@@ -35,7 +35,7 @@ DEFAULT_UA = f'HTTPie/{__version__}'
def collect_messages(
args: argparse.Namespace,
config_dir: Path
config_dir: Path,
) -> Iterable[Union[requests.PreparedRequest, requests.Response]]:
httpie_session = None
httpie_session_headers = None

54
httpie/config.py
View File

@@ -15,42 +15,43 @@ DEFAULT_CONFIG_DIR = Path(os.environ.get(
))
class ConfigFileError(Exception):
pass
class BaseConfigDict(dict):
name = None
helpurl = None
about = None
def _get_path(self) -> Path:
"""Return the config file path without side-effects."""
raise NotImplementedError()
def __init__(self, path: Path):
super().__init__()
self.path = path
def path(self) -> Path:
"""Return the config file path creating basedir, if needed."""
path = self._get_path()
def ensure_directory(self):
try:
path.parent.mkdir(mode=0o700, parents=True)
self.path.parent.mkdir(mode=0o700, parents=True)
except OSError as e:
if e.errno != errno.EEXIST:
raise
return path
def is_new(self) -> bool:
return not self._get_path().exists()
return not self.path.exists()
def load(self):
config_type = type(self).__name__.lower()
try:
with self.path().open('rt') as f:
with self.path.open('rt') as f:
try:
data = json.load(f)
except ValueError as e:
raise ValueError(
'Invalid %s JSON: %s [%s]' %
(type(self).__name__, str(e), self.path())
raise ConfigFileError(
f'invalid {config_type} file: {e} [{self.path}]'
)
self.update(data)
except IOError as e:
if e.errno != errno.ENOENT:
raise
raise ConfigFileError(f'cannot read {config_type} file: {e}')
def save(self, fail_silently=False):
self['__meta__'] = {
@@ -62,9 +63,17 @@ class BaseConfigDict(dict):
if self.about:
self['__meta__']['about'] = self.about
self.ensure_directory()
try:
with self.path().open('w') as f:
json.dump(self, f, indent=4, sort_keys=True, ensure_ascii=True)
with self.path.open('w') as f:
json.dump(
obj=self,
fp=f,
indent=4,
sort_keys=True,
ensure_ascii=True,
)
f.write('\n')
except IOError:
if not fail_silently:
@@ -72,27 +81,22 @@ class BaseConfigDict(dict):
def delete(self):
try:
self.path().unlink()
self.path.unlink()
except OSError as e:
if e.errno != errno.ENOENT:
raise
class Config(BaseConfigDict):
name = 'config'
helpurl = 'https://httpie.org/doc#config'
about = 'HTTPie configuration file'
FILENAME = 'config.json'
DEFAULTS = {
'default_options': []
}
def __init__(self, directory: Union[str, Path] = DEFAULT_CONFIG_DIR):
super().__init__()
self.update(self.DEFAULTS)
self.directory = Path(directory)
def _get_path(self) -> Path:
return self.directory / (self.name + '.json')
super().__init__(path=self.directory / self.FILENAME)
self.update(self.DEFAULTS)
@property
def default_options(self) -> list:

32
httpie/context.py
View File

@@ -1,3 +1,4 @@
import os
import sys
from pathlib import Path
from typing import Union, IO, Optional
@@ -9,7 +10,7 @@ except ImportError:
curses = None # Compiled w/o curses
from httpie.compat import is_windows
from httpie.config import DEFAULT_CONFIG_DIR, Config
from httpie.config import DEFAULT_CONFIG_DIR, Config, ConfigFileError
from httpie.utils import repr_dict
@@ -35,6 +36,7 @@ class Environment:
stderr: IO = sys.stderr
stderr_isatty: bool = stderr.isatty()
colors = 256
program_name: str = 'http'
if not is_windows:
if curses:
try:
@@ -79,16 +81,6 @@ class Environment:
self.stdout_encoding = getattr(
actual_stdout, 'encoding', None) or 'utf8'
@property
def config(self) -> Config:
if not hasattr(self, '_config'):
self._config = Config(directory=self.config_dir)
if self._config.is_new():
self._config.save(fail_silently=True)
else:
self._config.load()
return self._config
def __str__(self):
defaults = dict(type(self).__dict__)
actual = dict(defaults)
@@ -102,3 +94,21 @@ class Environment:
def __repr__(self):
return f'<{type(self).__name__} {self}>'
_config: Config = None
@property
def config(self) -> Config:
config = self._config
if not config:
self._config = config = Config(directory=self.config_dir)
if not config.is_new():
try:
config.load()
except ConfigFileError as e:
self.log_error(e, level='warning')
return config
def log_error(self, msg, level='error'):
assert level in ['error', 'warning']
self.stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n')

49
httpie/core.py
View File

@@ -1,25 +1,25 @@
import argparse
import os
import platform
import sys
from typing import Callable, List, Union
from typing import List, Union
import requests
from pygments import __version__ as pygments_version
from requests import __version__ as requests_version
from httpie import __version__ as httpie_version
from httpie.status import ExitStatus, http_status_to_exit_status
from httpie.client import collect_messages
from httpie.context import Environment
from httpie.downloads import Downloader
from httpie.output.writer import write_message, write_stream
from httpie.plugins import plugin_manager
from httpie.status import ExitStatus, http_status_to_exit_status
def main(
args: List[Union[str, bytes]] = sys.argv,
env=Environment(),
custom_log_error: Callable = None
) -> ExitStatus:
"""
The main function.
@@ -30,22 +30,16 @@ def main(
Return exit status code.
"""
args = decode_raw_args(args, env.stdin_encoding)
program_name, *args = args
env.program_name = os.path.basename(program_name)
args = decode_raw_args(args, env.stdin_encoding)
plugin_manager.load_installed_plugins()
def log_error(msg, level='error'):
assert level in ['error', 'warning']
env.stderr.write(f'\n{program_name}: {level}: {msg}\n')
from httpie.cli.definition import parser
if env.config.default_options:
args = env.config.default_options + args
if custom_log_error:
log_error = custom_log_error
include_debug_info = '--debug' in args
include_traceback = include_debug_info or '--traceback' in args
@@ -59,7 +53,6 @@ def main(
try:
parsed_args = parser.parse_args(
args=args,
program_name=program_name,
env=env,
)
except KeyboardInterrupt:
@@ -78,7 +71,6 @@ def main(
exit_status = program(
args=parsed_args,
env=env,
log_error=log_error,
)
except KeyboardInterrupt:
env.stderr.write('\n')
@@ -93,10 +85,10 @@ def main(
exit_status = ExitStatus.ERROR
except requests.Timeout:
exit_status = ExitStatus.ERROR_TIMEOUT
log_error(f'Request timed out ({parsed_args.timeout}s).')
env.log_error(f'Request timed out ({parsed_args.timeout}s).')
except requests.TooManyRedirects:
exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
log_error(
env.log_error(
f'Too many redirects'
f' (--max-redirects=parsed_args.max_redirects).'
)
@@ -110,7 +102,7 @@ def main(
f'{msg} while doing a {request.method}'
f' request to URL: {request.url}'
)
log_error(f'{type(e).__name__}: {msg}')
env.log_error(f'{type(e).__name__}: {msg}')
if include_traceback:
raise
exit_status = ExitStatus.ERROR
@@ -121,7 +113,6 @@ def main(
def program(
args: argparse.Namespace,
env: Environment,
log_error: Callable
) -> ExitStatus:
"""
The main program without error handling.
@@ -159,8 +150,9 @@ def program(
http_status=message.status_code,
follow=args.follow
)
if not env.stdout_isatty and exit_status != ExitStatus.SUCCESS:
log_error(
if (not env.stdout_isatty
and exit_status != ExitStatus.SUCCESS):
env.log_error(
f'HTTP {message.raw.status} {message.raw.reason}',
level='warning'
)
@@ -179,10 +171,11 @@ def program(
downloader.finish()
if downloader.interrupted:
exit_status = ExitStatus.ERROR
log_error('Incomplete download: size=%d; downloaded=%d' % (
downloader.status.total_size,
downloader.status.downloaded
))
env.log_error(
'Incomplete download: size=%d; downloaded=%d' % (
downloader.status.total_size,
downloader.status.downloaded
))
return exit_status
finally:
@@ -196,11 +189,11 @@ def program(
def print_debug_info(env: Environment):
env.stderr.writelines([
'HTTPie %s\n' % httpie_version,
'Requests %s\n' % requests_version,
'Pygments %s\n' % pygments_version,
'Python %s\n%s\n' % (sys.version, sys.executable),
'%s %s' % (platform.system(), platform.release()),
f'HTTPie {httpie_version}\n',
f'Requests {requests_version}\n',
f'Pygments {pygments_version}\n',
f'Python {sys.version}\n{sys.executable}\n',
f'{platform.system()} {platform.release()}',
])
env.stderr.write('\n\n')
env.stderr.write(repr(env))

9
httpie/sessions.py
View File

@@ -1,4 +1,5 @@
"""Persistent, JSON-serialized sessions.
"""
Persistent, JSON-serialized sessions.
"""
import os
@@ -53,8 +54,7 @@ class Session(BaseConfigDict):
about = 'HTTPie session file'
def __init__(self, path: Union[str, Path]):
super().__init__()
self._path = Path(path)
super().__init__(path=Path(path))
self['headers'] = {}
self['cookies'] = {}
self['auth'] = {
@@ -63,9 +63,6 @@ class Session(BaseConfigDict):
'password': None
}
def _get_path(self) -> Path:
return self._path
def update_headers(self, request_headers: RequestHeadersDict):
"""
Update the session headers with the request ones while ignoring

44
tests/test_config.py
View File

@@ -1,6 +1,5 @@
from httpie import __version__
from utils import MockEnvironment, http, HTTP_OK
from httpie.context import Environment
from httpie.config import Config
from utils import HTTP_OK, MockEnvironment, http
def test_default_options(httpbin):
@@ -8,15 +7,33 @@ def test_default_options(httpbin):
env.config['default_options'] = ['--form']
env.config.save()
r = http(httpbin.url + '/post', 'foo=bar', env=env)
assert r.json['form'] == {"foo": "bar"}
assert r.json['form'] == {
"foo": "bar"
}
def test_config_dir_not_writeable(httpbin):
r = http(httpbin + '/get', env=MockEnvironment(
config_dir='/',
create_temp_config_dir=False,
))
def test_config_file_not_valid(httpbin):
env = MockEnvironment()
env.create_temp_config_dir()
with (env.config_dir / Config.FILENAME).open('w') as f:
f.write('{invalid json}')
r = http(httpbin + '/get', env=env)
assert HTTP_OK in r
assert 'http: warning' in r.stderr
assert 'invalid config file' in r.stderr
def test_config_file_not_inaccessible(httpbin):
env = MockEnvironment()
env.create_temp_config_dir()
config_path = env.config_dir / Config.FILENAME
assert not config_path.exists()
config_path.touch(0o000)
assert config_path.exists()
r = http(httpbin + '/get', env=env)
assert HTTP_OK in r
assert 'http: warning' in r.stderr
assert 'cannot read config file' in r.stderr
def test_default_options_overwrite(httpbin):
@@ -24,9 +41,6 @@ def test_default_options_overwrite(httpbin):
env.config['default_options'] = ['--form']
env.config.save()
r = http('--json', httpbin.url + '/post', 'foo=bar', env=env)
assert r.json['json'] == {"foo": "bar"}
def test_current_version():
version = MockEnvironment().config['__meta__']['httpie']
assert version == __version__
assert r.json['json'] == {
"foo": "bar"
}

37
tests/test_errors.py
View File

@@ -4,37 +4,30 @@ from requests import Request
from requests.exceptions import ConnectionError
from httpie.status import ExitStatus
from httpie.core import main
from utils import HTTP_OK, http
error_msg = None
@mock.patch('httpie.core.program')
def test_error(program):
exc = ConnectionError('Connection aborted')
exc.request = Request(method='GET', url='http://www.google.com')
program.side_effect = exc
r = http('www.google.com', tolerate_error_exit_status=True)
assert r.exit_status == ExitStatus.ERROR
assert (
'ConnectionError: '
'Connection aborted while doing a GET request to URL: '
'http://www.google.com'
) in r.stderr
@mock.patch('httpie.core.program')
def test_error(get_response):
def error(msg, *args, **kwargs):
global error_msg
error_msg = msg % args
def test_error_traceback(program):
exc = ConnectionError('Connection aborted')
exc.request = Request(method='GET', url='http://www.google.com')
get_response.side_effect = exc
ret = main(['http', '--ignore-stdin', 'www.google.com'], custom_log_error=error)
assert ret == ExitStatus.ERROR
assert error_msg == (
'ConnectionError: '
'Connection aborted while doing a GET request to URL: '
'http://www.google.com')
@mock.patch('httpie.core.program')
def test_error_traceback(get_response):
exc = ConnectionError('Connection aborted')
exc.request = Request(method='GET', url='http://www.google.com')
get_response.side_effect = exc
program.side_effect = exc
with raises(ConnectionError):
main(['http', '--ignore-stdin', '--traceback', 'www.google.com'])
http('--traceback', 'www.google.com')
def test_max_headers_limit(httpbin_both):

31
tests/utils.py
View File

@@ -6,7 +6,7 @@ import time
import json
import tempfile
from pathlib import Path
from typing import Optional
from typing import Optional, Union
from httpie.status import ExitStatus
from httpie.config import Config
@@ -18,6 +18,7 @@ TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
CRLF = '\r\n'
COLOR = '\x1b['
HTTP_OK = '200 OK'
# noinspection GrazieInspection
HTTP_OK_COLOR = (
'HTTP\x1b[39m\x1b[38;5;245m/\x1b[39m\x1b'
'[38;5;37m1.1\x1b[39m\x1b[38;5;245m \x1b[39m\x1b[38;5;37m200'
@@ -62,10 +63,13 @@ class MockEnvironment(Environment):
def config(self) -> Config:
if (self._create_temp_config_dir
and self._temp_dir not in self.config_dir.parents):
self.config_dir = mk_config_dir()
self._delete_config_dir = True
self.create_temp_config_dir()
return super().config
def create_temp_config_dir(self):
self.config_dir = mk_config_dir()
self._delete_config_dir = True
def cleanup(self):
self.stdout.close()
self.stderr.close()
@@ -75,6 +79,7 @@ class MockEnvironment(Environment):
rmtree(self.config_dir)
def __del__(self):
# noinspection PyBroadException
try:
self.cleanup()
except Exception:
@@ -83,7 +88,7 @@ class MockEnvironment(Environment):
class BaseCLIResponse:
"""
Represents the result of simulated `$ http' invocation via `http()`.
Represents the result of simulated `$ http' invocation via `http()`.
Holds and provides access to:
@@ -113,8 +118,8 @@ class StrCLIResponse(str, BaseCLIResponse):
@property
def json(self) -> Optional[dict]:
"""
Return deserialized JSON body, if one included in the output
and is parsable.
Return deserialized the request or response JSON body,
if one (and only one) included in the output and is parsable.
"""
if not hasattr(self, '_json'):
@@ -147,7 +152,12 @@ class ExitStatusError(Exception):
pass
def http(*args, program_name='http', **kwargs):
def http(
*args,
program_name='http',
tolerate_error_exit_status=False,
**kwargs,
) -> Union[StrCLIResponse, BytesCLIResponse]:
# noinspection PyUnresolvedReferences
"""
Run HTTPie and capture stderr/out and exit status.
@@ -188,7 +198,6 @@ def http(*args, program_name='http', **kwargs):
True
"""
tolerate_error_exit_status = kwargs.pop('tolerate_error_exit_status', False)
env = kwargs.get('env')
if not env:
env = kwargs['env'] = MockEnvironment()
@@ -200,7 +209,8 @@ def http(*args, program_name='http', **kwargs):
args_with_config_defaults = args + env.config.default_options
add_to_args = []
if '--debug' not in args_with_config_defaults:
if not tolerate_error_exit_status and '--traceback' not in args_with_config_defaults:
if (not tolerate_error_exit_status
and '--traceback' not in args_with_config_defaults):
add_to_args.append('--traceback')
if not any('--timeout' in arg for arg in args_with_config_defaults):
add_to_args.append('--timeout=3')
@@ -228,7 +238,8 @@ def http(*args, program_name='http', **kwargs):
sys.stderr.write(stderr.read())
raise
else:
if not tolerate_error_exit_status and exit_status != ExitStatus.SUCCESS:
if (not tolerate_error_exit_status
and exit_status != ExitStatus.SUCCESS):
dump_stderr()
raise ExitStatusError(
'httpie.core.main() unexpectedly returned'