python/httpie-cli
1
0
Fork 0
mirror of https://github.com/httpie/cli.git synced 2025-08-10 22:42:05 +02:00
Code Activity

Implement new style cookies

Browse Source
This commit is contained in:
Batuhan Taskaya
2022-02-01 12:14:24 +03:00
parent b5623ccc87
commit 65ab7d5caa
27 changed files with 1406 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
httpie/client.py
View File

@@ -44,6 +44,7 @@ def collect_messages(
httpie_session_headers = None
if args.session or args.session_read_only:
httpie_session = get_httpie_session(
env=env,
config_dir=env.config.directory,
session_name=args.session or args.session_read_only,
host=args.headers.get('Host'),
@@ -130,10 +131,7 @@ def collect_messages(
if httpie_session:
if httpie_session.is_new() or not args.session_read_only:
httpie_session.cookies = requests_session.cookies
httpie_session.remove_cookies(
# TODO: take path & domain into account?
cookie['name'] for cookie in expired_cookies
)
httpie_session.remove_cookies(expired_cookies)
httpie_session.save()

58
httpie/config.py
View File

@@ -1,7 +1,7 @@
import json
import os
from pathlib import Path
from typing import Union
from typing import Any, Dict, Union
from . import __version__
from .compat import is_windows
@@ -62,6 +62,21 @@ class ConfigFileError(Exception):
pass
def read_raw_config(config_type: str, path: Path) -> Dict[str, Any]:
try:
with path.open(encoding=UTF8) as f:
try:
return json.load(f)
except ValueError as e:
raise ConfigFileError(
f'invalid {config_type} file: {e} [{path}]'
)
except FileNotFoundError:
pass
except OSError as e:
raise ConfigFileError(f'cannot read {config_type} file: {e}')
class BaseConfigDict(dict):
name = None
helpurl = None
@@ -77,26 +92,25 @@ class BaseConfigDict(dict):
def is_new(self) -> bool:
return not self.path.exists()
def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
"""Hook for processing the incoming config data."""
return data
def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
"""Hook for processing the outgoing config data."""
return data
def load(self):
config_type = type(self).__name__.lower()
try:
with self.path.open(encoding=UTF8) as f:
try:
data = json.load(f)
except ValueError as e:
raise ConfigFileError(
f'invalid {config_type} file: {e} [{self.path}]'
)
self.update(data)
except FileNotFoundError:
pass
except OSError as e:
raise ConfigFileError(f'cannot read {config_type} file: {e}')
data = read_raw_config(config_type, self.path)
if data is not None:
data = self.pre_process_data(data)
self.update(data)
def save(self):
self['__meta__'] = {
'httpie': __version__
}
def save(self, *, bump_version: bool = False):
self.setdefault('__meta__', {})
if bump_version or 'httpie' not in self['__meta__']:
self['__meta__']['httpie'] = __version__
if self.helpurl:
self['__meta__']['help'] = self.helpurl
@@ -106,13 +120,19 @@ class BaseConfigDict(dict):
self.ensure_directory()
json_string = json.dumps(
obj=self,
obj=self.post_process_data(self),
indent=4,
sort_keys=True,
ensure_ascii=True,
)
self.path.write_text(json_string + '\n', encoding=UTF8)
@property
def version(self):
return self.get(
'__meta__', {}
).get('httpie', __version__)
class Config(BaseConfigDict):
FILENAME = 'config.json'

43
httpie/manager/cli.py
View File

@@ -2,6 +2,15 @@ from textwrap import dedent
from httpie.cli.argparser import HTTPieManagerArgumentParser
from httpie import __version__
CLI_SESSION_UPGRADE_FLAGS = [
{
'variadic': ['--bind-cookies'],
'action': 'store_true',
'default': False,
'help': 'Bind domainless cookies to the host that session belongs.'
}
]
COMMANDS = {
'plugins': {
'help': 'Manage HTTPie plugins.',
@@ -34,6 +43,34 @@ COMMANDS = {
'List all installed HTTPie plugins.'
],
},
'cli': {
'help': 'Manage HTTPie for Terminal',
'sessions': {
'help': 'Manage HTTPie sessions',
'upgrade': [
'Upgrade the given HTTPie session with the latest '
'layout. A list of changes between different session versions '
'can be found in the official documentation.',
{
'dest': 'hostname',
'metavar': 'HOSTNAME',
'help': 'The host this session belongs.'
},
{
'dest': 'session',
'metavar': 'SESSION_NAME_OR_PATH',
'help': 'The name or the path for the session that will be upgraded.'
},
*CLI_SESSION_UPGRADE_FLAGS
],
'upgrade-all': [
'Upgrade all named sessions with the latest layout. A list of '
'changes between different session versions can be found in the official '
'documentation.',
*CLI_SESSION_UPGRADE_FLAGS
],
}
}
}
@@ -54,6 +91,8 @@ def generate_subparsers(root, parent_parser, definitions):
)
for command, properties in definitions.items():
is_subparser = isinstance(properties, dict)
properties = properties.copy()
descr = properties.pop('help', None) if is_subparser else properties.pop(0)
command_parser = actions.add_parser(command, description=descr)
command_parser.root = root
@@ -62,7 +101,9 @@ def generate_subparsers(root, parent_parser, definitions):
continue
for argument in properties:
command_parser.add_argument(**argument)
argument = argument.copy()
variadic = argument.pop('variadic', [])
command_parser.add_argument(*variadic, **argument)
parser = HTTPieManagerArgumentParser(

11
httpie/manager/core.py
View File

@@ -1,9 +1,11 @@
import argparse
from typing import Optional
from httpie.context import Environment
from httpie.manager.plugins import PluginInstaller
from httpie.status import ExitStatus
from httpie.manager.cli import missing_subcommand, parser
from httpie.manager.tasks import CLI_TASKS
MSG_COMMAND_CONFUSION = '''\
This command is only for managing HTTPie plugins.
@@ -22,6 +24,13 @@ MSG_NAKED_INVOCATION = f'''\
'''.rstrip("\n").format(args='POST pie.dev/post hello=world')
def dispatch_cli_task(env: Environment, action: Optional[str], args: argparse.Namespace) -> ExitStatus:
if action is None:
parser.error(missing_subcommand('cli'))
return CLI_TASKS[action](env, args)
def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
if args.action is None:
parser.error(MSG_NAKED_INVOCATION)
@@ -29,5 +38,7 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
if args.action == 'plugins':
plugins = PluginInstaller(env, debug=args.debug)
return plugins.run(args.plugins_action, args)
elif args.action == 'cli':
return dispatch_cli_task(env, args.cli_action, args)
return ExitStatus.SUCCESS

134
httpie/manager/tasks.py Normal file
View File

@@ -0,0 +1,134 @@
import argparse
from typing import TypeVar, Callable, Tuple
from httpie.sessions import SESSIONS_DIR_NAME, Session, get_httpie_session
from httpie.status import ExitStatus
from httpie.context import Environment
from httpie.manager.cli import missing_subcommand, parser
T = TypeVar('T')
CLI_TASKS = {}
def task(name: str) -> Callable[[T], T]:
def wrapper(func: T) -> T:
CLI_TASKS[name] = func
return func
return wrapper
@task('sessions')
def cli_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus:
action = args.cli_sessions_action
if action is None:
parser.error(missing_subcommand('cli', 'sessions'))
if action == 'upgrade':
return cli_upgrade_session(env, args)
elif action == 'upgrade-all':
return cli_upgrade_all_sessions(env, args)
else:
raise ValueError(f'Unexpected action: {action}')
def is_version_greater(version_1: str, version_2: str) -> bool:
# In an ideal scenerio, we would depend on `packaging` in order
# to offer PEP 440 compatible parsing. But since it might not be
# commonly available for outside packages, and since we are only
# going to parse HTTPie's own version it should be fine to compare
# this in a SemVer subset fashion.
def split_version(version: str) -> Tuple[int, ...]:
parts = []
for part in version.split('.')[:3]:
try:
parts.append(int(part))
except ValueError:
break
return tuple(parts)
return split_version(version_1) > split_version(version_2)
def fix_cookie_layout(session: Session, hostname: str, args: argparse.Namespace) -> None:
if not isinstance(session['cookies'], dict):
return None
session['cookies'] = [
{
'name': key,
**value
}
for key, value in session['cookies'].items()
]
for cookie in session.cookies:
if cookie.domain == '':
if args.bind_cookies:
cookie.domain = hostname
else:
cookie._rest['is_explicit_none'] = True
FIXERS_TO_VERSIONS = {
'3.1.0': fix_cookie_layout
}
def upgrade_session(env: Environment, args: argparse.Namespace, hostname: str, session_name: str):
session = get_httpie_session(
env=env,
config_dir=env.config.directory,
session_name=session_name,
host=hostname,
url=hostname,
refactor_mode=True
)
session_name = session.path.stem
if session.is_new():
env.log_error(f'{session_name!r} (for {hostname!r}) does not exist.')
return ExitStatus.ERROR
fixers = [
fixer
for version, fixer in FIXERS_TO_VERSIONS.items()
if is_version_greater(version, session.version)
]
if len(fixers) == 0:
env.stdout.write(f'{session_name!r} (for {hostname!r}) is already up-to-date.\n')
return ExitStatus.SUCCESS
for fixer in fixers:
fixer(session, hostname, args)
session.save(bump_version=True)
env.stdout.write(f'Refactored {session_name!r} (for {hostname!r}) to the version {session.version}.\n')
return ExitStatus.SUCCESS
def cli_upgrade_session(env: Environment, args: argparse.Namespace) -> ExitStatus:
return upgrade_session(
env,
args=args,
hostname=args.hostname,
session_name=args.session
)
def cli_upgrade_all_sessions(env: Environment, args: argparse.Namespace) -> ExitStatus:
session_dir_path = env.config_dir / SESSIONS_DIR_NAME
status = ExitStatus.SUCCESS
for host_path in session_dir_path.iterdir():
hostname = host_path.name
for session_path in host_path.glob("*.json"):
session_name = session_path.stem
status |= upgrade_session(
env,
args=args,
hostname=hostname,
session_name=session_name
)
return status

208
httpie/sessions.py
View File

@@ -6,15 +6,17 @@ import os
import re
from http.cookies import SimpleCookie
from http.cookiejar import Cookie
from pathlib import Path
from typing import Iterable, Optional, Union
from urllib.parse import urlsplit
from typing import Any, Dict, Optional, Union
from requests.auth import AuthBase
from requests.cookies import RequestsCookieJar, create_cookie
from requests.cookies import RequestsCookieJar, remove_cookie_by_name
from .context import Environment
from .cli.dicts import HTTPHeadersDict
from .config import BaseConfigDict, DEFAULT_CONFIG_DIR
from .utils import url_as_host
from .plugins.registry import plugin_manager
@@ -26,27 +28,88 @@ VALID_SESSION_NAME_PATTERN = re.compile('^[a-zA-Z0-9_.-]+$')
# <https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Requests>
SESSION_IGNORED_HEADER_PREFIXES = ['Content-', 'If-']
# Cookie related options
KEPT_COOKIE_OPTIONS = ['name', 'expires', 'path', 'value', 'domain', 'secure']
DEFAULT_COOKIE_PATH = '/'
INSECURE_COOKIE_JAR_WARNING = '''\
Outdated layout detected for the current session. Please consider updating it,
in order to not get affected by potential security problems.
For fixing the current session:
With binding all cookies to the current host (secure):
$ httpie cli sessions upgrade --bind-cookies {hostname} {session_id}
Without binding cookies (leaving them as is) (insecure):
$ httpie cli sessions upgrade {hostname} {session_id}
'''
INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS = '''\
For fixing all named sessions:
With binding all cookies to the current host (secure):
$ httpie cli sessions upgrade-all --bind-cookies
Without binding cookies (leaving them as is) (insecure):
$ httpie cli sessions upgrade-all
See https://pie.co/docs/security for more information.
'''
def is_anonymous_session(session_name: str) -> bool:
return os.path.sep in session_name
def materialize_cookie(cookie: Cookie) -> Dict[str, Any]:
materialized_cookie = {
option: getattr(cookie, option)
for option in KEPT_COOKIE_OPTIONS
}
if (
cookie._rest.get('is_explicit_none')
and materialized_cookie['domain'] == ''
):
materialized_cookie['domain'] = None
return materialized_cookie
def get_httpie_session(
env: Environment,
config_dir: Path,
session_name: str,
host: Optional[str],
url: str,
*,
refactor_mode: bool = False
) -> 'Session':
if os.path.sep in session_name:
path = os.path.expanduser(session_name)
else:
hostname = host or urlsplit(url).netloc.split('@')[-1]
if not hostname:
# HACK/FIXME: httpie-unixsocket's URLs have no hostname.
hostname = 'localhost'
bound_hostname = host or url_as_host(url)
if not bound_hostname:
# HACK/FIXME: httpie-unixsocket's URLs have no hostname.
bound_hostname = 'localhost'
# host:port => host_port
hostname = hostname.replace(':', '_')
# host:port => host_port
hostname = bound_hostname.replace(':', '_')
if is_anonymous_session(session_name):
path = os.path.expanduser(session_name)
session_id = path
else:
path = (
config_dir / SESSIONS_DIR_NAME / hostname / f'{session_name}.json'
)
session = Session(path)
session_id = session_name
session = Session(
path,
env=env,
session_id=session_id,
bound_host=bound_hostname.split(':')[0],
refactor_mode=refactor_mode
)
session.load()
return session
@@ -55,15 +118,86 @@ class Session(BaseConfigDict):
helpurl = 'https://httpie.io/docs#sessions'
about = 'HTTPie session file'
def __init__(self, path: Union[str, Path]):
def __init__(
self,
path: Union[str, Path],
env: Environment,
bound_host: str,
session_id: str,
refactor_mode: bool = False,
):
super().__init__(path=Path(path))
self['headers'] = {}
self['cookies'] = {}
self['cookies'] = []
self['auth'] = {
'type': None,
'username': None,
'password': None
}
self.env = env
self.cookie_jar = RequestsCookieJar()
self.session_id = session_id
self.bound_host = bound_host
self.refactor_mode = refactor_mode
def pre_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
cookies = data.get('cookies')
if isinstance(cookies, dict):
normalized_cookies = [
{
'name': key,
**value
}
for key, value in cookies.items()
]
elif isinstance(cookies, list):
normalized_cookies = cookies
else:
normalized_cookies = []
should_issue_warning = False
for cookie in normalized_cookies:
domain = cookie.get('domain', '')
if domain == '' and isinstance(cookies, dict):
should_issue_warning = True
elif domain is None:
# domain = None means explicitly lack of cookie, though
# requests requires domain to be string so we'll cast it
# manually.
cookie['domain'] = ''
cookie['rest'] = {'is_explicit_none': True}
self.cookie_jar.set(**cookie)
if should_issue_warning and not self.refactor_mode:
warning = INSECURE_COOKIE_JAR_WARNING.format(hostname=self.bound_host, session_id=self.session_id)
if not is_anonymous_session(self.session_id):
warning += INSECURE_COOKIE_JAR_WARNING_FOR_NAMED_SESSIONS
self.env.log_error(
warning,
level='warning'
)
return data
def post_process_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
cookies = data.get('cookies')
# Save in the old-style fashion
normalized_cookies = [
materialize_cookie(cookie)
for cookie in self.cookie_jar
]
if isinstance(cookies, dict):
data['cookies'] = {
cookie.pop('name'): cookie
for cookie in normalized_cookies
}
else:
data['cookies'] = normalized_cookies
return data
def update_headers(self, request_headers: HTTPHeadersDict):
"""
@@ -73,10 +207,10 @@ class Session(BaseConfigDict):
"""
headers = self.headers
for name, value in request_headers.copy().items():
if value is None:
continue # Ignore explicitly unset headers
original_value = value
if type(value) is not str:
value = value.decode()
@@ -85,8 +219,15 @@ class Session(BaseConfigDict):
if name.lower() == 'cookie':
for cookie_name, morsel in SimpleCookie(value).items():
self['cookies'][cookie_name] = {'value': morsel.value}
del request_headers[name]
if not morsel['path']:
morsel['path'] = DEFAULT_COOKIE_PATH
self.cookie_jar.set(cookie_name, morsel)
all_cookie_headers = request_headers.getall(name)
if len(all_cookie_headers) > 1:
all_cookie_headers.remove(original_value)
else:
request_headers.popall(name)
continue
for prefix in SESSION_IGNORED_HEADER_PREFIXES:
@@ -103,23 +244,21 @@ class Session(BaseConfigDict):
@property
def cookies(self) -> RequestsCookieJar:
jar = RequestsCookieJar()
for name, cookie_dict in self['cookies'].items():
jar.set_cookie(create_cookie(
name, cookie_dict.pop('value'), **cookie_dict))
jar.clear_expired_cookies()
return jar
self.cookie_jar.clear_expired_cookies()
return self.cookie_jar
@cookies.setter
def cookies(self, jar: RequestsCookieJar):
# <https://docs.python.org/3/library/cookielib.html#cookie-objects>
stored_attrs = ['value', 'path', 'secure', 'expires']
self['cookies'] = {}
for cookie in jar:
self['cookies'][cookie.name] = {
attname: getattr(cookie, attname)
for attname in stored_attrs
}
self.cookie_jar = jar
def remove_cookies(self, cookies: Dict[str, str]):
for cookie in cookies:
remove_cookie_by_name(
self.cookie_jar,
cookie['name'],
domain=cookie.get('domain', None),
path=cookie.get('path', None)
)
@property
def auth(self) -> Optional[AuthBase]:
@@ -154,8 +293,3 @@ class Session(BaseConfigDict):
def auth(self, auth: dict):
assert {'type', 'raw_auth'} == auth.keys()
self['auth'] = auth
def remove_cookies(self, names: Iterable[str]):
for name in names:
if name in self['cookies']:
del self['cookies'][name]

5
httpie/utils.py
View File

@@ -9,6 +9,7 @@ from collections import OrderedDict
from http.cookiejar import parse_ns_headers
from pathlib import Path
from pprint import pformat
from urllib.parse import urlsplit
from typing import Any, List, Optional, Tuple, Callable, Iterable, TypeVar
import requests.auth
@@ -237,3 +238,7 @@ def unwrap_context(exc: Exception) -> Optional[Exception]:
return unwrap_context(context)
else:
return exc
def url_as_host(url: str) -> str:
return urlsplit(url).netloc.split('@')[-1]