diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index c2775164..f9106a54 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -9,69 +9,14 @@ on: - "**/*.py" jobs: - very_old_python: - if: github.event.pull_request.draft == false - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-13, windows-2022] - python-version: ["3.6"] - - steps: - - uses: actions/checkout@v3 - - name: "Set up timezone to America/Los_Angeles" - uses: szenius/set-timezone@v1.2 - with: - timezoneLinux: "America/Los_Angeles" - timezoneMacos: "America/Los_Angeles" - timezoneWindows: "Pacific Standard Time" - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Test with unittest - run: | - python -m unittest discover tests - - old_python: - if: github.event.pull_request.draft == false - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-13, ubuntu-22.04, windows-2022] - python-version: ["3.7", "3.8", "3.9", "3.10"] - - steps: - - uses: actions/checkout@v3 - - name: "Set up timezone to America/Los_Angeles" - uses: szenius/set-timezone@v1.2 - with: - timezoneLinux: "America/Los_Angeles" - timezoneMacos: "America/Los_Angeles" - timezoneWindows: "Pacific Standard Time" - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Test with unittest - run: | - python -m unittest discover tests latest_python: if: github.event.pull_request.draft == false runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ["3.11", "3.12"] + os: [macos-15-intel, macos-latest, ubuntu-latest, ubuntu-24.04-arm, windows-latest] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v3 @@ -92,3 +37,59 @@ jobs: - name: Test with unittest run: | python -m unittest discover tests + + # very_old_python: + # if: github.event.pull_request.draft == false + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # os: [macos-13, windows-2022] + # python-version: ["3.6"] + + # steps: + # - uses: actions/checkout@v3 + # - name: "Set up timezone to America/Los_Angeles" + # uses: szenius/set-timezone@v1.2 + # with: + # timezoneLinux: "America/Los_Angeles" + # timezoneMacos: "America/Los_Angeles" + # timezoneWindows: "Pacific Standard Time" + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install -r requirements.txt + # - name: Test with unittest + # run: | + # python -m unittest discover tests + + # old_python: + # if: github.event.pull_request.draft == false + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # os: [macos-13, ubuntu-22.04, windows-2022] + # python-version: ["3.7", "3.8", "3.9", "3.10"] + + # steps: + # - uses: actions/checkout@v3 + # - name: "Set up timezone to America/Los_Angeles" + # uses: szenius/set-timezone@v1.2 + # with: + # timezoneLinux: "America/Los_Angeles" + # timezoneMacos: "America/Los_Angeles" + # timezoneWindows: "Pacific Standard Time" + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install -r requirements.txt + # - name: Test with unittest + # run: | + # python -m unittest discover tests diff --git a/CHANGELOG b/CHANGELOG index d911bb90..a4f892e0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,15 @@ jc changelog -202501012 v1.25.6 +20260330 v1.25.7 +- Add `typeset` and `declare` Bash internal command parser to convert variables + simple arrays, and associative arrays along with object metadata +- Enhance `rsync` and `rsync-s` parsers to add `--stats` or `--info=stats[1-3]` fields +- Fix `hashsum` command parser to correctly parse the `mode` indicator +- Fix `proc-pid-smaps` proc parser when unknown VmFlags are output +- Fix `ifconfig` command parser for incorrect stripping of leading zeros in some hex numbers +- Fix `iptables` command parser when Target is blank and verbose output is used + +20251012 v1.25.6 - Add `net-localgroup` Windows command parser - Add `net-user` Windows command parser - Add `route-print` Windows command parser diff --git a/README.md b/README.md index b5759ad8..597921d3 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ pip3 install jc | FreeBSD | `portsnap fetch update && cd /usr/ports/textproc/py-jc && make install clean` | | Ansible filter plugin | `ansible-galaxy collection install community.general` | | FortiSOAR connector | Install from FortiSOAR Connector Marketplace | +| Mise-en-place (Linux/MacOS) | `mise use -g jc@latest` | > For more OS Packages, see https://repology.org/project/jc/versions. diff --git a/jc/cli.py b/jc/cli.py index 855aaa78..7f559286 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -300,8 +300,8 @@ class JcCli(): Pages the parser documentation if a parser is found in the arguments, otherwise the general help text is printed. """ - self.indent = 4 - self.pad = 22 + self.indent = 2 + self.pad = 21 if self.show_categories: utils._safe_print(self.parser_categories_text()) @@ -569,7 +569,11 @@ class JcCli(): if self.debug: raise - error_msg = os.strerror(e.errno) + if e.errno: + error_msg = os.strerror(e.errno) + else: + error_msg = "no further information provided" + utils.error_message([ f'"{file}" file could not be opened: {error_msg}.' ]) @@ -594,7 +598,11 @@ class JcCli(): if self.debug: raise - error_msg = os.strerror(e.errno) + if e.errno: + error_msg = os.strerror(e.errno) + else: + error_msg = "no further information provided" + utils.error_message([ f'"{self.magic_run_command_str}" command could not be run: {error_msg}.' ]) diff --git a/jc/cli_data.py b/jc/cli_data.py index 261bc989..6cdaae10 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -62,52 +62,52 @@ jc converts the output of many commands, file-types, and strings to JSON or YAML Usage: - Standard syntax: + Standard syntax: - COMMAND | jc [SLICE] [OPTIONS] PARSER + COMMAND | jc [SLICE] [OPTIONS] PARSER - cat FILE | jc [SLICE] [OPTIONS] PARSER + cat FILE | jc [SLICE] [OPTIONS] PARSER - echo STRING | jc [SLICE] [OPTIONS] PARSER + echo STRING | jc [SLICE] [OPTIONS] PARSER - Magic syntax: + Magic syntax: - jc [SLICE] [OPTIONS] COMMAND + jc [SLICE] [OPTIONS] COMMAND - jc [SLICE] [OPTIONS] /proc/ + jc [SLICE] [OPTIONS] /proc/ Parsers: ''' slicetext_string: str = '''\ Slice: - [start]:[end] + [start]:[end] - start: [[-]index] - Zero-based start line, negative index for - counting from the end + start: [[-]index] - Zero-based start line, negative index for + counting from the end - end: [[-]index] - Zero-based end line (excluding the index), - negative index for counting from the end + end: [[-]index] - Zero-based end line (excluding the index), + negative index for counting from the end ''' helptext_end_string: str = '''\ Examples: - Standard Syntax: - $ dig www.google.com | jc --pretty --dig - $ cat /proc/meminfo | jc --pretty --proc + Standard Syntax: + $ dig www.google.com | jc --pretty --dig + $ cat /proc/meminfo | jc --pretty --proc - Magic Syntax: - $ jc --pretty dig www.google.com - $ jc --pretty /proc/meminfo + Magic Syntax: + $ jc --pretty dig www.google.com + $ jc --pretty /proc/meminfo - Line Slicing: - $ cat output.txt | jc 4:15 -- # Parse from line 4 to 14 - # with (zero-based) + Line Slicing: + $ cat output.txt | jc 4:15 -- # Parse from line 4 to 14 + # with (zero-based) - Parser Documentation: - $ jc --help --dig + Parser Documentation: + $ jc --help --dig - More Help: - $ jc -hh # show hidden parsers - $ jc -hhh # list parsers by category tags + More Help: + $ jc -hh # show hidden parsers + $ jc -hhh # list parsers by category tags ''' \ No newline at end of file diff --git a/jc/lib.py b/jc/lib.py index c21a0284..f29f3b3c 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -10,7 +10,7 @@ from jc import appdirs from jc import utils -__version__ = '1.25.6' +__version__ = '1.25.7' parsers: List[str] = [ 'acpi', @@ -216,6 +216,7 @@ parsers: List[str] = [ 'traceroute', 'traceroute-s', 'tune2fs', + 'typeset', 'udevadm', 'ufw', 'ufw-appinfo', diff --git a/jc/parsers/hashsum.py b/jc/parsers/hashsum.py index d8633997..7e96b904 100644 --- a/jc/parsers/hashsum.py +++ b/jc/parsers/hashsum.py @@ -28,6 +28,7 @@ Schema: [ { "filename": string, + "mode": string, "hash": string, } ] @@ -38,37 +39,44 @@ Examples: [ { "filename": "devtoolset-3-gcc-4.9.2-6.el7.x86_64.rpm", + "mode": "text", "hash": "65fc958c1add637ec23c4b137aecf3d3" }, { "filename": "digout", + "mode": "text", "hash": "5b9312ee5aff080927753c63a347707d" }, { "filename": "dmidecode.out", + "mode": "text", "hash": "716fd11c2ac00db109281f7110b8fb9d" }, { "filename": "file with spaces in the name", + "mode": "text", "hash": "d41d8cd98f00b204e9800998ecf8427e" }, { "filename": "id-centos.out", + "mode": "text", "hash": "4295be239a14ad77ef3253103de976d2" }, { "filename": "ifcfg.json", + "mode": "text", "hash": "01fda0d9ba9a75618b072e64ff512b43" }, ... ] """ +import re import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = 'hashsum command parser (`md5sum`, `shasum`, etc.)' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -81,6 +89,15 @@ class info(): __version__ = info.version +_mode_friendly_names = { + " ": "text", + "*": "binary", + # Perl shasum -- specific + "U": "universal", + "^": "bits", + # BSD-style format only supports binary mode + None: "binary" +} def _process(proc_data): """ @@ -95,7 +112,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ - # no further processing for this parser + for entry in proc_data: + entry['mode'] = _mode_friendly_names.get(entry['mode'],entry['mode']) + return proc_data @@ -127,18 +146,20 @@ def parse(data, raw=False, quiet=False): file_name = line.split('=', maxsplit=1)[0].strip() file_name = file_name[5:] file_name = file_name[:-1] + # filler, legacy md5 always uses binary mode + file_mode = None # standard md5sum and shasum command output else: - file_hash = line.split(maxsplit=1)[0] - file_name = line.split(maxsplit=1)[1] + m = re.match('([0-9a-f]+) (.)(.*)$', line) + if not m: + raise ValueError(f'Invalid line format: "{line}"') + file_hash, file_mode, file_name = m.groups() item = { 'filename': file_name, + 'mode': file_mode, 'hash': file_hash } raw_output.append(item) - if raw: - return raw_output - else: - return _process(raw_output) + return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index 021f5026..5e3bd41c 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -219,7 +219,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '2.4' + version = '2.5' description = '`ifconfig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -264,7 +264,7 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: try: if entry['ipv4_mask'].startswith('0x'): new_mask = entry['ipv4_mask'] - new_mask = new_mask.lstrip('0x') + new_mask = new_mask[2:] new_mask = '.'.join(str(int(i, 16)) for i in [new_mask[i:i + 2] for i in range(0, len(new_mask), 2)]) entry['ipv4_mask'] = new_mask except (ValueError, TypeError, AttributeError): @@ -289,7 +289,7 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: try: if ip_address['mask'].startswith('0x'): new_mask = ip_address['mask'] - new_mask = new_mask.lstrip('0x') + new_mask = new_mask[2:] new_mask = '.'.join(str(int(i, 16)) for i in [new_mask[i:i + 2] for i in range(0, len(new_mask), 2)]) ip_address['mask'] = new_mask except (ValueError, TypeError, AttributeError): diff --git a/jc/parsers/iptables.py b/jc/parsers/iptables.py index 9dd8b7b5..f5d21adb 100644 --- a/jc/parsers/iptables.py +++ b/jc/parsers/iptables.py @@ -173,7 +173,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.12' + version = '1.13' description = '`iptables` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -294,9 +294,16 @@ def parse(data, raw=False, quiet=False): else: # sometimes the "target" column is blank. Stuff in a dummy character - if headers[0] == 'target' and line.startswith(' '): + opt_values = {'--', '-f', '!f'} + line_split = line.split() + if headers[0] == 'target' and line.startswith(' '): # standard output line = '\u2063' + line + elif headers[0] == 'pkts' and line_split[3] in opt_values: # verbose output + first_section = line_split[:2] + second_section = line_split[2:] + line = ' '.join(first_section) + ' \u2063 ' + ' '.join(second_section) + rule = line.split(maxsplit=len(headers) - 1) temp_rule = dict(zip(headers, rule)) if temp_rule: diff --git a/jc/parsers/proc_pid_smaps.py b/jc/parsers/proc_pid_smaps.py index d7be98fc..bedd6a08 100644 --- a/jc/parsers/proc_pid_smaps.py +++ b/jc/parsers/proc_pid_smaps.py @@ -168,7 +168,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`/proc//smaps` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -205,33 +205,46 @@ def _process(proc_data: List[Dict]) -> List[Dict]: vmflags_map = { 'rd': 'readable', - 'wr': 'writeable', - 'ex': 'executable', - 'sh': 'shared', - 'mr': 'may read', - 'mw': 'may write', - 'me': 'may execute', - 'ms': 'may share', - 'mp': 'MPX-specific VMA', - 'gd': 'stack segment growns down', - 'pf': 'pure PFN range', - 'dw': 'disabled write to the mapped file', - 'lo': 'pages are locked in memory', - 'io': 'memory mapped I/O area', - 'sr': 'sequential read advise provided', - 'rr': 'random read advise provided', - 'dc': 'do not copy area on fork', - 'de': 'do not expand area on remapping', - 'ac': 'area is accountable', - 'nr': 'swap space is not reserved for the area', - 'ht': 'area uses huge tlb pages', - 'ar': 'architecture specific flag', - 'dd': 'do not include area into core dump', - 'sd': 'soft-dirty flag', - 'mm': 'mixed map area', - 'hg': 'huge page advise flag', - 'nh': 'no-huge page advise flag', - 'mg': 'mergable advise flag' + 'wr': 'writeable', + 'ex': 'executable', + 'sh': 'shared', + 'mr': 'may read', + 'mw': 'may write', + 'me': 'may execute', + 'ms': 'may share', + 'mp': 'MPX-specific VMA', + 'gd': 'stack segment growns down', + 'pf': 'pure PFN range', + 'dw': 'disabled write to the mapped file', + 'lo': 'pages are locked in memory', + 'io': 'memory mapped I/O area', + 'sr': 'sequential read advise provided', + 'rr': 'random read advise provided', + 'dc': 'do not copy area on fork', + 'de': 'do not expand area on remapping', + 'ac': 'area is accountable', + 'nr': 'swap space is not reserved for the area', + 'ht': 'area uses huge tlb pages', + 'sf': 'perform synchronous page faults', + 'nl': 'non-linear mapping', + 'ar': 'architecture specific flag', + 'wf': 'wipe on fork', + 'dd': 'do not include area into core dump', + 'sd': 'soft-dirty flag', + 'mm': 'mixed map area', + 'hg': 'huge page advise flag', + 'nh': 'no-huge page advise flag', + 'mg': 'mergable advise flag', + 'bt': 'arm64 BTI guarded page', + 'mt': 'arm64 MTE allocation tags are enabled', + 'um': 'userfaultfd missing pages tracking', + 'uw': 'userfaultfd wprotect pages tracking', + 'ui': 'userfaultfd minor fault', + 'ss': 'shadow/guarded control stack page', + 'sl': 'sealed', + 'lf': 'lock on fault pages', + 'dp': 'always lazily freeable mapping', + 'gu': 'maybe contains guard regions' } for entry in proc_data: @@ -245,7 +258,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: if 'VmFlags' in entry: entry['VmFlags'] = entry['VmFlags'].split() - entry['VmFlags_pretty'] = [vmflags_map[x] for x in entry['VmFlags']] + entry['VmFlags_pretty'] = [vmflags_map.get(x, x) for x in entry['VmFlags']] return proc_data diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index 195aae77..cee983b3 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -4,6 +4,8 @@ Supports the `-i` or `--itemize-changes` options with all levels of verbosity. This parser will process the `STDOUT` output or a log file generated with the `--log-file` option. +The `--stats` or `--info=stats[1-3]` options are also supported. + Usage (cli): $ rsync -i -a source/ dest | jc --rsync @@ -37,7 +39,21 @@ Schema: "false_alarms": integer, "data": integer, "bytes_sec": float, - "speedup": float + "speedup": float, + "total_files": integer, + "regular_files": integer, + "dir_files": integer, + "total_created_files": integer, + "created_regular_files": integer, + "created_dir_files": integer, + "deleted_files": integer, + "transferred_files": integer, + "transferred_file_size": integer, + "literal_data": integer, + "matched_data": integer, + "file_list_size": integer, + "file_list_generation_time": float, + "file_list_transfer_time": float, }, "files": [ { @@ -62,6 +78,8 @@ Schema: } ] + Size values are in bytes. + [0] 'file sent', 'file received', 'local change or creation', 'hard link', 'not updated', 'message' [1] 'file', 'directory', 'symlink', 'device', 'special file' @@ -137,7 +155,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = '`rsync` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -163,10 +181,16 @@ def _process(proc_data: List[Dict]) -> List[Dict]: """ int_list = { 'process', 'sent', 'received', 'total_size', 'matches', 'hash_hits', - 'false_alarms', 'data' + 'false_alarms', 'data', 'total_files', 'regular_files', 'dir_files', + 'total_created_files', 'created_regular_files', 'created_dir_files', + 'deleted_files', 'transferred_files', 'transferred_file_size', + 'literal_data', 'matched_data', 'file_list_size' } - float_list = {'bytes_sec', 'speedup'} + float_list = { + 'bytes_sec', 'speedup', 'file_list_generation_time', + 'file_list_transfer_time' + } for item in proc_data: for key in item['summary']: @@ -338,6 +362,17 @@ def parse( stat2_line_log_v_re = re.compile(r'(?P\d\d\d\d/\d\d/\d\d)\s+(?P