From 4d7a872670f915cc075f3b454b20e5db4d5ae741 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 21 Jan 2023 10:56:45 -0800 Subject: [PATCH 01/81] version bump --- CHANGELOG | 4 ++++ jc/lib.py | 2 +- setup.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 70348fc0..82e2fb4c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,9 @@ jc changelog +20230121 v1.23.0 +- Add ssh_conf file parser +- Add input slicing + 20230111 v1.22.5 - Add TOML file parser - Add INI with duplicate key support file parser diff --git a/jc/lib.py b/jc/lib.py index 55978c14..6333b81c 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -9,7 +9,7 @@ from .jc_types import ParserInfoType, JSONDictType from jc import appdirs -__version__ = '1.22.5' +__version__ = '1.23.0' parsers: List[str] = [ 'acpi', diff --git a/setup.py b/setup.py index 8d59c8d4..3485138c 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='1.22.5', + version='1.23.0', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', From ac9128fa0c06e2c45990a65bde82853faa6a0ac1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 21 Jan 2023 13:10:45 -0800 Subject: [PATCH 02/81] add ssh-conf parser --- jc/lib.py | 5 +- jc/parsers/paramiko/LICENSE | 504 ++++++++++++++++++++ jc/parsers/paramiko/__init__.py | 0 jc/parsers/paramiko/config.py | 679 +++++++++++++++++++++++++++ jc/parsers/paramiko/ssh_exception.py | 235 +++++++++ jc/parsers/ssh_conf.py | 93 ++++ 6 files changed, 1514 insertions(+), 2 deletions(-) create mode 100644 jc/parsers/paramiko/LICENSE create mode 100644 jc/parsers/paramiko/__init__.py create mode 100644 jc/parsers/paramiko/config.py create mode 100644 jc/parsers/paramiko/ssh_exception.py create mode 100644 jc/parsers/ssh_conf.py diff --git a/jc/lib.py b/jc/lib.py index 6333b81c..1ca94c88 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -3,7 +3,7 @@ import sys import os import re import importlib -from typing import List, Iterable, Union, Iterator +from typing import List, Iterable, Optional, Union, Iterator from types import ModuleType from .jc_types import ParserInfoType, JSONDictType from jc import appdirs @@ -159,6 +159,7 @@ parsers: List[str] = [ 'sfdisk', 'shadow', 'ss', + 'ssh-conf', 'sshd-conf', 'stat', 'stat-s', @@ -279,7 +280,7 @@ def parse( data: Union[str, bytes, Iterable[str]], quiet: bool = False, raw: bool = False, - ignore_exceptions: bool = None, + ignore_exceptions: Optional[bool] = None, **kwargs ) -> Union[JSONDictType, List[JSONDictType], Iterator[JSONDictType]]: """ diff --git a/jc/parsers/paramiko/LICENSE b/jc/parsers/paramiko/LICENSE new file mode 100644 index 00000000..d12bef0c --- /dev/null +++ b/jc/parsers/paramiko/LICENSE @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/jc/parsers/paramiko/__init__.py b/jc/parsers/paramiko/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/jc/parsers/paramiko/config.py b/jc/parsers/paramiko/config.py new file mode 100644 index 00000000..48bcb101 --- /dev/null +++ b/jc/parsers/paramiko/config.py @@ -0,0 +1,679 @@ +# Copyright (C) 2006-2007 Robey Pointer +# Copyright (C) 2012 Olle Lundberg +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Configuration file (aka ``ssh_config``) support. +""" + +import fnmatch +import getpass +import os +import re +import shlex +import socket +from hashlib import sha1 +from io import StringIO +from functools import partial + +invoke, invoke_import_error = None, None +try: + import invoke +except ImportError as e: + invoke_import_error = e + +from .ssh_exception import CouldNotCanonicalize, ConfigParseError + + +SSH_PORT = 22 + + +class SSHConfig: + """ + Representation of config information as stored in the format used by + OpenSSH. Queries can be made via `lookup`. The format is described in + OpenSSH's ``ssh_config`` man page. This class is provided primarily as a + convenience to posix users (since the OpenSSH format is a de-facto + standard on posix) but should work fine on Windows too. + + .. versionadded:: 1.6 + """ + + SETTINGS_REGEX = re.compile(r"(\w+)(?:\s*=\s*|\s+)(.+)") + + # TODO: do a full scan of ssh.c & friends to make sure we're fully + # compatible across the board, e.g. OpenSSH 8.1 added %n to ProxyCommand. + TOKENS_BY_CONFIG_KEY = { + "controlpath": ["%C", "%h", "%l", "%L", "%n", "%p", "%r", "%u"], + "hostname": ["%h"], + "identityfile": ["%C", "~", "%d", "%h", "%l", "%u", "%r"], + "proxycommand": ["~", "%h", "%p", "%r"], + "proxyjump": ["%h", "%p", "%r"], + # Doesn't seem worth making this 'special' for now, it will fit well + # enough (no actual match-exec config key to be confused with). + "match-exec": ["%C", "%d", "%h", "%L", "%l", "%n", "%p", "%r", "%u"], + } + + def __init__(self): + """ + Create a new OpenSSH config object. + + Note: the newer alternate constructors `from_path`, `from_file` and + `from_text` are simpler to use, as they parse on instantiation. For + example, instead of:: + + config = SSHConfig() + config.parse(open("some-path.config") + + you could:: + + config = SSHConfig.from_file(open("some-path.config")) + # Or more directly: + config = SSHConfig.from_path("some-path.config") + # Or if you have arbitrary ssh_config text from some other source: + config = SSHConfig.from_text("Host foo\\n\\tUser bar") + """ + self._config = [] + + @classmethod + def from_text(cls, text): + """ + Create a new, parsed `SSHConfig` from ``text`` string. + + .. versionadded:: 2.7 + """ + return cls.from_file(StringIO(text)) + + @classmethod + def from_path(cls, path): + """ + Create a new, parsed `SSHConfig` from the file found at ``path``. + + .. versionadded:: 2.7 + """ + with open(path) as flo: + return cls.from_file(flo) + + @classmethod + def from_file(cls, flo): + """ + Create a new, parsed `SSHConfig` from file-like object ``flo``. + + .. versionadded:: 2.7 + """ + obj = cls() + obj.parse(flo) + return obj + + def parse(self, file_obj): + """ + Read an OpenSSH config from the given file object. + + :param file_obj: a file-like object to read the config file from + """ + # Start out w/ implicit/anonymous global host-like block to hold + # anything not contained by an explicit one. + context = {"host": ["*"], "config": {}} + for line in file_obj: + # Strip any leading or trailing whitespace from the line. + # Refer to https://github.com/paramiko/paramiko/issues/499 + line = line.strip() + # Skip blanks, comments + if not line or line.startswith("#"): + continue + + # Parse line into key, value + match = re.match(self.SETTINGS_REGEX, line) + if not match: + raise ConfigParseError("Unparsable line {}".format(line)) + key = match.group(1).lower() + value = match.group(2) + + # Host keyword triggers switch to new block/context + if key in ("host", "match"): + self._config.append(context) + context = {"config": {}} + if key == "host": + # TODO 4.0: make these real objects or at least name this + # "hosts" to acknowledge it's an iterable. (Doing so prior + # to 3.0, despite it being a private API, feels bad - + # surely such an old codebase has folks actually relying on + # these keys.) + context["host"] = self._get_hosts(value) + else: + context["matches"] = self._get_matches(value) + # Special-case for noop ProxyCommands + elif key == "proxycommand" and value.lower() == "none": + # Store 'none' as None - not as a string implying that the + # proxycommand is the literal shell command "none"! + context["config"][key] = None + # All other keywords get stored, directly or via append + else: + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + + # identityfile, localforward, remoteforward keys are special + # cases, since they are allowed to be specified multiple times + # and they should be tried in order of specification. + if key in ["identityfile", "localforward", "remoteforward"]: + if key in context["config"]: + context["config"][key].append(value) + else: + context["config"][key] = [value] + elif key not in context["config"]: + context["config"][key] = value + # Store last 'open' block and we're done + self._config.append(context) + + def lookup(self, hostname): + """ + Return a dict (`SSHConfigDict`) of config options for a given hostname. + + The host-matching rules of OpenSSH's ``ssh_config`` man page are used: + For each parameter, the first obtained value will be used. The + configuration files contain sections separated by ``Host`` and/or + ``Match`` specifications, and that section is only applied for hosts + which match the given patterns or keywords + + Since the first obtained value for each parameter is used, more host- + specific declarations should be given near the beginning of the file, + and general defaults at the end. + + The keys in the returned dict are all normalized to lowercase (look for + ``"port"``, not ``"Port"``. The values are processed according to the + rules for substitution variable expansion in ``ssh_config``. + + Finally, please see the docs for `SSHConfigDict` for deeper info on + features such as optional type conversion methods, e.g.:: + + conf = my_config.lookup('myhost') + assert conf['passwordauthentication'] == 'yes' + assert conf.as_bool('passwordauthentication') is True + + .. note:: + If there is no explicitly configured ``HostName`` value, it will be + set to the being-looked-up hostname, which is as close as we can + get to OpenSSH's behavior around that particular option. + + :param str hostname: the hostname to lookup + + .. versionchanged:: 2.5 + Returns `SSHConfigDict` objects instead of dict literals. + .. versionchanged:: 2.7 + Added canonicalization support. + .. versionchanged:: 2.7 + Added ``Match`` support. + """ + # First pass + options = self._lookup(hostname=hostname) + # Inject HostName if it was not set (this used to be done incidentally + # during tokenization, for some reason). + if "hostname" not in options: + options["hostname"] = hostname + # Handle canonicalization + canon = options.get("canonicalizehostname", None) in ("yes", "always") + maxdots = int(options.get("canonicalizemaxdots", 1)) + if canon and hostname.count(".") <= maxdots: + # NOTE: OpenSSH manpage does not explicitly state this, but its + # implementation for CanonicalDomains is 'split on any whitespace'. + domains = options["canonicaldomains"].split() + hostname = self.canonicalize(hostname, options, domains) + # Overwrite HostName again here (this is also what OpenSSH does) + options["hostname"] = hostname + options = self._lookup(hostname, options, canonical=True) + return options + + def _lookup(self, hostname, options=None, canonical=False): + # Init + if options is None: + options = SSHConfigDict() + # Iterate all stanzas, applying any that match, in turn (so that things + # like Match can reference currently understood state) + for context in self._config: + if not ( + self._pattern_matches(context.get("host", []), hostname) + or self._does_match( + context.get("matches", []), hostname, canonical, options + ) + ): + continue + for key, value in context["config"].items(): + if key not in options: + # Create a copy of the original value, + # else it will reference the original list + # in self._config and update that value too + # when the extend() is being called. + options[key] = value[:] if value is not None else value + elif key == "identityfile": + options[key].extend( + x for x in value if x not in options[key] + ) + # Expand variables in resulting values (besides 'Match exec' which was + # already handled above) + options = self._expand_variables(options, hostname) + return options + + def canonicalize(self, hostname, options, domains): + """ + Return canonicalized version of ``hostname``. + + :param str hostname: Target hostname. + :param options: An `SSHConfigDict` from a previous lookup pass. + :param domains: List of domains (e.g. ``["paramiko.org"]``). + + :returns: A canonicalized hostname if one was found, else ``None``. + + .. versionadded:: 2.7 + """ + found = False + for domain in domains: + candidate = "{}.{}".format(hostname, domain) + family_specific = _addressfamily_host_lookup(candidate, options) + if family_specific is not None: + # TODO: would we want to dig deeper into other results? e.g. to + # find something that satisfies PermittedCNAMEs when that is + # implemented? + found = family_specific[0] + else: + # TODO: what does ssh use here and is there a reason to use + # that instead of gethostbyname? + try: + found = socket.gethostbyname(candidate) + except socket.gaierror: + pass + if found: + # TODO: follow CNAME (implied by found != candidate?) if + # CanonicalizePermittedCNAMEs allows it + return candidate + # If we got here, it means canonicalization failed. + # When CanonicalizeFallbackLocal is undefined or 'yes', we just spit + # back the original hostname. + if options.get("canonicalizefallbacklocal", "yes") == "yes": + return hostname + # And here, we failed AND fallback was set to a non-yes value, so we + # need to get mad. + raise CouldNotCanonicalize(hostname) + + def get_hostnames(self): + """ + Return the set of literal hostnames defined in the SSH config (both + explicit hostnames and wildcard entries). + """ + hosts = set() + for entry in self._config: + hosts.update(entry["host"]) + return hosts + + def _pattern_matches(self, patterns, target): + # Convenience auto-splitter if not already a list + if hasattr(patterns, "split"): + patterns = patterns.split(",") + match = False + for pattern in patterns: + # Short-circuit if target matches a negated pattern + if pattern.startswith("!") and fnmatch.fnmatch( + target, pattern[1:] + ): + return False + # Flag a match, but continue (in case of later negation) if regular + # match occurs + elif fnmatch.fnmatch(target, pattern): + match = True + return match + + def _does_match(self, match_list, target_hostname, canonical, options): + matched = [] + candidates = match_list[:] + local_username = getpass.getuser() + while candidates: + candidate = candidates.pop(0) + passed = None + # Obtain latest host/user value every loop, so later Match may + # reference values assigned within a prior Match. + configured_host = options.get("hostname", None) + configured_user = options.get("user", None) + type_, param = candidate["type"], candidate["param"] + # Canonical is a hard pass/fail based on whether this is a + # canonicalized re-lookup. + if type_ == "canonical": + if self._should_fail(canonical, candidate): + return False + # The parse step ensures we only see this by itself or after + # canonical, so it's also an easy hard pass. (No negation here as + # that would be uh, pretty weird?) + elif type_ == "all": + return True + # From here, we are testing various non-hard criteria, + # short-circuiting only on fail + elif type_ == "host": + hostval = configured_host or target_hostname + passed = self._pattern_matches(param, hostval) + elif type_ == "originalhost": + passed = self._pattern_matches(param, target_hostname) + elif type_ == "user": + user = configured_user or local_username + passed = self._pattern_matches(param, user) + elif type_ == "localuser": + passed = self._pattern_matches(param, local_username) + elif type_ == "exec": + exec_cmd = self._tokenize( + options, target_hostname, "match-exec", param + ) + # This is the laziest spot in which we can get mad about an + # inability to import Invoke. + if invoke is None: + raise invoke_import_error + # Like OpenSSH, we 'redirect' stdout but let stderr bubble up + passed = invoke.run(exec_cmd, hide="stdout", warn=True).ok + # Tackle any 'passed, but was negated' results from above + if passed is not None and self._should_fail(passed, candidate): + return False + # Made it all the way here? Everything matched! + matched.append(candidate) + # Did anything match? (To be treated as bool, usually.) + return matched + + def _should_fail(self, would_pass, candidate): + return would_pass if candidate["negate"] else not would_pass + + def _tokenize(self, config, target_hostname, key, value): + """ + Tokenize a string based on current config/hostname data. + + :param config: Current config data. + :param target_hostname: Original target connection hostname. + :param key: Config key being tokenized (used to filter token list). + :param value: Config value being tokenized. + + :returns: The tokenized version of the input ``value`` string. + """ + allowed_tokens = self._allowed_tokens(key) + # Short-circuit if no tokenization possible + if not allowed_tokens: + return value + # Obtain potentially configured hostname, for use with %h. + # Special-case where we are tokenizing the hostname itself, to avoid + # replacing %h with a %h-bearing value, etc. + configured_hostname = target_hostname + if key != "hostname": + configured_hostname = config.get("hostname", configured_hostname) + # Ditto the rest of the source values + if "port" in config: + port = config["port"] + else: + port = SSH_PORT + user = getpass.getuser() + if "user" in config: + remoteuser = config["user"] + else: + remoteuser = user + local_hostname = socket.gethostname().split(".")[0] + local_fqdn = LazyFqdn(config, local_hostname) + homedir = os.path.expanduser("~") + tohash = local_hostname + target_hostname + repr(port) + remoteuser + # The actual tokens! + replacements = { + # TODO: %%??? + "%C": sha1(tohash.encode()).hexdigest(), + "%d": homedir, + "%h": configured_hostname, + # TODO: %i? + "%L": local_hostname, + "%l": local_fqdn, + # also this is pseudo buggy when not in Match exec mode so document + # that. also WHY is that the case?? don't we do all of this late? + "%n": target_hostname, + "%p": port, + "%r": remoteuser, + # TODO: %T? don't believe this is possible however + "%u": user, + "~": homedir, + } + # Do the thing with the stuff + tokenized = value + for find, replace in replacements.items(): + if find not in allowed_tokens: + continue + tokenized = tokenized.replace(find, str(replace)) + # TODO: log? eg that value -> tokenized + return tokenized + + def _allowed_tokens(self, key): + """ + Given config ``key``, return list of token strings to tokenize. + + .. note:: + This feels like it wants to eventually go away, but is used to + preserve as-strict-as-possible compatibility with OpenSSH, which + for whatever reason only applies some tokens to some config keys. + """ + return self.TOKENS_BY_CONFIG_KEY.get(key, []) + + def _expand_variables(self, config, target_hostname): + """ + Return a dict of config options with expanded substitutions + for a given original & current target hostname. + + Please refer to :doc:`/api/config` for details. + + :param dict config: the currently parsed config + :param str hostname: the hostname whose config is being looked up + """ + for k in config: + if config[k] is None: + continue + tokenizer = partial(self._tokenize, config, target_hostname, k) + if isinstance(config[k], list): + for i, value in enumerate(config[k]): + config[k][i] = tokenizer(value) + else: + config[k] = tokenizer(config[k]) + return config + + def _get_hosts(self, host): + """ + Return a list of host_names from host value. + """ + try: + return shlex.split(host) + except ValueError: + raise ConfigParseError("Unparsable host {}".format(host)) + + def _get_matches(self, match): + """ + Parse a specific Match config line into a list-of-dicts for its values. + + Performs some parse-time validation as well. + """ + matches = [] + tokens = shlex.split(match) + while tokens: + match = {"type": None, "param": None, "negate": False} + type_ = tokens.pop(0) + # Handle per-keyword negation + if type_.startswith("!"): + match["negate"] = True + type_ = type_[1:] + match["type"] = type_ + # all/canonical have no params (everything else does) + if type_ in ("all", "canonical"): + matches.append(match) + continue + if not tokens: + raise ConfigParseError( + "Missing parameter to Match '{}' keyword".format(type_) + ) + match["param"] = tokens.pop(0) + matches.append(match) + # Perform some (easier to do now than in the middle) validation that is + # better handled here than at lookup time. + keywords = [x["type"] for x in matches] + if "all" in keywords: + allowable = ("all", "canonical") + ok, bad = ( + list(filter(lambda x: x in allowable, keywords)), + list(filter(lambda x: x not in allowable, keywords)), + ) + err = None + if any(bad): + err = "Match does not allow 'all' mixed with anything but 'canonical'" # noqa + elif "canonical" in ok and ok.index("canonical") > ok.index("all"): + err = "Match does not allow 'all' before 'canonical'" + if err is not None: + raise ConfigParseError(err) + return matches + + +def _addressfamily_host_lookup(hostname, options): + """ + Try looking up ``hostname`` in an IPv4 or IPv6 specific manner. + + This is an odd duck due to needing use in two divergent use cases. It looks + up ``AddressFamily`` in ``options`` and if it is ``inet`` or ``inet6``, + this function uses `socket.getaddrinfo` to perform a family-specific + lookup, returning the result if successful. + + In any other situation -- lookup failure, or ``AddressFamily`` being + unspecified or ``any`` -- ``None`` is returned instead and the caller is + expected to do something situation-appropriate like calling + `socket.gethostbyname`. + + :param str hostname: Hostname to look up. + :param options: `SSHConfigDict` instance w/ parsed options. + :returns: ``getaddrinfo``-style tuples, or ``None``, depending. + """ + address_family = options.get("addressfamily", "any").lower() + if address_family == "any": + return + try: + family = socket.AF_INET6 + if address_family == "inet": + family = socket.AF_INET + return socket.getaddrinfo( + hostname, + None, + family, + socket.SOCK_DGRAM, + socket.IPPROTO_IP, + socket.AI_CANONNAME, + ) + except socket.gaierror: + pass + + +class LazyFqdn: + """ + Returns the host's fqdn on request as string. + """ + + def __init__(self, config, host=None): + self.fqdn = None + self.config = config + self.host = host + + def __str__(self): + if self.fqdn is None: + # + # If the SSH config contains AddressFamily, use that when + # determining the local host's FQDN. Using socket.getfqdn() from + # the standard library is the most general solution, but can + # result in noticeable delays on some platforms when IPv6 is + # misconfigured or not available, as it calls getaddrinfo with no + # address family specified, so both IPv4 and IPv6 are checked. + # + + # Handle specific option + fqdn = None + results = _addressfamily_host_lookup(self.host, self.config) + if results is not None: + for res in results: + af, socktype, proto, canonname, sa = res + if canonname and "." in canonname: + fqdn = canonname + break + # Handle 'any' / unspecified / lookup failure + if fqdn is None: + fqdn = socket.getfqdn() + # Cache + self.fqdn = fqdn + return self.fqdn + + +class SSHConfigDict(dict): + """ + A dictionary wrapper/subclass for per-host configuration structures. + + This class introduces some usage niceties for consumers of `SSHConfig`, + specifically around the issue of variable type conversions: normal value + access yields strings, but there are now methods such as `as_bool` and + `as_int` that yield casted values instead. + + For example, given the following ``ssh_config`` file snippet:: + + Host foo.example.com + PasswordAuthentication no + Compression yes + ServerAliveInterval 60 + + the following code highlights how you can access the raw strings as well as + usefully Python type-casted versions (recalling that keys are all + normalized to lowercase first):: + + my_config = SSHConfig() + my_config.parse(open('~/.ssh/config')) + conf = my_config.lookup('foo.example.com') + + assert conf['passwordauthentication'] == 'no' + assert conf.as_bool('passwordauthentication') is False + assert conf['compression'] == 'yes' + assert conf.as_bool('compression') is True + assert conf['serveraliveinterval'] == '60' + assert conf.as_int('serveraliveinterval') == 60 + + .. versionadded:: 2.5 + """ + + def as_bool(self, key): + """ + Express given key's value as a boolean type. + + Typically, this is used for ``ssh_config``'s pseudo-boolean values + which are either ``"yes"`` or ``"no"``. In such cases, ``"yes"`` yields + ``True`` and any other value becomes ``False``. + + .. note:: + If (for whatever reason) the stored value is already boolean in + nature, it's simply returned. + + .. versionadded:: 2.5 + """ + val = self[key] + if isinstance(val, bool): + return val + return val.lower() == "yes" + + def as_int(self, key): + """ + Express given key's value as an integer, if possible. + + This method will raise ``ValueError`` or similar if the value is not + int-appropriate, same as the builtin `int` type. + + .. versionadded:: 2.5 + """ + return int(self[key]) diff --git a/jc/parsers/paramiko/ssh_exception.py b/jc/parsers/paramiko/ssh_exception.py new file mode 100644 index 00000000..9b1b44c3 --- /dev/null +++ b/jc/parsers/paramiko/ssh_exception.py @@ -0,0 +1,235 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import socket + + +class SSHException(Exception): + """ + Exception raised by failures in SSH2 protocol negotiation or logic errors. + """ + + pass + + +class AuthenticationException(SSHException): + """ + Exception raised when authentication failed for some reason. It may be + possible to retry with different credentials. (Other classes specify more + specific reasons.) + + .. versionadded:: 1.6 + """ + + pass + + +class PasswordRequiredException(AuthenticationException): + """ + Exception raised when a password is needed to unlock a private key file. + """ + + pass + + +class BadAuthenticationType(AuthenticationException): + """ + Exception raised when an authentication type (like password) is used, but + the server isn't allowing that type. (It may only allow public-key, for + example.) + + .. versionadded:: 1.1 + """ + + allowed_types = [] + + # TODO 4.0: remove explanation kwarg + def __init__(self, explanation, types): + # TODO 4.0: remove this supercall unless it's actually required for + # pickling (after fixing pickling) + AuthenticationException.__init__(self, explanation, types) + self.explanation = explanation + self.allowed_types = types + + def __str__(self): + return "{}; allowed types: {!r}".format( + self.explanation, self.allowed_types + ) + + +class PartialAuthentication(AuthenticationException): + """ + An internal exception thrown in the case of partial authentication. + """ + + allowed_types = [] + + def __init__(self, types): + AuthenticationException.__init__(self, types) + self.allowed_types = types + + def __str__(self): + return "Partial authentication; allowed types: {!r}".format( + self.allowed_types + ) + + +class ChannelException(SSHException): + """ + Exception raised when an attempt to open a new `.Channel` fails. + + :param int code: the error code returned by the server + + .. versionadded:: 1.6 + """ + + def __init__(self, code, text): + SSHException.__init__(self, code, text) + self.code = code + self.text = text + + def __str__(self): + return "ChannelException({!r}, {!r})".format(self.code, self.text) + + +class BadHostKeyException(SSHException): + """ + The host key given by the SSH server did not match what we were expecting. + + :param str hostname: the hostname of the SSH server + :param PKey got_key: the host key presented by the server + :param PKey expected_key: the host key expected + + .. versionadded:: 1.6 + """ + + def __init__(self, hostname, got_key, expected_key): + SSHException.__init__(self, hostname, got_key, expected_key) + self.hostname = hostname + self.key = got_key + self.expected_key = expected_key + + def __str__(self): + msg = "Host key for server '{}' does not match: got '{}', expected '{}'" # noqa + return msg.format( + self.hostname, + self.key.get_base64(), + self.expected_key.get_base64(), + ) + + +class IncompatiblePeer(SSHException): + """ + A disagreement arose regarding an algorithm required for key exchange. + + .. versionadded:: 2.9 + """ + + # TODO 4.0: consider making this annotate w/ 1..N 'missing' algorithms, + # either just the first one that would halt kex, or even updating the + # Transport logic so we record /all/ that /could/ halt kex. + # TODO: update docstrings where this may end up raised so they are more + # specific. + pass + + +class ProxyCommandFailure(SSHException): + """ + The "ProxyCommand" found in the .ssh/config file returned an error. + + :param str command: The command line that is generating this exception. + :param str error: The error captured from the proxy command output. + """ + + def __init__(self, command, error): + SSHException.__init__(self, command, error) + self.command = command + self.error = error + + def __str__(self): + return 'ProxyCommand("{}") returned nonzero exit status: {}'.format( + self.command, self.error + ) + + +class NoValidConnectionsError(socket.error): + """ + Multiple connection attempts were made and no families succeeded. + + This exception class wraps multiple "real" underlying connection errors, + all of which represent failed connection attempts. Because these errors are + not guaranteed to all be of the same error type (i.e. different errno, + `socket.error` subclass, message, etc) we expose a single unified error + message and a ``None`` errno so that instances of this class match most + normal handling of `socket.error` objects. + + To see the wrapped exception objects, access the ``errors`` attribute. + ``errors`` is a dict whose keys are address tuples (e.g. ``('127.0.0.1', + 22)``) and whose values are the exception encountered trying to connect to + that address. + + It is implied/assumed that all the errors given to a single instance of + this class are from connecting to the same hostname + port (and thus that + the differences are in the resolution of the hostname - e.g. IPv4 vs v6). + + .. versionadded:: 1.16 + """ + + def __init__(self, errors): + """ + :param dict errors: + The errors dict to store, as described by class docstring. + """ + addrs = sorted(errors.keys()) + body = ", ".join([x[0] for x in addrs[:-1]]) + tail = addrs[-1][0] + if body: + msg = "Unable to connect to port {0} on {1} or {2}" + else: + msg = "Unable to connect to port {0} on {2}" + super().__init__( + None, msg.format(addrs[0][1], body, tail) # stand-in for errno + ) + self.errors = errors + + def __reduce__(self): + return (self.__class__, (self.errors,)) + + +class CouldNotCanonicalize(SSHException): + """ + Raised when hostname canonicalization fails & fallback is disabled. + + .. versionadded:: 2.7 + """ + + pass + + +class ConfigParseError(SSHException): + """ + A fatal error was encountered trying to parse SSH config data. + + Typically this means a config file violated the ``ssh_config`` + specification in a manner that requires exiting immediately, such as not + matching ``key = value`` syntax or misusing certain ``Match`` keywords. + + .. versionadded:: 2.7 + """ + + pass diff --git a/jc/parsers/ssh_conf.py b/jc/parsers/ssh_conf.py new file mode 100644 index 00000000..b9fc71ce --- /dev/null +++ b/jc/parsers/ssh_conf.py @@ -0,0 +1,93 @@ +"""jc - JSON Convert ssh configuration file parser + +Usage (cli): + + $ cat ssh_conf | jc --ssh-conf + +Usage (module): + + import jc + result = jc.parse('ssh_conf', ssh_conf_file_output) + +Schema: + + [ + { + "ssh_conf": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ cat ssh_conf | jc --ssh-conf -p + [] + + $ cat ssh_conf | jc --ssh-conf -p -r + [] +""" +from typing import List, Dict +from jc.jc_types import JSONDictType +import jc.utils +from .paramiko.config import SSHConfig as sshconfig + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'ssh config file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + details = 'Using Paramiko library at https://github.com/paramiko/paramiko.' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + tags = ['file'] + + +__version__ = info.version + + +def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[JSONDictType]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List[Dict] = [] + + if jc.utils.has_data(data): + myconfig = sshconfig.from_text(data) + hostnames = myconfig.get_hostnames() + raw_output = [myconfig.lookup(x) for x in hostnames] + + return raw_output if raw else _process(raw_output) From 0648d2e9e3152f0fff5864ab802289c18f89892b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 21 Jan 2023 16:37:27 -0800 Subject: [PATCH 03/81] don't interpolate local host values --- jc/parsers/paramiko/config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/jc/parsers/paramiko/config.py b/jc/parsers/paramiko/config.py index 48bcb101..e5e8f582 100644 --- a/jc/parsers/paramiko/config.py +++ b/jc/parsers/paramiko/config.py @@ -21,6 +21,8 @@ Configuration file (aka ``ssh_config``) support. """ +# search jc_change for jc fixes + import fnmatch import getpass import os @@ -429,20 +431,20 @@ class SSHConfig: # The actual tokens! replacements = { # TODO: %%??? - "%C": sha1(tohash.encode()).hexdigest(), - "%d": homedir, + # "%C": sha1(tohash.encode()).hexdigest(), # jc_change + # "%d": homedir, # jc_change "%h": configured_hostname, # TODO: %i? - "%L": local_hostname, - "%l": local_fqdn, + # "%L": local_hostname, # jc_change + # "%l": local_fqdn, # jc_change # also this is pseudo buggy when not in Match exec mode so document # that. also WHY is that the case?? don't we do all of this late? "%n": target_hostname, "%p": port, "%r": remoteuser, # TODO: %T? don't believe this is possible however - "%u": user, - "~": homedir, + # "%u": user, # jc_change + # "~": homedir, # jc_change } # Do the thing with the stuff tokenized = value From 2a148d44a1ac74cb34f206919e12ba7b84261ffd Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 21 Jan 2023 17:00:22 -0800 Subject: [PATCH 04/81] add host to objects --- jc/parsers/ssh_conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jc/parsers/ssh_conf.py b/jc/parsers/ssh_conf.py index b9fc71ce..80b5c727 100644 --- a/jc/parsers/ssh_conf.py +++ b/jc/parsers/ssh_conf.py @@ -87,7 +87,9 @@ def parse( if jc.utils.has_data(data): myconfig = sshconfig.from_text(data) - hostnames = myconfig.get_hostnames() + hostnames = sorted(myconfig.get_hostnames()) raw_output = [myconfig.lookup(x) for x in hostnames] + for host, obj in zip(hostnames, raw_output.copy()): + obj.update({'host': host}) return raw_output if raw else _process(raw_output) From dbfe68267490eb5547d03880c233558226c87176 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 22 Jan 2023 08:57:50 -0800 Subject: [PATCH 05/81] fix for never fully discharging state --- jc/parsers/acpi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jc/parsers/acpi.py b/jc/parsers/acpi.py index ed69fd1a..5afd1546 100644 --- a/jc/parsers/acpi.py +++ b/jc/parsers/acpi.py @@ -227,7 +227,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`acpi` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -336,7 +336,9 @@ def parse(data, raw=False, quiet=False): if 'Charging' in line or 'Discharging' in line or 'Full' in line: output_line['state'] = line.split()[2][:-1] output_line['charge_percent'] = line.split()[3].rstrip('%,') - if 'rate information unavailable' not in line: + if 'will never fully discharge' in line: + pass + elif 'rate information unavailable' not in line: if 'Charging' in line: output_line['until_charged'] = line.split()[4] if 'Discharging' in line: From f8fbb2dce274ee92464e42cb134e5a8f5ee41e96 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 22 Jan 2023 10:36:29 -0800 Subject: [PATCH 06/81] use derivative of sshd_conf parser instead of paramiko --- CHANGELOG | 5 +- jc/parsers/paramiko/LICENSE | 504 -------------------- jc/parsers/paramiko/__init__.py | 0 jc/parsers/paramiko/config.py | 681 --------------------------- jc/parsers/paramiko/ssh_exception.py | 235 --------- jc/parsers/ssh_conf.py | 591 ++++++++++++++++++++++- jc/parsers/sshd_conf.py | 8 +- 7 files changed, 578 insertions(+), 1446 deletions(-) delete mode 100644 jc/parsers/paramiko/LICENSE delete mode 100644 jc/parsers/paramiko/__init__.py delete mode 100644 jc/parsers/paramiko/config.py delete mode 100644 jc/parsers/paramiko/ssh_exception.py diff --git a/CHANGELOG b/CHANGELOG index 82e2fb4c..e9152163 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,9 @@ jc changelog -20230121 v1.23.0 -- Add ssh_conf file parser +20230122 v1.23.0 +- Add `ssh` configuration file parser - Add input slicing +- Fix `acpi` command parser for "will never fully discharge" battery state 20230111 v1.22.5 - Add TOML file parser diff --git a/jc/parsers/paramiko/LICENSE b/jc/parsers/paramiko/LICENSE deleted file mode 100644 index d12bef0c..00000000 --- a/jc/parsers/paramiko/LICENSE +++ /dev/null @@ -1,504 +0,0 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts - as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.] - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some -specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You -can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better -strategy to use in any particular case, based on the explanations below. - - When we speak of free software, we are referring to freedom of use, -not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge -for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of -it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid -distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for -you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them -with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the -library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that -there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know -that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be -introduced by others. - - Finally, software patents pose a constant threat to the existence of -any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a -restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be -consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the -ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and -is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those -libraries into non-free programs. - - When a program is linked with a library, whether statically or using -a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary -General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General -Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it -does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less -of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many -libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to -encourage the widest possible use of a certain library, so that it becomes -a de-facto standard. To achieve this, non-free programs must be -allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this -case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free -programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in -non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating -system. - - Although the Lesser General Public License is Less protective of the -users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run -that program using a modified version of the Library. - - The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, whereas the latter must -be combined with the library in order to run. - - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other -program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of -this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. - - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also combine or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, - rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if - the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties with -this License. - - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add -an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus -excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random Hacker. - - , 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! - - diff --git a/jc/parsers/paramiko/__init__.py b/jc/parsers/paramiko/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/jc/parsers/paramiko/config.py b/jc/parsers/paramiko/config.py deleted file mode 100644 index e5e8f582..00000000 --- a/jc/parsers/paramiko/config.py +++ /dev/null @@ -1,681 +0,0 @@ -# Copyright (C) 2006-2007 Robey Pointer -# Copyright (C) 2012 Olle Lundberg -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -""" -Configuration file (aka ``ssh_config``) support. -""" - -# search jc_change for jc fixes - -import fnmatch -import getpass -import os -import re -import shlex -import socket -from hashlib import sha1 -from io import StringIO -from functools import partial - -invoke, invoke_import_error = None, None -try: - import invoke -except ImportError as e: - invoke_import_error = e - -from .ssh_exception import CouldNotCanonicalize, ConfigParseError - - -SSH_PORT = 22 - - -class SSHConfig: - """ - Representation of config information as stored in the format used by - OpenSSH. Queries can be made via `lookup`. The format is described in - OpenSSH's ``ssh_config`` man page. This class is provided primarily as a - convenience to posix users (since the OpenSSH format is a de-facto - standard on posix) but should work fine on Windows too. - - .. versionadded:: 1.6 - """ - - SETTINGS_REGEX = re.compile(r"(\w+)(?:\s*=\s*|\s+)(.+)") - - # TODO: do a full scan of ssh.c & friends to make sure we're fully - # compatible across the board, e.g. OpenSSH 8.1 added %n to ProxyCommand. - TOKENS_BY_CONFIG_KEY = { - "controlpath": ["%C", "%h", "%l", "%L", "%n", "%p", "%r", "%u"], - "hostname": ["%h"], - "identityfile": ["%C", "~", "%d", "%h", "%l", "%u", "%r"], - "proxycommand": ["~", "%h", "%p", "%r"], - "proxyjump": ["%h", "%p", "%r"], - # Doesn't seem worth making this 'special' for now, it will fit well - # enough (no actual match-exec config key to be confused with). - "match-exec": ["%C", "%d", "%h", "%L", "%l", "%n", "%p", "%r", "%u"], - } - - def __init__(self): - """ - Create a new OpenSSH config object. - - Note: the newer alternate constructors `from_path`, `from_file` and - `from_text` are simpler to use, as they parse on instantiation. For - example, instead of:: - - config = SSHConfig() - config.parse(open("some-path.config") - - you could:: - - config = SSHConfig.from_file(open("some-path.config")) - # Or more directly: - config = SSHConfig.from_path("some-path.config") - # Or if you have arbitrary ssh_config text from some other source: - config = SSHConfig.from_text("Host foo\\n\\tUser bar") - """ - self._config = [] - - @classmethod - def from_text(cls, text): - """ - Create a new, parsed `SSHConfig` from ``text`` string. - - .. versionadded:: 2.7 - """ - return cls.from_file(StringIO(text)) - - @classmethod - def from_path(cls, path): - """ - Create a new, parsed `SSHConfig` from the file found at ``path``. - - .. versionadded:: 2.7 - """ - with open(path) as flo: - return cls.from_file(flo) - - @classmethod - def from_file(cls, flo): - """ - Create a new, parsed `SSHConfig` from file-like object ``flo``. - - .. versionadded:: 2.7 - """ - obj = cls() - obj.parse(flo) - return obj - - def parse(self, file_obj): - """ - Read an OpenSSH config from the given file object. - - :param file_obj: a file-like object to read the config file from - """ - # Start out w/ implicit/anonymous global host-like block to hold - # anything not contained by an explicit one. - context = {"host": ["*"], "config": {}} - for line in file_obj: - # Strip any leading or trailing whitespace from the line. - # Refer to https://github.com/paramiko/paramiko/issues/499 - line = line.strip() - # Skip blanks, comments - if not line or line.startswith("#"): - continue - - # Parse line into key, value - match = re.match(self.SETTINGS_REGEX, line) - if not match: - raise ConfigParseError("Unparsable line {}".format(line)) - key = match.group(1).lower() - value = match.group(2) - - # Host keyword triggers switch to new block/context - if key in ("host", "match"): - self._config.append(context) - context = {"config": {}} - if key == "host": - # TODO 4.0: make these real objects or at least name this - # "hosts" to acknowledge it's an iterable. (Doing so prior - # to 3.0, despite it being a private API, feels bad - - # surely such an old codebase has folks actually relying on - # these keys.) - context["host"] = self._get_hosts(value) - else: - context["matches"] = self._get_matches(value) - # Special-case for noop ProxyCommands - elif key == "proxycommand" and value.lower() == "none": - # Store 'none' as None - not as a string implying that the - # proxycommand is the literal shell command "none"! - context["config"][key] = None - # All other keywords get stored, directly or via append - else: - if value.startswith('"') and value.endswith('"'): - value = value[1:-1] - - # identityfile, localforward, remoteforward keys are special - # cases, since they are allowed to be specified multiple times - # and they should be tried in order of specification. - if key in ["identityfile", "localforward", "remoteforward"]: - if key in context["config"]: - context["config"][key].append(value) - else: - context["config"][key] = [value] - elif key not in context["config"]: - context["config"][key] = value - # Store last 'open' block and we're done - self._config.append(context) - - def lookup(self, hostname): - """ - Return a dict (`SSHConfigDict`) of config options for a given hostname. - - The host-matching rules of OpenSSH's ``ssh_config`` man page are used: - For each parameter, the first obtained value will be used. The - configuration files contain sections separated by ``Host`` and/or - ``Match`` specifications, and that section is only applied for hosts - which match the given patterns or keywords - - Since the first obtained value for each parameter is used, more host- - specific declarations should be given near the beginning of the file, - and general defaults at the end. - - The keys in the returned dict are all normalized to lowercase (look for - ``"port"``, not ``"Port"``. The values are processed according to the - rules for substitution variable expansion in ``ssh_config``. - - Finally, please see the docs for `SSHConfigDict` for deeper info on - features such as optional type conversion methods, e.g.:: - - conf = my_config.lookup('myhost') - assert conf['passwordauthentication'] == 'yes' - assert conf.as_bool('passwordauthentication') is True - - .. note:: - If there is no explicitly configured ``HostName`` value, it will be - set to the being-looked-up hostname, which is as close as we can - get to OpenSSH's behavior around that particular option. - - :param str hostname: the hostname to lookup - - .. versionchanged:: 2.5 - Returns `SSHConfigDict` objects instead of dict literals. - .. versionchanged:: 2.7 - Added canonicalization support. - .. versionchanged:: 2.7 - Added ``Match`` support. - """ - # First pass - options = self._lookup(hostname=hostname) - # Inject HostName if it was not set (this used to be done incidentally - # during tokenization, for some reason). - if "hostname" not in options: - options["hostname"] = hostname - # Handle canonicalization - canon = options.get("canonicalizehostname", None) in ("yes", "always") - maxdots = int(options.get("canonicalizemaxdots", 1)) - if canon and hostname.count(".") <= maxdots: - # NOTE: OpenSSH manpage does not explicitly state this, but its - # implementation for CanonicalDomains is 'split on any whitespace'. - domains = options["canonicaldomains"].split() - hostname = self.canonicalize(hostname, options, domains) - # Overwrite HostName again here (this is also what OpenSSH does) - options["hostname"] = hostname - options = self._lookup(hostname, options, canonical=True) - return options - - def _lookup(self, hostname, options=None, canonical=False): - # Init - if options is None: - options = SSHConfigDict() - # Iterate all stanzas, applying any that match, in turn (so that things - # like Match can reference currently understood state) - for context in self._config: - if not ( - self._pattern_matches(context.get("host", []), hostname) - or self._does_match( - context.get("matches", []), hostname, canonical, options - ) - ): - continue - for key, value in context["config"].items(): - if key not in options: - # Create a copy of the original value, - # else it will reference the original list - # in self._config and update that value too - # when the extend() is being called. - options[key] = value[:] if value is not None else value - elif key == "identityfile": - options[key].extend( - x for x in value if x not in options[key] - ) - # Expand variables in resulting values (besides 'Match exec' which was - # already handled above) - options = self._expand_variables(options, hostname) - return options - - def canonicalize(self, hostname, options, domains): - """ - Return canonicalized version of ``hostname``. - - :param str hostname: Target hostname. - :param options: An `SSHConfigDict` from a previous lookup pass. - :param domains: List of domains (e.g. ``["paramiko.org"]``). - - :returns: A canonicalized hostname if one was found, else ``None``. - - .. versionadded:: 2.7 - """ - found = False - for domain in domains: - candidate = "{}.{}".format(hostname, domain) - family_specific = _addressfamily_host_lookup(candidate, options) - if family_specific is not None: - # TODO: would we want to dig deeper into other results? e.g. to - # find something that satisfies PermittedCNAMEs when that is - # implemented? - found = family_specific[0] - else: - # TODO: what does ssh use here and is there a reason to use - # that instead of gethostbyname? - try: - found = socket.gethostbyname(candidate) - except socket.gaierror: - pass - if found: - # TODO: follow CNAME (implied by found != candidate?) if - # CanonicalizePermittedCNAMEs allows it - return candidate - # If we got here, it means canonicalization failed. - # When CanonicalizeFallbackLocal is undefined or 'yes', we just spit - # back the original hostname. - if options.get("canonicalizefallbacklocal", "yes") == "yes": - return hostname - # And here, we failed AND fallback was set to a non-yes value, so we - # need to get mad. - raise CouldNotCanonicalize(hostname) - - def get_hostnames(self): - """ - Return the set of literal hostnames defined in the SSH config (both - explicit hostnames and wildcard entries). - """ - hosts = set() - for entry in self._config: - hosts.update(entry["host"]) - return hosts - - def _pattern_matches(self, patterns, target): - # Convenience auto-splitter if not already a list - if hasattr(patterns, "split"): - patterns = patterns.split(",") - match = False - for pattern in patterns: - # Short-circuit if target matches a negated pattern - if pattern.startswith("!") and fnmatch.fnmatch( - target, pattern[1:] - ): - return False - # Flag a match, but continue (in case of later negation) if regular - # match occurs - elif fnmatch.fnmatch(target, pattern): - match = True - return match - - def _does_match(self, match_list, target_hostname, canonical, options): - matched = [] - candidates = match_list[:] - local_username = getpass.getuser() - while candidates: - candidate = candidates.pop(0) - passed = None - # Obtain latest host/user value every loop, so later Match may - # reference values assigned within a prior Match. - configured_host = options.get("hostname", None) - configured_user = options.get("user", None) - type_, param = candidate["type"], candidate["param"] - # Canonical is a hard pass/fail based on whether this is a - # canonicalized re-lookup. - if type_ == "canonical": - if self._should_fail(canonical, candidate): - return False - # The parse step ensures we only see this by itself or after - # canonical, so it's also an easy hard pass. (No negation here as - # that would be uh, pretty weird?) - elif type_ == "all": - return True - # From here, we are testing various non-hard criteria, - # short-circuiting only on fail - elif type_ == "host": - hostval = configured_host or target_hostname - passed = self._pattern_matches(param, hostval) - elif type_ == "originalhost": - passed = self._pattern_matches(param, target_hostname) - elif type_ == "user": - user = configured_user or local_username - passed = self._pattern_matches(param, user) - elif type_ == "localuser": - passed = self._pattern_matches(param, local_username) - elif type_ == "exec": - exec_cmd = self._tokenize( - options, target_hostname, "match-exec", param - ) - # This is the laziest spot in which we can get mad about an - # inability to import Invoke. - if invoke is None: - raise invoke_import_error - # Like OpenSSH, we 'redirect' stdout but let stderr bubble up - passed = invoke.run(exec_cmd, hide="stdout", warn=True).ok - # Tackle any 'passed, but was negated' results from above - if passed is not None and self._should_fail(passed, candidate): - return False - # Made it all the way here? Everything matched! - matched.append(candidate) - # Did anything match? (To be treated as bool, usually.) - return matched - - def _should_fail(self, would_pass, candidate): - return would_pass if candidate["negate"] else not would_pass - - def _tokenize(self, config, target_hostname, key, value): - """ - Tokenize a string based on current config/hostname data. - - :param config: Current config data. - :param target_hostname: Original target connection hostname. - :param key: Config key being tokenized (used to filter token list). - :param value: Config value being tokenized. - - :returns: The tokenized version of the input ``value`` string. - """ - allowed_tokens = self._allowed_tokens(key) - # Short-circuit if no tokenization possible - if not allowed_tokens: - return value - # Obtain potentially configured hostname, for use with %h. - # Special-case where we are tokenizing the hostname itself, to avoid - # replacing %h with a %h-bearing value, etc. - configured_hostname = target_hostname - if key != "hostname": - configured_hostname = config.get("hostname", configured_hostname) - # Ditto the rest of the source values - if "port" in config: - port = config["port"] - else: - port = SSH_PORT - user = getpass.getuser() - if "user" in config: - remoteuser = config["user"] - else: - remoteuser = user - local_hostname = socket.gethostname().split(".")[0] - local_fqdn = LazyFqdn(config, local_hostname) - homedir = os.path.expanduser("~") - tohash = local_hostname + target_hostname + repr(port) + remoteuser - # The actual tokens! - replacements = { - # TODO: %%??? - # "%C": sha1(tohash.encode()).hexdigest(), # jc_change - # "%d": homedir, # jc_change - "%h": configured_hostname, - # TODO: %i? - # "%L": local_hostname, # jc_change - # "%l": local_fqdn, # jc_change - # also this is pseudo buggy when not in Match exec mode so document - # that. also WHY is that the case?? don't we do all of this late? - "%n": target_hostname, - "%p": port, - "%r": remoteuser, - # TODO: %T? don't believe this is possible however - # "%u": user, # jc_change - # "~": homedir, # jc_change - } - # Do the thing with the stuff - tokenized = value - for find, replace in replacements.items(): - if find not in allowed_tokens: - continue - tokenized = tokenized.replace(find, str(replace)) - # TODO: log? eg that value -> tokenized - return tokenized - - def _allowed_tokens(self, key): - """ - Given config ``key``, return list of token strings to tokenize. - - .. note:: - This feels like it wants to eventually go away, but is used to - preserve as-strict-as-possible compatibility with OpenSSH, which - for whatever reason only applies some tokens to some config keys. - """ - return self.TOKENS_BY_CONFIG_KEY.get(key, []) - - def _expand_variables(self, config, target_hostname): - """ - Return a dict of config options with expanded substitutions - for a given original & current target hostname. - - Please refer to :doc:`/api/config` for details. - - :param dict config: the currently parsed config - :param str hostname: the hostname whose config is being looked up - """ - for k in config: - if config[k] is None: - continue - tokenizer = partial(self._tokenize, config, target_hostname, k) - if isinstance(config[k], list): - for i, value in enumerate(config[k]): - config[k][i] = tokenizer(value) - else: - config[k] = tokenizer(config[k]) - return config - - def _get_hosts(self, host): - """ - Return a list of host_names from host value. - """ - try: - return shlex.split(host) - except ValueError: - raise ConfigParseError("Unparsable host {}".format(host)) - - def _get_matches(self, match): - """ - Parse a specific Match config line into a list-of-dicts for its values. - - Performs some parse-time validation as well. - """ - matches = [] - tokens = shlex.split(match) - while tokens: - match = {"type": None, "param": None, "negate": False} - type_ = tokens.pop(0) - # Handle per-keyword negation - if type_.startswith("!"): - match["negate"] = True - type_ = type_[1:] - match["type"] = type_ - # all/canonical have no params (everything else does) - if type_ in ("all", "canonical"): - matches.append(match) - continue - if not tokens: - raise ConfigParseError( - "Missing parameter to Match '{}' keyword".format(type_) - ) - match["param"] = tokens.pop(0) - matches.append(match) - # Perform some (easier to do now than in the middle) validation that is - # better handled here than at lookup time. - keywords = [x["type"] for x in matches] - if "all" in keywords: - allowable = ("all", "canonical") - ok, bad = ( - list(filter(lambda x: x in allowable, keywords)), - list(filter(lambda x: x not in allowable, keywords)), - ) - err = None - if any(bad): - err = "Match does not allow 'all' mixed with anything but 'canonical'" # noqa - elif "canonical" in ok and ok.index("canonical") > ok.index("all"): - err = "Match does not allow 'all' before 'canonical'" - if err is not None: - raise ConfigParseError(err) - return matches - - -def _addressfamily_host_lookup(hostname, options): - """ - Try looking up ``hostname`` in an IPv4 or IPv6 specific manner. - - This is an odd duck due to needing use in two divergent use cases. It looks - up ``AddressFamily`` in ``options`` and if it is ``inet`` or ``inet6``, - this function uses `socket.getaddrinfo` to perform a family-specific - lookup, returning the result if successful. - - In any other situation -- lookup failure, or ``AddressFamily`` being - unspecified or ``any`` -- ``None`` is returned instead and the caller is - expected to do something situation-appropriate like calling - `socket.gethostbyname`. - - :param str hostname: Hostname to look up. - :param options: `SSHConfigDict` instance w/ parsed options. - :returns: ``getaddrinfo``-style tuples, or ``None``, depending. - """ - address_family = options.get("addressfamily", "any").lower() - if address_family == "any": - return - try: - family = socket.AF_INET6 - if address_family == "inet": - family = socket.AF_INET - return socket.getaddrinfo( - hostname, - None, - family, - socket.SOCK_DGRAM, - socket.IPPROTO_IP, - socket.AI_CANONNAME, - ) - except socket.gaierror: - pass - - -class LazyFqdn: - """ - Returns the host's fqdn on request as string. - """ - - def __init__(self, config, host=None): - self.fqdn = None - self.config = config - self.host = host - - def __str__(self): - if self.fqdn is None: - # - # If the SSH config contains AddressFamily, use that when - # determining the local host's FQDN. Using socket.getfqdn() from - # the standard library is the most general solution, but can - # result in noticeable delays on some platforms when IPv6 is - # misconfigured or not available, as it calls getaddrinfo with no - # address family specified, so both IPv4 and IPv6 are checked. - # - - # Handle specific option - fqdn = None - results = _addressfamily_host_lookup(self.host, self.config) - if results is not None: - for res in results: - af, socktype, proto, canonname, sa = res - if canonname and "." in canonname: - fqdn = canonname - break - # Handle 'any' / unspecified / lookup failure - if fqdn is None: - fqdn = socket.getfqdn() - # Cache - self.fqdn = fqdn - return self.fqdn - - -class SSHConfigDict(dict): - """ - A dictionary wrapper/subclass for per-host configuration structures. - - This class introduces some usage niceties for consumers of `SSHConfig`, - specifically around the issue of variable type conversions: normal value - access yields strings, but there are now methods such as `as_bool` and - `as_int` that yield casted values instead. - - For example, given the following ``ssh_config`` file snippet:: - - Host foo.example.com - PasswordAuthentication no - Compression yes - ServerAliveInterval 60 - - the following code highlights how you can access the raw strings as well as - usefully Python type-casted versions (recalling that keys are all - normalized to lowercase first):: - - my_config = SSHConfig() - my_config.parse(open('~/.ssh/config')) - conf = my_config.lookup('foo.example.com') - - assert conf['passwordauthentication'] == 'no' - assert conf.as_bool('passwordauthentication') is False - assert conf['compression'] == 'yes' - assert conf.as_bool('compression') is True - assert conf['serveraliveinterval'] == '60' - assert conf.as_int('serveraliveinterval') == 60 - - .. versionadded:: 2.5 - """ - - def as_bool(self, key): - """ - Express given key's value as a boolean type. - - Typically, this is used for ``ssh_config``'s pseudo-boolean values - which are either ``"yes"`` or ``"no"``. In such cases, ``"yes"`` yields - ``True`` and any other value becomes ``False``. - - .. note:: - If (for whatever reason) the stored value is already boolean in - nature, it's simply returned. - - .. versionadded:: 2.5 - """ - val = self[key] - if isinstance(val, bool): - return val - return val.lower() == "yes" - - def as_int(self, key): - """ - Express given key's value as an integer, if possible. - - This method will raise ``ValueError`` or similar if the value is not - int-appropriate, same as the builtin `int` type. - - .. versionadded:: 2.5 - """ - return int(self[key]) diff --git a/jc/parsers/paramiko/ssh_exception.py b/jc/parsers/paramiko/ssh_exception.py deleted file mode 100644 index 9b1b44c3..00000000 --- a/jc/parsers/paramiko/ssh_exception.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import socket - - -class SSHException(Exception): - """ - Exception raised by failures in SSH2 protocol negotiation or logic errors. - """ - - pass - - -class AuthenticationException(SSHException): - """ - Exception raised when authentication failed for some reason. It may be - possible to retry with different credentials. (Other classes specify more - specific reasons.) - - .. versionadded:: 1.6 - """ - - pass - - -class PasswordRequiredException(AuthenticationException): - """ - Exception raised when a password is needed to unlock a private key file. - """ - - pass - - -class BadAuthenticationType(AuthenticationException): - """ - Exception raised when an authentication type (like password) is used, but - the server isn't allowing that type. (It may only allow public-key, for - example.) - - .. versionadded:: 1.1 - """ - - allowed_types = [] - - # TODO 4.0: remove explanation kwarg - def __init__(self, explanation, types): - # TODO 4.0: remove this supercall unless it's actually required for - # pickling (after fixing pickling) - AuthenticationException.__init__(self, explanation, types) - self.explanation = explanation - self.allowed_types = types - - def __str__(self): - return "{}; allowed types: {!r}".format( - self.explanation, self.allowed_types - ) - - -class PartialAuthentication(AuthenticationException): - """ - An internal exception thrown in the case of partial authentication. - """ - - allowed_types = [] - - def __init__(self, types): - AuthenticationException.__init__(self, types) - self.allowed_types = types - - def __str__(self): - return "Partial authentication; allowed types: {!r}".format( - self.allowed_types - ) - - -class ChannelException(SSHException): - """ - Exception raised when an attempt to open a new `.Channel` fails. - - :param int code: the error code returned by the server - - .. versionadded:: 1.6 - """ - - def __init__(self, code, text): - SSHException.__init__(self, code, text) - self.code = code - self.text = text - - def __str__(self): - return "ChannelException({!r}, {!r})".format(self.code, self.text) - - -class BadHostKeyException(SSHException): - """ - The host key given by the SSH server did not match what we were expecting. - - :param str hostname: the hostname of the SSH server - :param PKey got_key: the host key presented by the server - :param PKey expected_key: the host key expected - - .. versionadded:: 1.6 - """ - - def __init__(self, hostname, got_key, expected_key): - SSHException.__init__(self, hostname, got_key, expected_key) - self.hostname = hostname - self.key = got_key - self.expected_key = expected_key - - def __str__(self): - msg = "Host key for server '{}' does not match: got '{}', expected '{}'" # noqa - return msg.format( - self.hostname, - self.key.get_base64(), - self.expected_key.get_base64(), - ) - - -class IncompatiblePeer(SSHException): - """ - A disagreement arose regarding an algorithm required for key exchange. - - .. versionadded:: 2.9 - """ - - # TODO 4.0: consider making this annotate w/ 1..N 'missing' algorithms, - # either just the first one that would halt kex, or even updating the - # Transport logic so we record /all/ that /could/ halt kex. - # TODO: update docstrings where this may end up raised so they are more - # specific. - pass - - -class ProxyCommandFailure(SSHException): - """ - The "ProxyCommand" found in the .ssh/config file returned an error. - - :param str command: The command line that is generating this exception. - :param str error: The error captured from the proxy command output. - """ - - def __init__(self, command, error): - SSHException.__init__(self, command, error) - self.command = command - self.error = error - - def __str__(self): - return 'ProxyCommand("{}") returned nonzero exit status: {}'.format( - self.command, self.error - ) - - -class NoValidConnectionsError(socket.error): - """ - Multiple connection attempts were made and no families succeeded. - - This exception class wraps multiple "real" underlying connection errors, - all of which represent failed connection attempts. Because these errors are - not guaranteed to all be of the same error type (i.e. different errno, - `socket.error` subclass, message, etc) we expose a single unified error - message and a ``None`` errno so that instances of this class match most - normal handling of `socket.error` objects. - - To see the wrapped exception objects, access the ``errors`` attribute. - ``errors`` is a dict whose keys are address tuples (e.g. ``('127.0.0.1', - 22)``) and whose values are the exception encountered trying to connect to - that address. - - It is implied/assumed that all the errors given to a single instance of - this class are from connecting to the same hostname + port (and thus that - the differences are in the resolution of the hostname - e.g. IPv4 vs v6). - - .. versionadded:: 1.16 - """ - - def __init__(self, errors): - """ - :param dict errors: - The errors dict to store, as described by class docstring. - """ - addrs = sorted(errors.keys()) - body = ", ".join([x[0] for x in addrs[:-1]]) - tail = addrs[-1][0] - if body: - msg = "Unable to connect to port {0} on {1} or {2}" - else: - msg = "Unable to connect to port {0} on {2}" - super().__init__( - None, msg.format(addrs[0][1], body, tail) # stand-in for errno - ) - self.errors = errors - - def __reduce__(self): - return (self.__class__, (self.errors,)) - - -class CouldNotCanonicalize(SSHException): - """ - Raised when hostname canonicalization fails & fallback is disabled. - - .. versionadded:: 2.7 - """ - - pass - - -class ConfigParseError(SSHException): - """ - A fatal error was encountered trying to parse SSH config data. - - Typically this means a config file violated the ``ssh_config`` - specification in a manner that requires exiting immediately, such as not - matching ``key = value`` syntax or misusing certain ``Match`` keywords. - - .. versionadded:: 2.7 - """ - - pass diff --git a/jc/parsers/ssh_conf.py b/jc/parsers/ssh_conf.py index 80b5c727..1e9883f5 100644 --- a/jc/parsers/ssh_conf.py +++ b/jc/parsers/ssh_conf.py @@ -1,47 +1,493 @@ -"""jc - JSON Convert ssh configuration file parser +"""jc - JSON Convert ssh configuration file and `ssh -G` command output parser + +This parser will work with `ssh` configuration files or the output of +`ssh -G`. Any `Match` blocks in the `ssh` configuration file will be +ignored. Usage (cli): - $ cat ssh_conf | jc --ssh-conf + $ ssh -G hostname | jc --ssh-conf + +or + + $ jc ssh -G hostname + +or + + $ cat ~/.ssh/config | jc --ssh-conf Usage (module): import jc - result = jc.parse('ssh_conf', ssh_conf_file_output) + result = jc.parse('ssh_conf', ssh_conf_output) Schema: [ { - "ssh_conf": string, - "bar": boolean, - "baz": integer + "host": string, + "addkeystoagent": string, + "addressfamily": string, + "batchmode": string, + "bindaddress": string, + "bindinterface": string, + "canonicaldomains": [ + string + ], + "canonicalizefallbacklocal": string, + "canonicalizehostname": string, + "canonicalizemaxdots": integer, + "canonicalizepermittedcnames": [ + string + ], + "casignaturealgorithms": [ + string + ], + "certificatefile": [ + string + ], + "checkhostip": string, + "ciphers": [ + string + ], + "clearallforwardings": string, + "compression": string, + "connectionattempts": integer, + "connecttimeout": integer, + "controlmaster": string, + "controlpath": string, + "controlpersist": string, + "dynamicforward": string, + "enableescapecommandline": string, + "enablesshkeysign": string, + "escapechar": string, + "exitonforwardfailure": string, + "fingerprinthash": string, + "forkafterauthentication": string, + "forwardagent": string, + "forwardx11": string, + "forwardx11timeout": integer, + "forwardx11trusted": string, + "gatewayports": string, + "globalknownhostsfile": [ + string + ], + "gssapiauthentication": string, + "gssapidelegatecredentials": string, + "hashknownhosts": string, + "hostbasedacceptedalgorithms": [ + string + ], + "hostbasedauthentication": string, + "hostkeyalgorithms": [ + string + ], + "hostkeyalias": string, + "hostname": string, + "identitiesonly": string, + "identityagent": string, + "identityfile": [ + string + ], + "ignoreunknown": string, + "include": [ + string + ], + "ipqos": [ + string + ], + "kbdinteractiveauthentication": string, + "kbdinteractivedevices": [ + string + ], + "kexalgorithms": [ + string + ], + "kexalgorithms_strategy": string, + "knownhostscommand": string, + "localcommand": string, + "localforward": [ + string + ], + "loglevel": string, + "logverbose": [ + string + ], + "macs": [ + string + ], + "macs_strategy": string, + "nohostauthenticationforlocalhost": string, + "numberofpasswordprompts": integer, + "passwordauthentication": string, + "permitlocalcommand": string, + "permitremoteopen": [ + string + ], + "pkcs11provider": string, + "port": integer, + "preferredauthentications": [ + string + ], + "protocol": integer, + "proxycommand": string, + "proxyjump": [ + string + ], + "proxyusefdpass": string, + "pubkeyacceptedalgorithms": [ + string + ], + "pubkeyacceptedalgorithms_strategy": string, + "pubkeyauthentication": string, + "rekeylimit": string, + "remotecommand": string, + "remoteforward": string, + "requesttty": string, + "requiredrsasize": integer, + "revokedhostkeys": string, + "securitykeyprovider": string, + "sendenv": [ + string + ], + "serveralivecountmax": integer, + "serveraliveinterval": integer, + "sessiontype": string, + "setenv": [ + string + ], + "stdinnull": string, + "streamlocalbindmask": string, + "streamlocalbindunlink": string, + "stricthostkeychecking": string, + "syslogfacility": string, + "tcpkeepalive": string, + "tunnel": string, + "tunneldevice": string, + "updatehostkeys": string, + "user": string, + "userknownhostsfile": [ + string + ], + "verifyhostkeydns": string, + "visualhostkey": string, + "xauthlocation": string } ] Examples: - $ cat ssh_conf | jc --ssh-conf -p - [] + $ ssh -G - | jc --ssh-conf -p + [ + { + "user": "foo", + "hostname": "-", + "port": 22, + "addressfamily": "any", + "batchmode": "no", + "canonicalizefallbacklocal": "yes", + "canonicalizehostname": "false", + "checkhostip": "no", + "compression": "no", + "controlmaster": "false", + "enablesshkeysign": "no", + "clearallforwardings": "no", + "exitonforwardfailure": "no", + "fingerprinthash": "SHA256", + "forwardx11": "no", + "forwardx11trusted": "no", + "gatewayports": "no", + "gssapiauthentication": "no", + "gssapidelegatecredentials": "no", + "hashknownhosts": "no", + "hostbasedauthentication": "no", + "identitiesonly": "no", + "kbdinteractiveauthentication": "yes", + "nohostauthenticationforlocalhost": "no", + "passwordauthentication": "yes", + "permitlocalcommand": "no", + "proxyusefdpass": "no", + "pubkeyauthentication": "true", + "requesttty": "auto", + "sessiontype": "default", + "stdinnull": "no", + "forkafterauthentication": "no", + "streamlocalbindunlink": "no", + "stricthostkeychecking": "ask", + "tcpkeepalive": "yes", + "tunnel": "false", + "verifyhostkeydns": "false", + "visualhostkey": "no", + "updatehostkeys": "true", + "applemultipath": "no", + "canonicalizemaxdots": 1, + "connectionattempts": 1, + "forwardx11timeout": 1200, + "numberofpasswordprompts": 3, + "serveralivecountmax": 3, + "serveraliveinterval": 0, + "ciphers": [ + "chacha20-poly1305@openssh.com", + "aes128-ctr", + "aes192-ctr", + "aes256-ctr", + "aes128-gcm@openssh.com", + "aes256-gcm@openssh.com" + ], + "hostkeyalgorithms": [ + "ssh-ed25519-cert-v01@openssh.com", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521-cert-v01@openssh.com", + "rsa-sha2-512-cert-v01@openssh.com", + "rsa-sha2-256-cert-v01@openssh.com", + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "hostbasedacceptedalgorithms": [ + "ssh-ed25519-cert-v01@openssh.com", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521-cert-v01@openssh.com", + "rsa-sha2-512-cert-v01@openssh.com", + "rsa-sha2-256-cert-v01@openssh.com", + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "kexalgorithms": [ + "sntrup761x25519-sha512@openssh.com", + "curve25519-sha256", + "curve25519-sha256@libssh.org", + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "ecdh-sha2-nistp521", + "diffie-hellman-group-exchange-sha256", + "diffie-hellman-group16-sha512", + "diffie-hellman-group18-sha512", + "diffie-hellman-group14-sha256" + ], + "casignaturealgorithms": [ + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "loglevel": "INFO", + "macs": [ + "umac-64-etm@openssh.com", + "umac-128-etm@openssh.com", + "hmac-sha2-256-etm@openssh.com", + "hmac-sha2-512-etm@openssh.com", + "hmac-sha1-etm@openssh.com", + "umac-64@openssh.com", + "umac-128@openssh.com", + "hmac-sha2-256", + "hmac-sha2-512", + "hmac-sha1" + ], + "securitykeyprovider": "$SSH_SK_PROVIDER", + "pubkeyacceptedalgorithms": [ + "ssh-ed25519-cert-v01@openssh.com", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521-cert-v01@openssh.com", + "rsa-sha2-512-cert-v01@openssh.com", + "rsa-sha2-256-cert-v01@openssh.com", + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "xauthlocation": "/usr/X11R6/bin/xauth", + "identityfile": [ + "~/.ssh/id_rsa", + "~/.ssh/id_ecdsa", + "~/.ssh/id_ecdsa_sk", + "~/.ssh/id_ed25519", + "~/.ssh/id_ed25519_sk", + "~/.ssh/id_xmss", + "~/.ssh/id_dsa" + ], + "canonicaldomains": [ + "none" + ], + "globalknownhostsfile": [ + "/etc/ssh/ssh_known_hosts", + "/etc/ssh/ssh_known_hosts2" + ], + "userknownhostsfile": [ + "/Users/foo/.ssh/known_hosts", + "/Users/foo/.ssh/known_hosts2" + ], + "sendenv": [ + "LANG", + "LC_*" + ], + "logverbose": [ + "none" + ], + "permitremoteopen": [ + "any" + ], + "addkeystoagent": "false", + "forwardagent": "no", + "connecttimeout": null, + "tunneldevice": "any:any", + "canonicalizepermittedcnames": [ + "none" + ], + "controlpersist": "no", + "escapechar": "~", + "ipqos": [ + "af21", + "cs1" + ], + "rekeylimit": "0 0", + "streamlocalbindmask": "0177", + "syslogfacility": "USER" + } + ] - $ cat ssh_conf | jc --ssh-conf -p -r - [] + $ cat ~/.ssh/config | jc --ssh-conf -p + [ + { + "host": "server1", + "hostname": "server1.cyberciti.biz", + "user": "nixcraft", + "port": 4242, + "identityfile": [ + "/nfs/shared/users/nixcraft/keys/server1/id_rsa" + ] + }, + { + "host": "nas01", + "hostname": "192.168.1.100", + "user": "root", + "identityfile": [ + "~/.ssh/nas01.key" + ] + }, + { + "host": "aws.apache", + "hostname": "1.2.3.4", + "user": "wwwdata", + "identityfile": [ + "~/.ssh/aws.apache.key" + ] + }, + { + "host": "uk.gw.lan uk.lan", + "hostname": "192.168.0.251", + "user": "nixcraft", + "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" + }, + { + "host": "proxyus", + "hostname": "vps1.cyberciti.biz", + "user": "breakfree", + "identityfile": [ + "~/.ssh/vps1.cyberciti.biz.key" + ], + "localforward": [ + "3128 127.0.0.1:3128" + ] + }, + { + "host": "*", + "forwardagent": "no", + "forwardx11": "no", + "forwardx11trusted": "yes", + "user": "nixcraft", + "port": 22, + "protocol": 2, + "serveraliveinterval": 60, + "serveralivecountmax": 30 + } + ] + + $ cat ~/.ssh/config | jc --ssh-conf -p -r + [ + { + "host": "server1", + "hostname": "server1.cyberciti.biz", + "user": "nixcraft", + "port": "4242", + "identityfile": [ + "/nfs/shared/users/nixcraft/keys/server1/id_rsa" + ] + }, + { + "host": "nas01", + "hostname": "192.168.1.100", + "user": "root", + "identityfile": [ + "~/.ssh/nas01.key" + ] + }, + { + "host": "aws.apache", + "hostname": "1.2.3.4", + "user": "wwwdata", + "identityfile": [ + "~/.ssh/aws.apache.key" + ] + }, + { + "host": "uk.gw.lan uk.lan", + "hostname": "192.168.0.251", + "user": "nixcraft", + "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" + }, + { + "host": "proxyus", + "hostname": "vps1.cyberciti.biz", + "user": "breakfree", + "identityfile": [ + "~/.ssh/vps1.cyberciti.biz.key" + ], + "localforward": [ + "3128 127.0.0.1:3128" + ] + }, + { + "host": "*", + "forwardagent": "no", + "forwardx11": "no", + "forwardx11trusted": "yes", + "user": "nixcraft", + "port": "22", + "protocol": "2", + "serveraliveinterval": "60", + "serveralivecountmax": "30" + } + ] """ -from typing import List, Dict +from typing import Set, List, Dict from jc.jc_types import JSONDictType import jc.utils -from .paramiko.config import SSHConfig as sshconfig class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = 'ssh config file parser' + description = 'ssh config file and `ssh -G` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - details = 'Using Paramiko library at https://github.com/paramiko/paramiko.' - compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - tags = ['file'] + compatible = ['linux', 'darwin', 'freebsd'] + magic_commands = ['ssh -G'] + tags = ['command', 'file'] __version__ = info.version @@ -59,6 +505,47 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: List of Dictionaries. Structured to conform to the schema. """ + split_fields_space: Set[str] = { + 'canonicaldomains', 'globalknownhostsfile', 'include', 'ipqos', + 'permitremoteopen', 'sendenv', 'setenv', 'userknownhostsfile' + } + + split_fields_comma: Set[str] = { + 'canonicalizepermittedcnames', 'casignaturealgorithms', 'ciphers', + 'hostbasedacceptedalgorithms', 'hostkeyalgorithms', + 'kbdinteractivedevices', 'kexalgorithms', 'logverbose', 'macs', + 'preferredauthentications', 'proxyjump', 'pubkeyacceptedalgorithms' + } + + int_list: Set[str] = { + 'canonicalizemaxdots', 'connectionattempts', 'connecttimeout', + 'forwardx11timeout', 'numberofpasswordprompts', 'port', 'protocol', + 'requiredrsasize', 'serveralivecountmax', 'serveraliveinterval' + } + + for host in proc_data: + dict_copy = host.copy() + for key, val in dict_copy.items(): + # these are list values + if key == 'sendenv' or key == 'setenv' or key == 'include': + new_list: List[str] = [] + for item in val: + new_list.extend(item.split()) + host[key] = new_list + continue + + if key in split_fields_space: + host[key] = val.split() + continue + + if key in split_fields_comma: + host[key] = val.split(',') + continue + + for key, val in host.items(): + if key in int_list: + host[key] = jc.utils.convert_to_int(val) + return proc_data @@ -83,13 +570,73 @@ def parse( jc.utils.compatibility(__name__, info.compatible, quiet) jc.utils.input_type_check(data) - raw_output: List[Dict] = [] + raw_output: List = [] + host: Dict = {} + + multi_fields: Set[str] = { + 'certificatefile', 'identityfile', 'include', 'localforward', + 'sendenv', 'setenv' + } + + modified_fields: Set[str] = { + 'casignaturealgorithms', 'ciphers', 'hostbasedacceptedalgorithms', + 'HostKeyAlgorithms', 'kexalgorithms', 'macs', + 'pubkeyacceptedalgorithms' + } + + modifiers: Set[str] = {'+', '-', '^'} + + match_block_found = False if jc.utils.has_data(data): - myconfig = sshconfig.from_text(data) - hostnames = sorted(myconfig.get_hostnames()) - raw_output = [myconfig.lookup(x) for x in hostnames] - for host, obj in zip(hostnames, raw_output.copy()): - obj.update({'host': host}) + + for line in filter(None, data.splitlines()): + # skip any lines with only whitespace + if not line.strip(): + continue + + # support configuration file by skipping commented lines + if line.strip().startswith('#'): + continue + + if line.strip().startswith('Host '): + if host: + raw_output.append(host) + host = {'host': line.split()[1]} + + # support configuration file by ignoring all lines between + # Match xxx and Match any + if line.strip().startswith('Match all'): + match_block_found = False + continue + + if line.strip().startswith('Match'): + match_block_found = True + continue + + if match_block_found: + continue + + key, val = line.split(maxsplit=1) + + # support configuration file by converting to lower case + key = key.lower() + + if key in multi_fields: + if key not in host: + host[key] = [] + host[key].append(val) + continue + + if key in modified_fields and val[0] in modifiers: + host[key] = val[1:] + host[key + '_strategy'] = val[0] + continue + + host[key] = val + continue + + if host: + raw_output.append(host) return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/sshd_conf.py b/jc/parsers/sshd_conf.py index b6a19e64..62782ade 100644 --- a/jc/parsers/sshd_conf.py +++ b/jc/parsers/sshd_conf.py @@ -483,13 +483,13 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = 'sshd config file and `sshd -T` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['sshd -T'] - tags = ['file'] + tags = ['command', 'file'] __version__ = info.version @@ -622,6 +622,10 @@ def parse( if jc.utils.has_data(data): for line in filter(None, data.splitlines()): + # skip any lines with only whitespace + if not line.strip(): + continue + # support configuration file by skipping commented lines if line.strip().startswith('#'): continue From 4c6eebaa333028c06eb9d64bc9c595bbbe7895b5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 22 Jan 2023 10:37:27 -0800 Subject: [PATCH 07/81] doc update --- README.md | 1 + completions/jc_bash_completion.sh | 4 +- completions/jc_zsh_completion.sh | 6 +- docs/lib.md | 2 +- docs/parsers/acpi.md | 2 +- docs/parsers/ssh_conf.md | 507 ++++++++++++++++++++++++++++++ docs/parsers/sshd_conf.md | 2 +- man/jc.1 | 7 +- 8 files changed, 523 insertions(+), 8 deletions(-) create mode 100644 docs/parsers/ssh_conf.md diff --git a/README.md b/README.md index e397cb46..e8cf0ce6 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,7 @@ option. | ` --sfdisk` | `sfdisk` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sfdisk) | | ` --shadow` | `/etc/shadow` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/shadow) | | ` --ss` | `ss` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ss) | +| ` --ssh-conf` | ssh config file and `ssh -G` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ssh_conf) | | ` --sshd-conf` | sshd config file and `sshd -T` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sshd_conf) | | ` --stat` | `stat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat) | | ` --stat-s` | `stat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s) | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index f1060a83..74dcf97a 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -3,8 +3,8 @@ _jc() local cur prev words cword jc_commands jc_parsers jc_options \ jc_about_options jc_about_mod_options jc_help_options jc_special_options - jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index d9d1addc..c718ddd2 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -9,7 +9,7 @@ _jc() { jc_help_options jc_help_options_describe \ jc_special_options jc_special_options_describe - jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) + jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) jc_commands_describe=( 'acpi:run "acpi" command with magic syntax.' 'airport:run "airport" command with magic syntax.' @@ -76,6 +76,7 @@ _jc() { 'sha512sum:run "sha512sum" command with magic syntax.' 'shasum:run "shasum" command with magic syntax.' 'ss:run "ss" command with magic syntax.' + 'ssh:run "ssh" command with magic syntax.' 'sshd:run "sshd" command with magic syntax.' 'stat:run "stat" command with magic syntax.' 'sum:run "sum" command with magic syntax.' @@ -102,7 +103,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -250,6 +251,7 @@ _jc() { '--sfdisk:`sfdisk` command parser' '--shadow:`/etc/shadow` file parser' '--ss:`ss` command parser' + '--ssh-conf:ssh config file and `ssh -G` command parser' '--sshd-conf:sshd config file and `sshd -T` command parser' '--stat:`stat` command parser' '--stat-s:`stat` command streaming parser' diff --git a/docs/lib.md b/docs/lib.md index 7853b606..c1a2b488 100644 --- a/docs/lib.md +++ b/docs/lib.md @@ -26,7 +26,7 @@ def parse( data: Union[str, bytes, Iterable[str]], quiet: bool = False, raw: bool = False, - ignore_exceptions: bool = None, + ignore_exceptions: Optional[bool] = None, **kwargs ) -> Union[JSONDictType, List[JSONDictType], Iterator[JSONDictType]] ``` diff --git a/docs/parsers/acpi.md b/docs/parsers/acpi.md index 096db581..22a79f54 100644 --- a/docs/parsers/acpi.md +++ b/docs/parsers/acpi.md @@ -250,4 +250,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/ssh_conf.md b/docs/parsers/ssh_conf.md new file mode 100644 index 00000000..077db643 --- /dev/null +++ b/docs/parsers/ssh_conf.md @@ -0,0 +1,507 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.ssh\_conf + +jc - JSON Convert ssh configuration file and `ssh -G` command output parser + +This parser will work with `ssh` configuration files or the output of +`ssh -G`. Any `Match` blocks in the `ssh` configuration file will be +ignored. + +Usage (cli): + + $ ssh -G hostname | jc --ssh-conf + +or + + $ jc ssh -G hostname + +or + + $ cat ~/.ssh/config | jc --ssh-conf + +Usage (module): + + import jc + result = jc.parse('ssh_conf', ssh_conf_output) + +Schema: + + [ + { + "host": string, + "addkeystoagent": string, + "addressfamily": string, + "batchmode": string, + "bindaddress": string, + "bindinterface": string, + "canonicaldomains": [ + string + ], + "canonicalizefallbacklocal": string, + "canonicalizehostname": string, + "canonicalizemaxdots": integer, + "canonicalizepermittedcnames": [ + string + ], + "casignaturealgorithms": [ + string + ], + "certificatefile": [ + string + ], + "checkhostip": string, + "ciphers": [ + string + ], + "clearallforwardings": string, + "compression": string, + "connectionattempts": integer, + "connecttimeout": integer, + "controlmaster": string, + "controlpath": string, + "controlpersist": string, + "dynamicforward": string, + "enableescapecommandline": string, + "enablesshkeysign": string, + "escapechar": string, + "exitonforwardfailure": string, + "fingerprinthash": string, + "forkafterauthentication": string, + "forwardagent": string, + "forwardx11": string, + "forwardx11timeout": integer, + "forwardx11trusted": string, + "gatewayports": string, + "globalknownhostsfile": [ + string + ], + "gssapiauthentication": string, + "gssapidelegatecredentials": string, + "hashknownhosts": string, + "hostbasedacceptedalgorithms": [ + string + ], + "hostbasedauthentication": string, + "hostkeyalgorithms": [ + string + ], + "hostkeyalias": string, + "hostname": string, + "identitiesonly": string, + "identityagent": string, + "identityfile": [ + string + ], + "ignoreunknown": string, + "include": [ + string + ], + "ipqos": [ + string + ], + "kbdinteractiveauthentication": string, + "kbdinteractivedevices": [ + string + ], + "kexalgorithms": [ + string + ], + "kexalgorithms_strategy": string, + "knownhostscommand": string, + "localcommand": string, + "localforward": [ + string + ], + "loglevel": string, + "logverbose": [ + string + ], + "macs": [ + string + ], + "macs_strategy": string, + "nohostauthenticationforlocalhost": string, + "numberofpasswordprompts": integer, + "passwordauthentication": string, + "permitlocalcommand": string, + "permitremoteopen": [ + string + ], + "pkcs11provider": string, + "port": integer, + "preferredauthentications": [ + string + ], + "protocol": integer, + "proxycommand": string, + "proxyjump": [ + string + ], + "proxyusefdpass": string, + "pubkeyacceptedalgorithms": [ + string + ], + "pubkeyacceptedalgorithms_strategy": string, + "pubkeyauthentication": string, + "rekeylimit": string, + "remotecommand": string, + "remoteforward": string, + "requesttty": string, + "requiredrsasize": integer, + "revokedhostkeys": string, + "securitykeyprovider": string, + "sendenv": [ + string + ], + "serveralivecountmax": integer, + "serveraliveinterval": integer, + "sessiontype": string, + "setenv": [ + string + ], + "stdinnull": string, + "streamlocalbindmask": string, + "streamlocalbindunlink": string, + "stricthostkeychecking": string, + "syslogfacility": string, + "tcpkeepalive": string, + "tunnel": string, + "tunneldevice": string, + "updatehostkeys": string, + "user": string, + "userknownhostsfile": [ + string + ], + "verifyhostkeydns": string, + "visualhostkey": string, + "xauthlocation": string + } + ] + +Examples: + + $ ssh -G - | jc --ssh-conf -p + [ + { + "user": "foo", + "hostname": "-", + "port": 22, + "addressfamily": "any", + "batchmode": "no", + "canonicalizefallbacklocal": "yes", + "canonicalizehostname": "false", + "checkhostip": "no", + "compression": "no", + "controlmaster": "false", + "enablesshkeysign": "no", + "clearallforwardings": "no", + "exitonforwardfailure": "no", + "fingerprinthash": "SHA256", + "forwardx11": "no", + "forwardx11trusted": "no", + "gatewayports": "no", + "gssapiauthentication": "no", + "gssapidelegatecredentials": "no", + "hashknownhosts": "no", + "hostbasedauthentication": "no", + "identitiesonly": "no", + "kbdinteractiveauthentication": "yes", + "nohostauthenticationforlocalhost": "no", + "passwordauthentication": "yes", + "permitlocalcommand": "no", + "proxyusefdpass": "no", + "pubkeyauthentication": "true", + "requesttty": "auto", + "sessiontype": "default", + "stdinnull": "no", + "forkafterauthentication": "no", + "streamlocalbindunlink": "no", + "stricthostkeychecking": "ask", + "tcpkeepalive": "yes", + "tunnel": "false", + "verifyhostkeydns": "false", + "visualhostkey": "no", + "updatehostkeys": "true", + "applemultipath": "no", + "canonicalizemaxdots": 1, + "connectionattempts": 1, + "forwardx11timeout": 1200, + "numberofpasswordprompts": 3, + "serveralivecountmax": 3, + "serveraliveinterval": 0, + "ciphers": [ + "chacha20-poly1305@openssh.com", + "aes128-ctr", + "aes192-ctr", + "aes256-ctr", + "aes128-gcm@openssh.com", + "aes256-gcm@openssh.com" + ], + "hostkeyalgorithms": [ + "ssh-ed25519-cert-v01@openssh.com", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521-cert-v01@openssh.com", + "rsa-sha2-512-cert-v01@openssh.com", + "rsa-sha2-256-cert-v01@openssh.com", + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "hostbasedacceptedalgorithms": [ + "ssh-ed25519-cert-v01@openssh.com", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521-cert-v01@openssh.com", + "rsa-sha2-512-cert-v01@openssh.com", + "rsa-sha2-256-cert-v01@openssh.com", + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "kexalgorithms": [ + "sntrup761x25519-sha512@openssh.com", + "curve25519-sha256", + "curve25519-sha256@libssh.org", + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "ecdh-sha2-nistp521", + "diffie-hellman-group-exchange-sha256", + "diffie-hellman-group16-sha512", + "diffie-hellman-group18-sha512", + "diffie-hellman-group14-sha256" + ], + "casignaturealgorithms": [ + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "loglevel": "INFO", + "macs": [ + "umac-64-etm@openssh.com", + "umac-128-etm@openssh.com", + "hmac-sha2-256-etm@openssh.com", + "hmac-sha2-512-etm@openssh.com", + "hmac-sha1-etm@openssh.com", + "umac-64@openssh.com", + "umac-128@openssh.com", + "hmac-sha2-256", + "hmac-sha2-512", + "hmac-sha1" + ], + "securitykeyprovider": "$SSH_SK_PROVIDER", + "pubkeyacceptedalgorithms": [ + "ssh-ed25519-cert-v01@openssh.com", + "ecdsa-sha2-nistp256-cert-v01@openssh.com", + "ecdsa-sha2-nistp384-cert-v01@openssh.com", + "ecdsa-sha2-nistp521-cert-v01@openssh.com", + "rsa-sha2-512-cert-v01@openssh.com", + "rsa-sha2-256-cert-v01@openssh.com", + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256" + ], + "xauthlocation": "/usr/X11R6/bin/xauth", + "identityfile": [ + "~/.ssh/id_rsa", + "~/.ssh/id_ecdsa", + "~/.ssh/id_ecdsa_sk", + "~/.ssh/id_ed25519", + "~/.ssh/id_ed25519_sk", + "~/.ssh/id_xmss", + "~/.ssh/id_dsa" + ], + "canonicaldomains": [ + "none" + ], + "globalknownhostsfile": [ + "/etc/ssh/ssh_known_hosts", + "/etc/ssh/ssh_known_hosts2" + ], + "userknownhostsfile": [ + "/Users/foo/.ssh/known_hosts", + "/Users/foo/.ssh/known_hosts2" + ], + "sendenv": [ + "LANG", + "LC_*" + ], + "logverbose": [ + "none" + ], + "permitremoteopen": [ + "any" + ], + "addkeystoagent": "false", + "forwardagent": "no", + "connecttimeout": null, + "tunneldevice": "any:any", + "canonicalizepermittedcnames": [ + "none" + ], + "controlpersist": "no", + "escapechar": "~", + "ipqos": [ + "af21", + "cs1" + ], + "rekeylimit": "0 0", + "streamlocalbindmask": "0177", + "syslogfacility": "USER" + } + ] + + $ cat ~/.ssh/config | jc --ssh-conf -p + [ + { + "host": "server1", + "hostname": "server1.cyberciti.biz", + "user": "nixcraft", + "port": 4242, + "identityfile": [ + "/nfs/shared/users/nixcraft/keys/server1/id_rsa" + ] + }, + { + "host": "nas01", + "hostname": "192.168.1.100", + "user": "root", + "identityfile": [ + "~/.ssh/nas01.key" + ] + }, + { + "host": "aws.apache", + "hostname": "1.2.3.4", + "user": "wwwdata", + "identityfile": [ + "~/.ssh/aws.apache.key" + ] + }, + { + "host": "uk.gw.lan uk.lan", + "hostname": "192.168.0.251", + "user": "nixcraft", + "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" + }, + { + "host": "proxyus", + "hostname": "vps1.cyberciti.biz", + "user": "breakfree", + "identityfile": [ + "~/.ssh/vps1.cyberciti.biz.key" + ], + "localforward": [ + "3128 127.0.0.1:3128" + ] + }, + { + "host": "*", + "forwardagent": "no", + "forwardx11": "no", + "forwardx11trusted": "yes", + "user": "nixcraft", + "port": 22, + "protocol": 2, + "serveraliveinterval": 60, + "serveralivecountmax": 30 + } + ] + + $ cat ~/.ssh/config | jc --ssh-conf -p -r + [ + { + "host": "server1", + "hostname": "server1.cyberciti.biz", + "user": "nixcraft", + "port": "4242", + "identityfile": [ + "/nfs/shared/users/nixcraft/keys/server1/id_rsa" + ] + }, + { + "host": "nas01", + "hostname": "192.168.1.100", + "user": "root", + "identityfile": [ + "~/.ssh/nas01.key" + ] + }, + { + "host": "aws.apache", + "hostname": "1.2.3.4", + "user": "wwwdata", + "identityfile": [ + "~/.ssh/aws.apache.key" + ] + }, + { + "host": "uk.gw.lan uk.lan", + "hostname": "192.168.0.251", + "user": "nixcraft", + "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" + }, + { + "host": "proxyus", + "hostname": "vps1.cyberciti.biz", + "user": "breakfree", + "identityfile": [ + "~/.ssh/vps1.cyberciti.biz.key" + ], + "localforward": [ + "3128 127.0.0.1:3128" + ] + }, + { + "host": "*", + "forwardagent": "no", + "forwardx11": "no", + "forwardx11trusted": "yes", + "user": "nixcraft", + "port": "22", + "protocol": "2", + "serveraliveinterval": "60", + "serveralivecountmax": "30" + } + ] + + + +### parse + +```python +def parse(data: str, + raw: bool = False, + quiet: bool = False) -> List[JSONDictType] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/sshd_conf.md b/docs/parsers/sshd_conf.md index 18b88afa..2926b684 100644 --- a/docs/parsers/sshd_conf.md +++ b/docs/parsers/sshd_conf.md @@ -504,4 +504,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, freebsd -Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/man/jc.1 b/man/jc.1 index 2011a01f..af9d0613 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-01-11 1.22.5 "JSON Convert" +.TH jc 1 2023-01-22 1.23.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS @@ -765,6 +765,11 @@ Semantic Version string parser \fB--ss\fP `ss` command parser +.TP +.B +\fB--ssh-conf\fP +ssh config file and `ssh -G` command parser + .TP .B \fB--sshd-conf\fP From 13ffe8a84d27f8c630c590f8d6ce019d7e4153da Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 22 Jan 2023 12:44:16 -0800 Subject: [PATCH 08/81] doc update --- README.md | 4 ++-- completions/jc_zsh_completion.sh | 4 ++-- docs/parsers/ssh_conf.md | 2 +- docs/parsers/sshd_conf.md | 2 +- jc/parsers/ssh_conf.py | 4 ++-- jc/parsers/sshd_conf.py | 4 ++-- man/jc.1 | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e8cf0ce6..8a643ea3 100644 --- a/README.md +++ b/README.md @@ -251,8 +251,8 @@ option. | ` --sfdisk` | `sfdisk` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sfdisk) | | ` --shadow` | `/etc/shadow` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/shadow) | | ` --ss` | `ss` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ss) | -| ` --ssh-conf` | ssh config file and `ssh -G` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ssh_conf) | -| ` --sshd-conf` | sshd config file and `sshd -T` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sshd_conf) | +| ` --ssh-conf` | `ssh` config file and `ssh -G` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ssh_conf) | +| ` --sshd-conf` | `sshd` config file and `sshd -T` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sshd_conf) | | ` --stat` | `stat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat) | | ` --stat-s` | `stat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s) | | ` --sysctl` | `sysctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sysctl) | diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index c718ddd2..8823e30e 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -251,8 +251,8 @@ _jc() { '--sfdisk:`sfdisk` command parser' '--shadow:`/etc/shadow` file parser' '--ss:`ss` command parser' - '--ssh-conf:ssh config file and `ssh -G` command parser' - '--sshd-conf:sshd config file and `sshd -T` command parser' + '--ssh-conf:`ssh` config file and `ssh -G` command parser' + '--sshd-conf:`sshd` config file and `sshd -T` command parser' '--stat:`stat` command parser' '--stat-s:`stat` command streaming parser' '--sysctl:`sysctl` command parser' diff --git a/docs/parsers/ssh_conf.md b/docs/parsers/ssh_conf.md index 077db643..77acfeaa 100644 --- a/docs/parsers/ssh_conf.md +++ b/docs/parsers/ssh_conf.md @@ -3,7 +3,7 @@ # jc.parsers.ssh\_conf -jc - JSON Convert ssh configuration file and `ssh -G` command output parser +jc - JSON Convert `ssh` configuration file and `ssh -G` command output parser This parser will work with `ssh` configuration files or the output of `ssh -G`. Any `Match` blocks in the `ssh` configuration file will be diff --git a/docs/parsers/sshd_conf.md b/docs/parsers/sshd_conf.md index 2926b684..bffeb82f 100644 --- a/docs/parsers/sshd_conf.md +++ b/docs/parsers/sshd_conf.md @@ -3,7 +3,7 @@ # jc.parsers.sshd\_conf -jc - JSON Convert sshd configuration file and `sshd -T` command output parser +jc - JSON Convert `sshd` configuration file and `sshd -T` command output parser This parser will work with `sshd` configuration files or the output of `sshd -T`. Any `Match` blocks in the `sshd` configuration file will be diff --git a/jc/parsers/ssh_conf.py b/jc/parsers/ssh_conf.py index 1e9883f5..45ed2cdd 100644 --- a/jc/parsers/ssh_conf.py +++ b/jc/parsers/ssh_conf.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert ssh configuration file and `ssh -G` command output parser +"""jc - JSON Convert `ssh` configuration file and `ssh -G` command output parser This parser will work with `ssh` configuration files or the output of `ssh -G`. Any `Match` blocks in the `ssh` configuration file will be @@ -482,7 +482,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = 'ssh config file and `ssh -G` command parser' + description = '`ssh` config file and `ssh -G` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'freebsd'] diff --git a/jc/parsers/sshd_conf.py b/jc/parsers/sshd_conf.py index 62782ade..a295e237 100644 --- a/jc/parsers/sshd_conf.py +++ b/jc/parsers/sshd_conf.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert sshd configuration file and `sshd -T` command output parser +"""jc - JSON Convert `sshd` configuration file and `sshd -T` command output parser This parser will work with `sshd` configuration files or the output of `sshd -T`. Any `Match` blocks in the `sshd` configuration file will be @@ -484,7 +484,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" version = '1.1' - description = 'sshd config file and `sshd -T` command parser' + description = '`sshd` config file and `sshd -T` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'freebsd'] diff --git a/man/jc.1 b/man/jc.1 index af9d0613..a0a4c126 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -768,12 +768,12 @@ Semantic Version string parser .TP .B \fB--ssh-conf\fP -ssh config file and `ssh -G` command parser +`ssh` config file and `ssh -G` command parser .TP .B \fB--sshd-conf\fP -sshd config file and `sshd -T` command parser +`sshd` config file and `sshd -T` command parser .TP .B From 0c82fe7e4dc7ffd26a7675ec11a1110f25b30825 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 23 Jan 2023 13:57:48 -0800 Subject: [PATCH 09/81] add slicer functionality --- jc/cli.py | 172 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 116 insertions(+), 56 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index ec7434f3..fda2a9ed 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -5,11 +5,13 @@ JC cli module import io import sys import os +import re +from itertools import islice from datetime import datetime, timezone import textwrap import shlex import subprocess -from typing import List, Dict, Union, Optional, TextIO +from typing import List, Dict, Iterable, Union, Optional, TextIO from types import ModuleType from .lib import ( __version__, parser_info, all_parser_info, parsers, _get_parser, _parser_is_streaming, @@ -69,11 +71,11 @@ class JcCli(): 'help_me', 'pretty', 'quiet', 'ignore_exceptions', 'raw', 'meta_out', 'unbuffer', 'version_info', 'yaml_output', 'bash_comp', 'zsh_comp', 'magic_found_parser', 'magic_options', 'magic_run_command', 'magic_run_command_str', 'magic_stdout', - 'magic_stderr', 'magic_returncode' + 'magic_stderr', 'magic_returncode', 'slice_start', 'slice_end' ) def __init__(self) -> None: - self.data_in: Optional[Union[str, bytes, TextIO]] = None + self.data_in: Optional[Union[str, bytes, TextIO, Iterable[str]]] = None self.data_out: Optional[Union[List[JSONDictType], JSONDictType]] = None self.options: List[str] = [] self.args: List[str] = [] @@ -89,6 +91,10 @@ class JcCli(): self.json_indent: Optional[int] = None self.run_timestamp: Optional[datetime] = None + # slicer + self.slice_start: Optional[int] = None + self.slice_end: Optional[int] = None + # cli options self.about: bool = False self.debug: bool = False @@ -574,59 +580,6 @@ class JcCli(): utils.error_message(['Missing piped data. Use "jc -h" for help.']) self.exit_error() - def streaming_parse_and_print(self) -> None: - """only supports UTF-8 string data for now""" - self.data_in = sys.stdin - if self.parser_module: - result = self.parser_module.parse( - self.data_in, - raw=self.raw, - quiet=self.quiet, - ignore_exceptions=self.ignore_exceptions - ) - - for line in result: - self.data_out = line - if self.meta_out: - self.run_timestamp = datetime.now(timezone.utc) - self.add_metadata_to_output() - - self.safe_print_out() - - def standard_parse_and_print(self) -> None: - """supports binary and UTF-8 string data""" - self.data_in = self.magic_stdout or sys.stdin.buffer.read() - - # convert to UTF-8, if possible. Otherwise, leave as bytes - try: - if isinstance(self.data_in, bytes): - self.data_in = self.data_in.decode('utf-8') - except UnicodeDecodeError: - pass - - if self.parser_module: - self.data_out = self.parser_module.parse( - self.data_in, - raw=self.raw, - quiet=self.quiet - ) - - if self.meta_out: - self.run_timestamp = datetime.now(timezone.utc) - self.add_metadata_to_output() - - self.safe_print_out() - - def exit_clean(self) -> None: - exit_code: int = self.magic_returncode + JC_CLEAN_EXIT - exit_code = min(exit_code, MAX_EXIT) - sys.exit(exit_code) - - def exit_error(self) -> None: - exit_code: int = self.magic_returncode + JC_ERROR_EXIT - exit_code = min(exit_code, MAX_EXIT) - sys.exit(exit_code) - def add_metadata_to_output(self) -> None: """ This function mutates data_out in place. If the _jc_meta field @@ -669,6 +622,113 @@ class JcCli(): utils.error_message(['Parser returned an unsupported object type.']) self.exit_error() + @staticmethod + def lazy_splitlines(text: str) -> Iterable[str]: + start = 0 + for m in re.finditer(r'(\r\n|\r|\n)', text): + begin, end = m.span() + if begin != start: + yield text[start:begin] + start = end + + if text[start:]: + yield text[start:] + + def slicer(self) -> None: + """Slice input data lazily, if possible. Updates self.data_in""" + + self.slice_start = -20001 # 5 + self.slice_end = -20000 # -2 or 8 + + if not self.slice_start is None or not self.slice_end is None: + # standard parsers UTF-8 input + if isinstance(self.data_in, str): + data_in_iter = self.lazy_splitlines(self.data_in) + + # positive slices + if (self.slice_start is None or self.slice_start >= 0) \ + and (self.slice_end is None or self.slice_end >= 0): + + self.data_in = '\n'.join(islice(data_in_iter, self.slice_start, self.slice_end)) + + # negative slices found (non-lazy, uses more memory) + else: + self.data_in = '\n'.join(list(data_in_iter)[self.slice_start:self.slice_end]) + + # standard parsers bytes input + elif isinstance(self.data_in, bytes): + utils.warning_message(['Cannot slice bytes data.']) + + # streaming parsers UTF-8 input + else: + # positive slices + if (self.slice_start is None or self.slice_start >= 0) \ + and (self.slice_end is None or self.slice_end >= 0) \ + and self.data_in: + + self.data_in = islice(self.data_in, self.slice_start, self.slice_end) + + # negative slices found (non-lazy, uses more memory) + elif self.data_in: + self.data_in = list(self.data_in)[self.slice_start:self.slice_end] + + def streaming_parse_and_print(self) -> None: + """only supports UTF-8 string data for now""" + self.data_in = sys.stdin + self.slicer() + + if self.parser_module: + result = self.parser_module.parse( + self.data_in, + raw=self.raw, + quiet=self.quiet, + ignore_exceptions=self.ignore_exceptions + ) + + for line in result: + self.data_out = line + if self.meta_out: + self.run_timestamp = datetime.now(timezone.utc) + self.add_metadata_to_output() + + self.safe_print_out() + + def standard_parse_and_print(self) -> None: + """supports binary and UTF-8 string data""" + self.data_in = self.magic_stdout or sys.stdin.buffer.read() + + # convert to UTF-8, if possible. Otherwise, leave as bytes + try: + if isinstance(self.data_in, bytes): + self.data_in = self.data_in.decode('utf-8') + except UnicodeDecodeError: + pass + + self.slicer() + + if self.parser_module: + self.data_out = self.parser_module.parse( + self.data_in, + raw=self.raw, + quiet=self.quiet + ) + + if self.meta_out: + self.run_timestamp = datetime.now(timezone.utc) + self.add_metadata_to_output() + + self.safe_print_out() + + def exit_clean(self) -> None: + exit_code: int = self.magic_returncode + JC_CLEAN_EXIT + exit_code = min(exit_code, MAX_EXIT) + sys.exit(exit_code) + + def exit_error(self) -> None: + exit_code: int = self.magic_returncode + JC_ERROR_EXIT + exit_code = min(exit_code, MAX_EXIT) + sys.exit(exit_code) + def _run(self) -> None: # enable colors for Windows cmd.exe terminal if sys.platform.startswith('win32'): From f5f5f102fba1c3fb8f64edc5b0182270c9f22ebc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 23 Jan 2023 16:13:17 -0800 Subject: [PATCH 10/81] full functioning slicer feature --- jc/cli.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index fda2a9ed..ed098e83 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -42,6 +42,10 @@ except Exception: JC_CLEAN_EXIT: int = 0 JC_ERROR_EXIT: int = 100 MAX_EXIT: int = 255 +SLICER_PATTERN: str = r'-?[0-9]*\:-?[0-9]*$' +SLICER_RE = re.compile(SLICER_PATTERN) +NEWLINES_PATTERN: str = r'(\r\n|\r|\n)' +NEWLINES_RE = re.compile(NEWLINES_PATTERN) class info(): @@ -71,7 +75,7 @@ class JcCli(): 'help_me', 'pretty', 'quiet', 'ignore_exceptions', 'raw', 'meta_out', 'unbuffer', 'version_info', 'yaml_output', 'bash_comp', 'zsh_comp', 'magic_found_parser', 'magic_options', 'magic_run_command', 'magic_run_command_str', 'magic_stdout', - 'magic_stderr', 'magic_returncode', 'slice_start', 'slice_end' + 'magic_stderr', 'magic_returncode', 'slice_str', 'slice_start', 'slice_end' ) def __init__(self) -> None: @@ -92,6 +96,7 @@ class JcCli(): self.run_timestamp: Optional[datetime] = None # slicer + self.slice_str: str = '' self.slice_start: Optional[int] = None self.slice_end: Optional[int] = None @@ -438,6 +443,17 @@ class JcCli(): self.magic_options = [] return + # slicer found + if ':' in arg: + if SLICER_RE.match(arg): + self.slice_str = arg + args_given.pop(0) + continue + else: + utils.warning_message(['Invalid slice syntax.']) + args_given.pop(0) + continue + # option found - populate option list if arg.startswith('-'): self.magic_options.extend(args_given.pop(0)[1:]) @@ -625,7 +641,7 @@ class JcCli(): @staticmethod def lazy_splitlines(text: str) -> Iterable[str]: start = 0 - for m in re.finditer(r'(\r\n|\r|\n)', text): + for m in NEWLINES_RE.finditer(text): begin, end = m.span() if begin != start: yield text[start:begin] @@ -636,9 +652,12 @@ class JcCli(): def slicer(self) -> None: """Slice input data lazily, if possible. Updates self.data_in""" - - self.slice_start = -20001 # 5 - self.slice_end = -20000 # -2 or 8 + if self.slice_str: + slice_start_str, slice_end_str = self.slice_str.split(':', maxsplit=1) + if slice_start_str: + self.slice_start = int(slice_start_str) + if slice_end_str: + self.slice_end = int(slice_end_str) if not self.slice_start is None or not self.slice_end is None: # standard parsers UTF-8 input @@ -744,6 +763,12 @@ class JcCli(): # find options if magic_parser did not find a command if not self.magic_found_parser: for opt in self.args: + if ':' in opt: + if SLICER_RE.match(opt): + self.slice_str = opt + else: + utils.warning_message(['Invalid slice syntax.']) + if opt in long_options_map: self.options.extend(long_options_map[opt][0]) From 3bb1d891656e7575236429accbdeffd72e3d350a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 23 Jan 2023 16:50:41 -0800 Subject: [PATCH 11/81] fix multiple slicer warnings --- jc/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index ed098e83..316c85ec 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -766,8 +766,6 @@ class JcCli(): if ':' in opt: if SLICER_RE.match(opt): self.slice_str = opt - else: - utils.warning_message(['Invalid slice syntax.']) if opt in long_options_map: self.options.extend(long_options_map[opt][0]) From 0e6cec62c165d18b89f75f172c7143f50a2118af Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 23 Jan 2023 16:53:06 -0800 Subject: [PATCH 12/81] remove unneeded slicer syntax check --- jc/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index 316c85ec..36936370 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -763,9 +763,8 @@ class JcCli(): # find options if magic_parser did not find a command if not self.magic_found_parser: for opt in self.args: - if ':' in opt: - if SLICER_RE.match(opt): - self.slice_str = opt + if SLICER_RE.match(opt): + self.slice_str = opt if opt in long_options_map: self.options.extend(long_options_map[opt][0]) From a5a87c7da18ac2eafdb15bacb1c3039f0e571ef2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 25 Jan 2023 15:58:27 -0800 Subject: [PATCH 13/81] add ver parser --- CHANGELOG | 5 +- jc/lib.py | 1 + jc/parsers/ver.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 jc/parsers/ver.py diff --git a/CHANGELOG b/CHANGELOG index e9152163..982bbe9e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,9 @@ jc changelog -20230122 v1.23.0 +20230125 v1.23.0 +- Add input slicing as a `jc` command-line option - Add `ssh` configuration file parser -- Add input slicing +- Add `ver` Version string parser - Fix `acpi` command parser for "will never fully discharge" battery state 20230111 v1.22.5 diff --git a/jc/lib.py b/jc/lib.py index 1ca94c88..6139acea 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -190,6 +190,7 @@ parsers: List[str] = [ 'upower', 'uptime', 'url', + 'ver', 'vmstat', 'vmstat-s', 'w', diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py new file mode 100644 index 00000000..a303590a --- /dev/null +++ b/jc/parsers/ver.py @@ -0,0 +1,151 @@ +"""jc - JSON Convert Version string output parser + +Best effort attempt to parse various styles of version numbers. This parser +is based off of the version parser included in the CPython distutils +libary. + +If the version string conforms to some de facto-standard versioning rules +followed by many developers a `strict` key will be present in the output +with a value of `true` along with the named parsed components. + +All other version strings will have a `strict` value of `false` and a +`components` key will contain a list of detected parts of the version +string. + +Usage (cli): + + $ echo '1.2b' | jc --ver + + +Usage (module): + + import jc + result = jc.parse('ver', version_string_output) + +Schema: + + [ + { + "version": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ echo '1.2b' | jc --ver -p + [] +""" +import re +from typing import List, Dict +from jc.jc_types import JSONDictType +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'Version string parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + details = 'Based on distutils/version.py from CPython 3.9.5' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + tags = ['generic', 'string'] + + +__version__ = info.version + + +def _process(proc_data: JSONDictType) -> JSONDictType: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def strict_parse(vstring): + version_re = re.compile(r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$', re.VERBOSE) + match = version_re.match(vstring) + if not match: + raise ValueError("invalid version number '%s'" % vstring) + + (major, minor, patch, prerelease, prerelease_num) = \ + match.group(1, 2, 4, 5, 6) + + if patch: + version = tuple(map(int, [major, minor, patch])) + else: + version = tuple(map(int, [major, minor])) + (0,) + + if prerelease: + prerelease = (prerelease[0], int(prerelease_num)) + else: + prerelease = None + + return { + 'major': major, + 'minor': minor, + 'patch': patch, + 'prerelease': prerelease, + 'prerelease_num': prerelease_num + } + + +def loose_parse(vstring): + component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) + components = [x for x in component_re.split(vstring) if x and x != '.'] + + return components + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> JSONDictType: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + strict = True + + if jc.utils.has_data(data): + + # based on distutils/version.py from CPython 3.9.5 + # PSF License (see https://opensource.org/licenses/Python-2.0) + + data = data.strip() + + try: + raw_output = strict_parse(data) + + except ValueError: + raw_output['components'] = loose_parse(data) + strict = False + + if raw_output: + raw_output['strict'] = strict + + return raw_output if raw else _process(raw_output) From 1559cd9f5c2e9a4deccda8e51026c91eeebffd3a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 25 Jan 2023 16:13:33 -0800 Subject: [PATCH 14/81] fixup patch numbers --- jc/parsers/ver.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index a303590a..25013c65 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -81,13 +81,11 @@ def strict_parse(vstring): (major, minor, patch, prerelease, prerelease_num) = \ match.group(1, 2, 4, 5, 6) - if patch: - version = tuple(map(int, [major, minor, patch])) - else: - version = tuple(map(int, [major, minor])) + (0,) + if not patch: + patch = '0' if prerelease: - prerelease = (prerelease[0], int(prerelease_num)) + prerelease = prerelease[0] else: prerelease = None From 5d872b1535c5923d162d7331627a409f41b8717e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 25 Jan 2023 16:30:20 -0800 Subject: [PATCH 15/81] docstring update: add see also section. --- jc/parsers/semver.py | 2 ++ jc/parsers/ver.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/jc/parsers/semver.py b/jc/parsers/semver.py index 3660e3e9..37e509ec 100644 --- a/jc/parsers/semver.py +++ b/jc/parsers/semver.py @@ -2,6 +2,8 @@ This parser conforms to the specification at https://semver.org/ +See also: `ver` parser. + Usage (cli): $ echo 1.2.3-rc.1+44837 | jc --semver diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index 25013c65..f1e6361a 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -12,6 +12,8 @@ All other version strings will have a `strict` value of `false` and a `components` key will contain a list of detected parts of the version string. +See Also: `semver` parser. + Usage (cli): $ echo '1.2b' | jc --ver From f05c3d6113cde87395eb1c7833db36cf0906106c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 25 Jan 2023 16:57:15 -0800 Subject: [PATCH 16/81] docstring update --- jc/parsers/ver.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index f1e6361a..03e7b641 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -16,7 +16,7 @@ See Also: `semver` parser. Usage (cli): - $ echo '1.2b' | jc --ver + $ echo 1.2a1 | jc --ver Usage (module): @@ -36,8 +36,26 @@ Schema: Examples: - $ echo '1.2b' | jc --ver -p - [] + $ echo 1.2a1 | jc --ver -p + { + "major": "1", + "minor": "2", + "patch": "0", + "prerelease": "a", + "prerelease_num": "1", + "strict": true + } + + $ echo 1.2beta3 | jc --ver -p + { + "components": [ + "1", + "2", + "beta", + "3" + ], + "strict": false + } """ import re from typing import List, Dict From caa516db72169bd2c6167de5fbd091cf9a5e0e57 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 25 Jan 2023 16:59:49 -0800 Subject: [PATCH 17/81] formatting --- jc/parsers/ver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index 03e7b641..bb742461 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -18,7 +18,6 @@ Usage (cli): $ echo 1.2a1 | jc --ver - Usage (module): import jc From ae7ed4eec5e7b2c1e7c0d5948effaafb428d8aac Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 08:48:46 -0800 Subject: [PATCH 18/81] doc update --- jc/parsers/semver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/semver.py b/jc/parsers/semver.py index 37e509ec..e433de62 100644 --- a/jc/parsers/semver.py +++ b/jc/parsers/semver.py @@ -2,7 +2,7 @@ This parser conforms to the specification at https://semver.org/ -See also: `ver` parser. +See Also: `ver` parser. Usage (cli): From 103168c25b23876eec975d50f2a470b0c40c1d54 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 08:48:59 -0800 Subject: [PATCH 19/81] add integer conversions --- jc/parsers/ver.py | 61 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index bb742461..74537d74 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -1,6 +1,6 @@ """jc - JSON Convert Version string output parser -Best effort attempt to parse various styles of version numbers. This parser +Best-effort attempt to parse various styles of version numbers. This parser is based off of the version parser included in the CPython distutils libary. @@ -25,17 +25,31 @@ Usage (module): Schema: - [ - { - "version": string, - "bar": boolean, - "baz": integer - } - ] + { + "major": integer, + "minor": integer, + "patch": integer, + "prerelease": string, + "prerelease_num": integer, + "components": [ + integer/string, + ] + "strict": boolean + } Examples: $ echo 1.2a1 | jc --ver -p + { + "major": 1, + "minor": 2, + "patch": 0, + "prerelease": "a", + "prerelease_num": 1, + "strict": true + } + + $ echo 1.2a1 | jc --ver -p -r { "major": "1", "minor": "2", @@ -46,6 +60,17 @@ Examples: } $ echo 1.2beta3 | jc --ver -p + { + "components": [ + 1, + 2, + "beta", + 3 + ], + "strict": false + } + + $ echo 1.2beta3 | jc --ver -p -r { "components": [ "1", @@ -57,7 +82,7 @@ Examples: } """ import re -from typing import List, Dict +from typing import Dict from jc.jc_types import JSONDictType import jc.utils @@ -68,7 +93,7 @@ class info(): description = 'Version string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - details = 'Based on distutils/version.py from CPython 3.9.5' + details = 'Based on distutils/version.py from CPython 3.9.5.' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] tags = ['generic', 'string'] @@ -88,6 +113,22 @@ def _process(proc_data: JSONDictType) -> JSONDictType: List of Dictionaries. Structured to conform to the schema. """ + int_list = {'major', 'minor', 'patch', 'prerelease', 'prerelease_num'} + + for k, v in proc_data.items(): + if k in int_list: + try: + proc_data[k] = int(v) + except Exception: + pass + + if 'components' in proc_data: + for i, obj in enumerate(proc_data['components']): + try: + proc_data['components'][i] = int(obj) + except Exception: + pass + return proc_data From fa693a8bb1390be762d0087cfdabadf4e233de40 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 08:53:06 -0800 Subject: [PATCH 20/81] formatting --- jc/parsers/ver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index 74537d74..50fbe54b 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -33,7 +33,7 @@ Schema: "prerelease_num": integer, "components": [ integer/string, - ] + ], "strict": boolean } From f23715a7835d76ddda21dfb8695551dd46706ef9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 08:53:50 -0800 Subject: [PATCH 21/81] formatting --- jc/parsers/ver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index 50fbe54b..43126804 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -32,7 +32,7 @@ Schema: "prerelease": string, "prerelease_num": integer, "components": [ - integer/string, + integer/string ], "strict": boolean } From 8e86a04448598882c722ec644d0d1dfb85a7fd75 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 16:18:54 -0800 Subject: [PATCH 22/81] add slicing info --- README.md | 56 +++++++++++++++++++++++++++++++++++---- templates/readme_template | 55 ++++++++++++++++++++++++++++++++++---- 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8a643ea3..0aea7fd9 100644 --- a/README.md +++ b/README.md @@ -133,9 +133,9 @@ on Github. `jc` accepts piped input from `STDIN` and outputs a JSON representation of the previous command's output to `STDOUT`. ```bash -COMMAND | jc [OPTIONS] PARSER -cat FILE | jc [OPTIONS] PARSER -echo STRING | jc [OPTIONS] PARSER +COMMAND | jc [SLICE] [OPTIONS] PARSER +cat FILE | jc [SLICE] [OPTIONS] PARSER +echo STRING | jc [SLICE] [OPTIONS] PARSER ``` Alternatively, the "magic" syntax can be used by prepending `jc` to the command @@ -143,8 +143,8 @@ to be converted or in front of the absolute path for Proc files. Options can be passed to `jc` immediately before the command or Proc file path is given. (Note: command aliases and shell builtins are not supported) ```bash -jc [OPTIONS] COMMAND -jc [OPTIONS] /proc/ +jc [SLICE] [OPTIONS] COMMAND +jc [SLICE] [OPTIONS] /proc/ ``` The JSON output can be compact (default) or pretty formatted with the `-p` @@ -282,6 +282,7 @@ option. | ` --upower` | `upower` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/upower) | | ` --uptime` | `uptime` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/uptime) | | ` --url` | URL string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/url) | +| ` --ver` | Version string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ver) | | ` --vmstat` | `vmstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/vmstat) | | ` --vmstat-s` | `vmstat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/vmstat_s) | | ` --w` | `w` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/w) | @@ -312,6 +313,51 @@ option. | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | +### Slice +Line slicing is supported using the `START:STOP` syntax similar to Python +slicing. This allows you to skip lines at the beginning and/or end of the +output you would like `jc` to convert. + +`START` and `STOP` can be positive or negative integers and allow you to +specify how many lines to skip and how many lines to process. Positive +slices are the most memory efficient. Any negative slices will use more +memory. + +For example, to skip the first and last line of the following text, you +could express the slice in a couple ways: + +```bash +$ cat table.txt +We want to skip this header information +col1 col2 +foo 1 +bar 1 +We also want to skip this footer +$ cat table.txt | jc 1:-1 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +$ cat table.txt | jc 1:4 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +``` +Notice how when using positive integers the index location of `STOP` is +non-inclusive. Positive slices count from the first line of the output +toward the end starting at `0` as the first line. Negative slices count from +the last line toward the beginning starting at `-1` as the last line. This +is also the way [Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) +feature works. + +Here is a quick breakdown: + +| Slice | Description | +|-----------------|------------------------------------------------------------| +| `START:STOP` | lines `START` through `STOP - 1` | +| `START:` | lines `START` through the rest of the output | +| `:STOP` | lines from the beginning through `STOP - 1` | +| `-START:STOP` | `START` lines from the end through `STOP - 1` | +| `START:-STOP` | lines `START` through `STOP` lines from the end | +| `-START:` | `START` lines from the end through the rest or the output | +| `:-STOP` | lines from the beginning through `STOP` lines from the end | +| `:` | all lines | + ### Exit Codes Any fatal errors within `jc` will generate an exit code of `100`, otherwise the exit code will be `0`. diff --git a/templates/readme_template b/templates/readme_template index 8a07465f..f7097cb8 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -133,9 +133,9 @@ on Github. `jc` accepts piped input from `STDIN` and outputs a JSON representation of the previous command's output to `STDOUT`. ```bash -COMMAND | jc [OPTIONS] PARSER -cat FILE | jc [OPTIONS] PARSER -echo STRING | jc [OPTIONS] PARSER +COMMAND | jc [SLICE] [OPTIONS] PARSER +cat FILE | jc [SLICE] [OPTIONS] PARSER +echo STRING | jc [SLICE] [OPTIONS] PARSER ``` Alternatively, the "magic" syntax can be used by prepending `jc` to the command @@ -143,8 +143,8 @@ to be converted or in front of the absolute path for Proc files. Options can be passed to `jc` immediately before the command or Proc file path is given. (Note: command aliases and shell builtins are not supported) ```bash -jc [OPTIONS] COMMAND -jc [OPTIONS] /proc/ +jc [SLICE] [OPTIONS] COMMAND +jc [SLICE] [OPTIONS] /proc/ ``` The JSON output can be compact (default) or pretty formatted with the `-p` @@ -175,6 +175,51 @@ option. | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | +### Slice +Line slicing is supported using the `START:STOP` syntax similar to Python +slicing. This allows you to skip lines at the beginning and/or end of the +output you would like `jc` to convert. + +`START` and `STOP` can be positive or negative integers and allow you to +specify how many lines to skip and how many lines to process. Positive +slices are the most memory efficient. Any negative slices will use more +memory. + +For example, to skip the first and last line of the following text, you +could express the slice in a couple ways: + +```bash +$ cat table.txt +We want to skip this header information +col1 col2 +foo 1 +bar 1 +We also want to skip this footer +$ cat table.txt | jc 1:-1 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +$ cat table.txt | jc 1:4 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +``` +Notice how when using positive integers the index location of `STOP` is +non-inclusive. Positive slices count from the first line of the output +toward the end starting at `0` as the first line. Negative slices count from +the last line toward the beginning starting at `-1` as the last line. This +is also the way [Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) +feature works. + +Here is a quick breakdown: + +| Slice | Description | +|-----------------|------------------------------------------------------------| +| `START:STOP` | lines `START` through `STOP - 1` | +| `START:` | lines `START` through the rest of the output | +| `:STOP` | lines from the beginning through `STOP - 1` | +| `-START:STOP` | `START` lines from the end through `STOP - 1` | +| `START:-STOP` | lines `START` through `STOP` lines from the end | +| `-START:` | `START` lines from the end through the rest or the output | +| `:-STOP` | lines from the beginning through `STOP` lines from the end | +| `:` | all lines | + ### Exit Codes Any fatal errors within `jc` will generate an exit code of `100`, otherwise the exit code will be `0`. From 61b8e9f7b50e35b5d57c9429d9456a4c0c276c38 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 16:24:32 -0800 Subject: [PATCH 23/81] doc update --- README.md | 10 ++++++---- templates/readme_template | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0aea7fd9..2c3a335a 100644 --- a/README.md +++ b/README.md @@ -329,15 +329,17 @@ could express the slice in a couple ways: ```bash $ cat table.txt We want to skip this header information -col1 col2 -foo 1 -bar 1 + col1 col2 + foo 1 + bar 1 We also want to skip this footer $ cat table.txt | jc 1:-1 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] $ cat table.txt | jc 1:4 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] ``` +In this example `1:-1` and `1:4` line slices provide the same output. + Notice how when using positive integers the index location of `STOP` is non-inclusive. Positive slices count from the first line of the output toward the end starting at `0` as the first line. Negative slices count from @@ -354,7 +356,7 @@ Here is a quick breakdown: | `:STOP` | lines from the beginning through `STOP - 1` | | `-START:STOP` | `START` lines from the end through `STOP - 1` | | `START:-STOP` | lines `START` through `STOP` lines from the end | -| `-START:` | `START` lines from the end through the rest or the output | +| `-START:` | `START` lines from the end through the rest of the output | | `:-STOP` | lines from the beginning through `STOP` lines from the end | | `:` | all lines | diff --git a/templates/readme_template b/templates/readme_template index f7097cb8..0c1d6701 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -191,15 +191,17 @@ could express the slice in a couple ways: ```bash $ cat table.txt We want to skip this header information -col1 col2 -foo 1 -bar 1 + col1 col2 + foo 1 + bar 1 We also want to skip this footer $ cat table.txt | jc 1:-1 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] $ cat table.txt | jc 1:4 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] ``` +In this example `1:-1` and `1:4` line slices provide the same output. + Notice how when using positive integers the index location of `STOP` is non-inclusive. Positive slices count from the first line of the output toward the end starting at `0` as the first line. Negative slices count from @@ -216,7 +218,7 @@ Here is a quick breakdown: | `:STOP` | lines from the beginning through `STOP - 1` | | `-START:STOP` | `START` lines from the end through `STOP - 1` | | `START:-STOP` | lines `START` through `STOP` lines from the end | -| `-START:` | `START` lines from the end through the rest or the output | +| `-START:` | `START` lines from the end through the rest of the output | | `:-STOP` | lines from the beginning through `STOP` lines from the end | | `:` | all lines | From 81447ec9e6dfb86041b97b4f4c5e649f3dbb3bc6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 16:44:26 -0800 Subject: [PATCH 24/81] formatting --- README.md | 10 +++++----- templates/readme_template | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2c3a335a..14ef6d1a 100644 --- a/README.md +++ b/README.md @@ -328,11 +328,11 @@ could express the slice in a couple ways: ```bash $ cat table.txt -We want to skip this header information - col1 col2 - foo 1 - bar 1 -We also want to skip this footer + We want to skip this header + col1 col2 + foo 1 + bar 1 + We also want to skip this footer $ cat table.txt | jc 1:-1 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] $ cat table.txt | jc 1:4 --asciitable diff --git a/templates/readme_template b/templates/readme_template index 0c1d6701..8cc7d4d9 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -190,11 +190,11 @@ could express the slice in a couple ways: ```bash $ cat table.txt -We want to skip this header information - col1 col2 - foo 1 - bar 1 -We also want to skip this footer + We want to skip this header + col1 col2 + foo 1 + bar 1 + We also want to skip this footer $ cat table.txt | jc 1:-1 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] $ cat table.txt | jc 1:4 --asciitable From 976fea4eb254dfa5888b5de3fe28454f3a6adf38 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 16:46:54 -0800 Subject: [PATCH 25/81] Formatting --- README.md | 10 +++++----- templates/readme_template | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 14ef6d1a..b94ec57f 100644 --- a/README.md +++ b/README.md @@ -328,11 +328,11 @@ could express the slice in a couple ways: ```bash $ cat table.txt - We want to skip this header - col1 col2 - foo 1 - bar 1 - We also want to skip this footer + ### We want to skip this header ### + col1 col2 + foo 1 + bar 1 + ### We want to skip this footer ### $ cat table.txt | jc 1:-1 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] $ cat table.txt | jc 1:4 --asciitable diff --git a/templates/readme_template b/templates/readme_template index 8cc7d4d9..634e5714 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -190,11 +190,11 @@ could express the slice in a couple ways: ```bash $ cat table.txt - We want to skip this header - col1 col2 - foo 1 - bar 1 - We also want to skip this footer + ### We want to skip this header ### + col1 col2 + foo 1 + bar 1 + ### We want to skip this footer ### $ cat table.txt | jc 1:-1 --asciitable [{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] $ cat table.txt | jc 1:4 --asciitable From 910cb34b09458d69aa95267aa9f3e4f9c540f093 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 16:49:09 -0800 Subject: [PATCH 26/81] formatting --- templates/readme_template | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/readme_template b/templates/readme_template index 634e5714..fb17b500 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -193,12 +193,12 @@ $ cat table.txt ### We want to skip this header ### col1 col2 foo 1 - bar 1 + bar 2 ### We want to skip this footer ### $ cat table.txt | jc 1:-1 --asciitable -[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] $ cat table.txt | jc 1:4 --asciitable -[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] ``` In this example `1:-1` and `1:4` line slices provide the same output. From e758aa41ef0606b9b07a8a3614267a6e1d9ba24d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 16:49:27 -0800 Subject: [PATCH 27/81] formatting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b94ec57f..01e347a9 100644 --- a/README.md +++ b/README.md @@ -331,12 +331,12 @@ $ cat table.txt ### We want to skip this header ### col1 col2 foo 1 - bar 1 + bar 2 ### We want to skip this footer ### $ cat table.txt | jc 1:-1 --asciitable -[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] $ cat table.txt | jc 1:4 --asciitable -[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"1"}] +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] ``` In this example `1:-1` and `1:4` line slices provide the same output. From 79305a50d0f1b7058bb6c36d33e65a1d0b4a337c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 16:59:15 -0800 Subject: [PATCH 28/81] formatting --- README.md | 8 ++++---- templates/readme_template | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 01e347a9..17a34a5b 100644 --- a/README.md +++ b/README.md @@ -318,10 +318,10 @@ Line slicing is supported using the `START:STOP` syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the output you would like `jc` to convert. -`START` and `STOP` can be positive or negative integers and allow you to -specify how many lines to skip and how many lines to process. Positive -slices are the most memory efficient. Any negative slices will use more -memory. +`START` and `STOP` can be positive or negative integers or blank and allow +you to specify how many lines to skip and how many lines to process. +Positive and blank slices are the most memory efficient. Any negative +integers in the slice will use more memory. For example, to skip the first and last line of the following text, you could express the slice in a couple ways: diff --git a/templates/readme_template b/templates/readme_template index fb17b500..7b2439d2 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -180,10 +180,10 @@ Line slicing is supported using the `START:STOP` syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the output you would like `jc` to convert. -`START` and `STOP` can be positive or negative integers and allow you to -specify how many lines to skip and how many lines to process. Positive -slices are the most memory efficient. Any negative slices will use more -memory. +`START` and `STOP` can be positive or negative integers or blank and allow +you to specify how many lines to skip and how many lines to process. +Positive and blank slices are the most memory efficient. Any negative +integers in the slice will use more memory. For example, to skip the first and last line of the following text, you could express the slice in a couple ways: From d6c665f74b69bd93c144b4d0f4ece28fc3f83310 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 26 Jan 2023 17:06:07 -0800 Subject: [PATCH 29/81] add both negative slice description --- README.md | 21 +++++++++++---------- templates/readme_template | 21 +++++++++++---------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 17a34a5b..363de283 100644 --- a/README.md +++ b/README.md @@ -349,16 +349,17 @@ feature works. Here is a quick breakdown: -| Slice | Description | -|-----------------|------------------------------------------------------------| -| `START:STOP` | lines `START` through `STOP - 1` | -| `START:` | lines `START` through the rest of the output | -| `:STOP` | lines from the beginning through `STOP - 1` | -| `-START:STOP` | `START` lines from the end through `STOP - 1` | -| `START:-STOP` | lines `START` through `STOP` lines from the end | -| `-START:` | `START` lines from the end through the rest of the output | -| `:-STOP` | lines from the beginning through `STOP` lines from the end | -| `:` | all lines | +| Slice | Description | +|----------------|--------------------------------------------------------------| +| `START:STOP` | lines `START` through `STOP - 1` | +| `START:` | lines `START` through the rest of the output | +| `:STOP` | lines from the beginning through `STOP - 1` | +| `-START:STOP` | `START` lines from the end through `STOP - 1` | +| `START:-STOP` | lines `START` through `STOP` lines from the end | +| `-START:-STOP` | `START` lines from the end through `STOP` lines from the end | +| `-START:` | `START` lines from the end through the rest of the output | +| `:-STOP` | lines from the beginning through `STOP` lines from the end | +| `:` | all lines | ### Exit Codes Any fatal errors within `jc` will generate an exit code of `100`, otherwise the diff --git a/templates/readme_template b/templates/readme_template index 7b2439d2..50df6b3f 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -211,16 +211,17 @@ feature works. Here is a quick breakdown: -| Slice | Description | -|-----------------|------------------------------------------------------------| -| `START:STOP` | lines `START` through `STOP - 1` | -| `START:` | lines `START` through the rest of the output | -| `:STOP` | lines from the beginning through `STOP - 1` | -| `-START:STOP` | `START` lines from the end through `STOP - 1` | -| `START:-STOP` | lines `START` through `STOP` lines from the end | -| `-START:` | `START` lines from the end through the rest of the output | -| `:-STOP` | lines from the beginning through `STOP` lines from the end | -| `:` | all lines | +| Slice | Description | +|----------------|--------------------------------------------------------------| +| `START:STOP` | lines `START` through `STOP - 1` | +| `START:` | lines `START` through the rest of the output | +| `:STOP` | lines from the beginning through `STOP - 1` | +| `-START:STOP` | `START` lines from the end through `STOP - 1` | +| `START:-STOP` | lines `START` through `STOP` lines from the end | +| `-START:-STOP` | `START` lines from the end through `STOP` lines from the end | +| `-START:` | `START` lines from the end through the rest of the output | +| `:-STOP` | lines from the beginning through `STOP` lines from the end | +| `:` | all lines | ### Exit Codes Any fatal errors within `jc` will generate an exit code of `100`, otherwise the From 21ee3c0e78818fcb312d2ea5e089ec8d4dec1205 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 08:01:32 -0800 Subject: [PATCH 30/81] clarify slice usage --- README.md | 14 +++++++------- templates/readme_template | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 363de283..74d35c7b 100644 --- a/README.md +++ b/README.md @@ -316,7 +316,7 @@ option. ### Slice Line slicing is supported using the `START:STOP` syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the -output you would like `jc` to convert. +`STDIN` input you would like `jc` to convert. `START` and `STOP` can be positive or negative integers or blank and allow you to specify how many lines to skip and how many lines to process. @@ -340,16 +340,16 @@ $ cat table.txt | jc 1:4 --asciitable ``` In this example `1:-1` and `1:4` line slices provide the same output. -Notice how when using positive integers the index location of `STOP` is -non-inclusive. Positive slices count from the first line of the output -toward the end starting at `0` as the first line. Negative slices count from -the last line toward the beginning starting at `-1` as the last line. This -is also the way [Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) +When using positive integers the index location of `STOP` is non-inclusive. +Positive slices count from the first line of the input toward the end +starting at `0` as the first line. Negative slices count from the last line +toward the beginning starting at `-1` as the last line. This is also the way +[Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) feature works. Here is a quick breakdown: -| Slice | Description | +| Slice Notation | Input lines processed | |----------------|--------------------------------------------------------------| | `START:STOP` | lines `START` through `STOP - 1` | | `START:` | lines `START` through the rest of the output | diff --git a/templates/readme_template b/templates/readme_template index 50df6b3f..c5fbe074 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -178,7 +178,7 @@ option. ### Slice Line slicing is supported using the `START:STOP` syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the -output you would like `jc` to convert. +`STDIN` input you would like `jc` to convert. `START` and `STOP` can be positive or negative integers or blank and allow you to specify how many lines to skip and how many lines to process. @@ -202,16 +202,16 @@ $ cat table.txt | jc 1:4 --asciitable ``` In this example `1:-1` and `1:4` line slices provide the same output. -Notice how when using positive integers the index location of `STOP` is -non-inclusive. Positive slices count from the first line of the output -toward the end starting at `0` as the first line. Negative slices count from -the last line toward the beginning starting at `-1` as the last line. This -is also the way [Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) +When using positive integers the index location of `STOP` is non-inclusive. +Positive slices count from the first line of the input toward the end +starting at `0` as the first line. Negative slices count from the last line +toward the beginning starting at `-1` as the last line. This is also the way +[Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) feature works. Here is a quick breakdown: -| Slice | Description | +| Slice Notation | Input lines processed | |----------------|--------------------------------------------------------------| | `START:STOP` | lines `START` through `STOP - 1` | | `START:` | lines `START` through the rest of the output | From aef0fdb4352cb07c80c385a6814e355aeb34bbc5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 08:02:07 -0800 Subject: [PATCH 31/81] formatting --- README.md | 2 +- templates/readme_template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 74d35c7b..00e27bd5 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ feature works. Here is a quick breakdown: -| Slice Notation | Input lines processed | +| Slice Notation | Input Lines Processed | |----------------|--------------------------------------------------------------| | `START:STOP` | lines `START` through `STOP - 1` | | `START:` | lines `START` through the rest of the output | diff --git a/templates/readme_template b/templates/readme_template index c5fbe074..633ba069 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -211,7 +211,7 @@ feature works. Here is a quick breakdown: -| Slice Notation | Input lines processed | +| Slice Notation | Input Lines Processed | |----------------|--------------------------------------------------------------| | `START:STOP` | lines `START` through `STOP - 1` | | `START:` | lines `START` through the rest of the output | From dc997821f4eaba6d4352516f27104aba85214bed Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 08:12:07 -0800 Subject: [PATCH 32/81] ad jc-api demo link --- README.md | 2 +- templates/readme_template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 00e27bd5..b1757a97 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ > Check out the `jc` Python [package documentation](https://github.com/kellyjonbrazil/jc/tree/master/docs) for developers -> Try the `jc` [web demo](https://jc-web.onrender.com/) +> Try the `jc` [web demo](https://jc-web.onrender.com/) and [REST API demo](https://jc-api.onrender.com) > JC is [now available](https://galaxy.ansible.com/community/general) as an Ansible filter plugin in the `community.general` collection. See this diff --git a/templates/readme_template b/templates/readme_template index 633ba069..465bb456 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -3,7 +3,7 @@ > Check out the `jc` Python [package documentation](https://github.com/kellyjonbrazil/jc/tree/master/docs) for developers -> Try the `jc` [web demo](https://jc-web.onrender.com/) +> Try the `jc` [web demo](https://jc-web.onrender.com/) and [REST API demo](https://jc-api.onrender.com) > JC is [now available](https://galaxy.ansible.com/community/general) as an Ansible filter plugin in the `community.general` collection. See this From 118f98222a1e82654f490b014c4ce7ee187ec9b8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 08:15:53 -0800 Subject: [PATCH 33/81] change restapi link --- README.md | 2 +- templates/readme_template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b1757a97..58d05bbc 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ > Check out the `jc` Python [package documentation](https://github.com/kellyjonbrazil/jc/tree/master/docs) for developers -> Try the `jc` [web demo](https://jc-web.onrender.com/) and [REST API demo](https://jc-api.onrender.com) +> Try the `jc` [web demo](https://jc-web.onrender.com/) and [REST API](https://github.com/kellyjonbrazil/jc-restapi) > JC is [now available](https://galaxy.ansible.com/community/general) as an Ansible filter plugin in the `community.general` collection. See this diff --git a/templates/readme_template b/templates/readme_template index 465bb456..f0ddac9a 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -3,7 +3,7 @@ > Check out the `jc` Python [package documentation](https://github.com/kellyjonbrazil/jc/tree/master/docs) for developers -> Try the `jc` [web demo](https://jc-web.onrender.com/) and [REST API demo](https://jc-api.onrender.com) +> Try the `jc` [web demo](https://jc-web.onrender.com/) and [REST API](https://github.com/kellyjonbrazil/jc-restapi) > JC is [now available](https://galaxy.ansible.com/community/general) as an Ansible filter plugin in the `community.general` collection. See this From 9370b336d84d31a4df57947bccce1beac98768e2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 11:31:16 -0800 Subject: [PATCH 34/81] add slice to man page --- README.md | 2 +- man/jc.1 | 96 +++++++++++++++++++++++++++++++++++--- templates/manpage_template | 89 +++++++++++++++++++++++++++++++++-- templates/readme_template | 2 +- 4 files changed, 176 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 58d05bbc..9b934893 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ toward the beginning starting at `-1` as the last line. This is also the way [Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) feature works. -Here is a quick breakdown: +Here is a breakdown of line slice options: | Slice Notation | Input Lines Processed | |----------------|--------------------------------------------------------------| diff --git a/man/jc.1 b/man/jc.1 index a0a4c126..9127f289 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-01-22 1.23.0 "JSON Convert" +.TH jc 1 2023-01-27 1.23.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS @@ -6,19 +6,19 @@ Standard syntax: .RS -COMMAND | \fBjc\fP [OPTIONS] PARSER +COMMAND | \fBjc\fP [SLICE] [OPTIONS] PARSER -cat FILE | \fBjc\fP [OPTIONS] PARSER +cat FILE | \fBjc\fP [SLICE] [OPTIONS] PARSER -echo STRING | \fBjc\fP [OPTIONS] PARSER +echo STRING | \fBjc\fP [SLICE] [OPTIONS] PARSER .RE Magic syntax: .RS -\fBjc\fP [OPTIONS] COMMAND +\fBjc\fP [SLICE] [OPTIONS] COMMAND -\fBjc\fP [OPTIONS] /proc/ +\fBjc\fP [SLICE] [OPTIONS] /proc/ .RE .SH DESCRIPTION @@ -920,6 +920,11 @@ TOML file parser \fB--url\fP URL string parser +.TP +.B +\fB--ver\fP +Version string parser + .TP .B \fB--vmstat\fP @@ -1034,6 +1039,85 @@ Generate Bash shell completion script \fB-Z\fP, \fB--zsh-comp\fP Generate Zsh shell completion script +.RE +.PP +.B +Slice: +.RS +Line slicing is supported using the \fBSTART:STOP\fP syntax similar to Python +slicing. This allows you to skip lines at the beginning and/or end of the +\fBSTDIN\fP input you would like \fBjc\fP to convert. + +\fBSTART\fP and \fBSTOP\fP can be positive or negative integers or blank and allow +you to specify how many lines to skip and how many lines to process. +Positive and blank slices are the most memory efficient. Any negative +integers in the slice will use more memory. + +For example, to skip the first and last line of the following text, you +could express the slice in a couple ways: + +.RS +.nf +$ cat table.txt + ### We want to skip this header ### + col1 col2 + foo 1 + bar 2 + ### We want to skip this footer ### +$ cat table.txt | jc 1:-1 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] +$ cat table.txt | jc 1:4 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] +.fi +.RE + +In this example \fB1:-1\fP and \fB1:4\fP line slices provide the same output. + +When using positive integers the index location of \fBSTOP\fP is non-inclusive. +Positive slices count from the first line of the input toward the end +starting at \fB0\fP as the first line. Negative slices count from the last line +toward the beginning starting at \fB-1\fP as the last line. This is also the way +Python's slicing feature works. + +Here is a breakdown of line slice options: + +.TP +.B +\fBSTART:STOP\fP +lines \fBSTART\fP through \fBSTOP - 1\fP +.TP +.B +\fBSTART:\fP +lines \fBSTART\fP through the rest of the output +.TP +.B +\fB:STOP\fP +lines from the beginning through \fBSTOP - 1\fP +.TP +.B +\fB-START:STOP\fP +\fBSTART\fP lines from the end through \fBSTOP - 1\fP +.TP +.B +\fBSTART:-STOP\fP +lines \fBSTART\fP through \fBSTOP\fP lines from the end +.TP +.B +\fB-START:-STOP\fP +\fBSTART\fP lines from the end through \fBSTOP\fP lines from the end +.TP +.B +\fB-START:\fP +\fBSTART\fP lines from the end through the rest of the output +.TP +.B +\fB:-STOP\fP +lines from the beginning through \fBSTOP\fP lines from the end +.TP +.B +\fB:\fP +all lines + .SH EXIT CODES Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. diff --git a/templates/manpage_template b/templates/manpage_template index 2536c2a3..a2839a33 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -6,19 +6,19 @@ Standard syntax: .RS -COMMAND | \fBjc\fP [OPTIONS] PARSER +COMMAND | \fBjc\fP [SLICE] [OPTIONS] PARSER -cat FILE | \fBjc\fP [OPTIONS] PARSER +cat FILE | \fBjc\fP [SLICE] [OPTIONS] PARSER -echo STRING | \fBjc\fP [OPTIONS] PARSER +echo STRING | \fBjc\fP [SLICE] [OPTIONS] PARSER .RE Magic syntax: .RS -\fBjc\fP [OPTIONS] COMMAND +\fBjc\fP [SLICE] [OPTIONS] COMMAND -\fBjc\fP [OPTIONS] /proc/ +\fBjc\fP [SLICE] [OPTIONS] /proc/ .RE .SH DESCRIPTION @@ -99,6 +99,85 @@ Generate Bash shell completion script \fB-Z\fP, \fB--zsh-comp\fP Generate Zsh shell completion script +.RE +.PP +.B +Slice: +.RS +Line slicing is supported using the \fBSTART:STOP\fP syntax similar to Python +slicing. This allows you to skip lines at the beginning and/or end of the +\fBSTDIN\fP input you would like \fBjc\fP to convert. + +\fBSTART\fP and \fBSTOP\fP can be positive or negative integers or blank and allow +you to specify how many lines to skip and how many lines to process. +Positive and blank slices are the most memory efficient. Any negative +integers in the slice will use more memory. + +For example, to skip the first and last line of the following text, you +could express the slice in a couple ways: + +.RS +.nf +$ cat table.txt + ### We want to skip this header ### + col1 col2 + foo 1 + bar 2 + ### We want to skip this footer ### +$ cat table.txt | jc 1:-1 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] +$ cat table.txt | jc 1:4 --asciitable +[{"col1":"foo","col2":"1"},{"col1":"bar","col2":"2"}] +.fi +.RE + +In this example \fB1:-1\fP and \fB1:4\fP line slices provide the same output. + +When using positive integers the index location of \fBSTOP\fP is non-inclusive. +Positive slices count from the first line of the input toward the end +starting at \fB0\fP as the first line. Negative slices count from the last line +toward the beginning starting at \fB-1\fP as the last line. This is also the way +Python's slicing feature works. + +Here is a breakdown of line slice options: + +.TP +.B +\fBSTART:STOP\fP +lines \fBSTART\fP through \fBSTOP - 1\fP +.TP +.B +\fBSTART:\fP +lines \fBSTART\fP through the rest of the output +.TP +.B +\fB:STOP\fP +lines from the beginning through \fBSTOP - 1\fP +.TP +.B +\fB-START:STOP\fP +\fBSTART\fP lines from the end through \fBSTOP - 1\fP +.TP +.B +\fBSTART:-STOP\fP +lines \fBSTART\fP through \fBSTOP\fP lines from the end +.TP +.B +\fB-START:-STOP\fP +\fBSTART\fP lines from the end through \fBSTOP\fP lines from the end +.TP +.B +\fB-START:\fP +\fBSTART\fP lines from the end through the rest of the output +.TP +.B +\fB:-STOP\fP +lines from the beginning through \fBSTOP\fP lines from the end +.TP +.B +\fB:\fP +all lines + .SH EXIT CODES Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. diff --git a/templates/readme_template b/templates/readme_template index f0ddac9a..c6daa75e 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -209,7 +209,7 @@ toward the beginning starting at `-1` as the last line. This is also the way [Python's slicing](https://stackoverflow.com/questions/509211/understanding-slicing) feature works. -Here is a quick breakdown: +Here is a breakdown of line slice options: | Slice Notation | Input Lines Processed | |----------------|--------------------------------------------------------------| From a8d97a25210e36b0d43335ae4b2491ec2335fcf1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 11:41:21 -0800 Subject: [PATCH 35/81] add slice info to help and doc update --- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 3 ++- jc/cli_data.py | 13 ++++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 74dcf97a..18b826b7 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 8823e30e..0e199db7 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -103,7 +103,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -282,6 +282,7 @@ _jc() { '--upower:`upower` command parser' '--uptime:`uptime` command parser' '--url:URL string parser' + '--ver:Version string parser' '--vmstat:`vmstat` command parser' '--vmstat-s:`vmstat` command streaming parser' '--w:`w` command parser' diff --git a/jc/cli_data.py b/jc/cli_data.py index 426dc3ab..736ba512 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -63,17 +63,17 @@ Usage: Standard syntax: - COMMAND | jc [OPTIONS] PARSER + COMMAND | jc [SLICE] [OPTIONS] PARSER - cat FILE | jc [OPTIONS] PARSER + cat FILE | jc [SLICE] [OPTIONS] PARSER - echo STRING | jc [OPTIONS] PARSER + echo STRING | jc [SLICE] [OPTIONS] PARSER Magic syntax: - jc [OPTIONS] COMMAND + jc [SLICE] [OPTIONS] COMMAND - jc [OPTIONS] /proc/ + jc [SLICE] [OPTIONS] /proc/ Parsers: ''' @@ -88,6 +88,9 @@ Examples: $ jc --pretty dig www.google.com $ jc --pretty /proc/meminfo + Line Slicing: + $ cat file.csv | jc :102 --csv # parse first 100 records + Parser Documentation: $ jc --help --dig From df7aee9e4b08fe1584a8e810d53f1abb4c5cb5b8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 14:47:08 -0800 Subject: [PATCH 36/81] doc update --- README.md | 4 +- man/jc.1 | 108 +++++++++++++++++++++++++++---------- templates/manpage_template | 108 +++++++++++++++++++++++++++---------- templates/readme_template | 4 +- 4 files changed, 166 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 9b934893..a31bcdb5 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,8 @@ $ jc dig example.com | jq -r '.[].answer[].data' 93.184.216.34 ``` -`jc` can also be used as a python library. In this case the output will be -a python dictionary, a list of dictionaries, or even a +`jc` can also be used as a python library. In this case the returned value +will be a python dictionary, a list of dictionaries, or even a [lazy iterable of dictionaries](#using-streaming-parsers-as-python-modules) instead of JSON: ```python diff --git a/man/jc.1 b/man/jc.1 index 9127f289..61cffdb7 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,6 +1,7 @@ .TH jc 1 2023-01-27 1.23.0 "JSON Convert" .SH NAME -\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings +\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, +and strings .SH SYNOPSIS Standard syntax: @@ -22,7 +23,13 @@ Magic syntax: .RE .SH DESCRIPTION -\fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) +\fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings +for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and +outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. +Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the +command to be converted. Options can be passed to \fBjc\fP immediately before +the command is given. (Note: "Magic" syntax does not support shell builtins or +command aliases) .SH OPTIONS .B @@ -989,7 +996,8 @@ About \fBjc\fP (JSON or YAML output) .TP .B \fB-C\fP, \fB--force-color\fP -Force color output even when using pipes (overrides \fB-m\fP and the \fBNO_COLOR\fP env variable) +Force color output even when using pipes (overrides \fB-m\fP and the +\fBNO_COLOR\fP env variable) .TP .B \fB-d\fP, \fB--debug\fP @@ -997,7 +1005,8 @@ Debug - show traceback (use \fB-dd\fP for verbose traceback) .TP .B \fB-h\fP, \fB--help\fP -Help (\fB--help --parser_name\fP for parser documentation). Use twice to show hidden parsers (e.g. \fB-hh\fP) +Help (\fB--help --parser_name\fP for parser documentation). Use twice to show +\hidden parsers (e.g. \fB-hh\fP) .TP .B \fB-m\fP, \fB--monochrome\fP @@ -1005,7 +1014,8 @@ Monochrome output .TP .B \fB-M\fP, \fB--meta-out\fP -Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. +Add metadata to output including timestamp, parser name, magic command, magic +command exit code, etc. .TP .B \fB-p\fP, \fB--pretty\fP @@ -1013,11 +1023,13 @@ Pretty print output .TP .B \fB-q\fP, \fB--quiet\fP -Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming parser errors) +Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming +parser errors) .TP .B \fB-r\fP, \fB--raw\fP -Raw output. Provides more literal output, typically with string values and no additional semantic processing +Raw output. Provides more literal output, typically with string values and no +additional semantic processing .TP .B \fB-u\fP, \fB--unbuffer\fP @@ -1048,8 +1060,8 @@ Line slicing is supported using the \fBSTART:STOP\fP syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the \fBSTDIN\fP input you would like \fBjc\fP to convert. -\fBSTART\fP and \fBSTOP\fP can be positive or negative integers or blank and allow -you to specify how many lines to skip and how many lines to process. +\fBSTART\fP and \fBSTOP\fP can be positive or negative integers or blank and +allow you to specify how many lines to skip and how many lines to process. Positive and blank slices are the most memory efficient. Any negative integers in the slice will use more memory. @@ -1119,9 +1131,13 @@ lines from the beginning through \fBSTOP\fP lines from the end all lines .SH EXIT CODES -Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. +Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, +otherwise the exit code will be \fB0\fP. -When using the "magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP. +When using the "magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store +the exit code of the program being parsed and add it to the \fBjc\fP exit code. +This way it is easier to determine if an error was from the parsed program or +\fBjc\fP. Consider the following examples using \fBifconfig\fP: @@ -1136,9 +1152,9 @@ ifconfig exit code = \fB1\fP, jc exit code = \fB100\fP, combined exit code = \fB .RE When using the "magic" syntax you can also retrieve the exit code of the called -program by using the \fB--meta-out\fP or \fB-M\fP option. This will append a \fB_jc_meta\fP -object to the output that will include the magic command information, including -the exit code. +program by using the \fB--meta-out\fP or \fB-M\fP option. This will append a +\fB_jc_meta\fP object to the output that will include the magic command +information, including the exit code. Here is an example with \fBping\fP: .RS @@ -1180,11 +1196,16 @@ $ echo $? \fBCustom Colors\fP -You can specify custom colors via the \fBJC_COLORS\fP environment variable. The \fBJC_COLORS\fP environment variable takes four comma separated string values in the following format: +You can specify custom colors via the \fBJC_COLORS\fP environment variable. The +\fBJC_COLORS\fP environment variable takes four comma separated string values in +the following format: JC_COLORS=,,, -Where colors are: \fBblack\fP, \fBred\fP, \fBgreen\fP, \fByellow\fP, \fBblue\fP, \fBmagenta\fP, \fBcyan\fP, \fBgray\fP, \fBbrightblack\fP, \fBbrightred\fP, \fBbrightgreen\fP, \fBbrightyellow\fP, \fBbrightblue\fP, \fBbrightmagenta\fP, \fBbrightcyan\fP, \fBwhite\fP, or \fBdefault\fP +Where colors are: \fBblack\fP, \fBred\fP, \fBgreen\fP, \fByellow\fP, \fBblue\fP, +\fBmagenta\fP, \fBcyan\fP, \fBgray\fP, \fBbrightblack\fP, \fBbrightred\fP, +\fBbrightgreen\fP, \fBbrightyellow\fP, \fBbrightblue\fP, \fBbrightmagenta\fP, +\fBbrightcyan\fP, \fBwhite\fP, or \fBdefault\fP For example, to set to the default colors: @@ -1198,10 +1219,20 @@ JC_COLORS=default,default,default,default \fBDisable Color Output\fP -You can set the \fBNO_COLOR\fP environment variable to any value to disable color output in \fBjc\fP. Note that using the \fB-C\fP option to force color output will override both the \fBNO_COLOR\fP environment variable and the \fB-m\fP option. +You can set the \fBNO_COLOR\fP environment variable to any value to disable +color output in \fBjc\fP. Note that using the \fB-C\fP option to force color +output will override both the \fBNO_COLOR\fP environment variable and the +\fB-m\fP option. .SH STREAMING PARSERS -Most parsers load all of the data from \fBSTDIN\fP, parse it, then output the entire JSON document serially. There are some streaming parsers (e.g. \fBls-s\fP, \fBping-s\fP, etc.) that immediately start processing and outputting the data line-by-line as JSON Lines (aka NDJSON) while it is being received from \fBSTDIN\fP. This can significantly reduce the amount of memory required to parse large amounts of command output (e.g. \fBls -lR /\fP) and can sometimes process the data more quickly. Streaming parsers have slightly different behavior than standard parsers as outlined below. +Most parsers load all of the data from \fBSTDIN\fP, parse it, then output the +entire JSON document serially. There are some streaming parsers (e.g. +\fBls-s\fP, \fBping-s\fP, etc.) that immediately start processing and outputting +the data line-by-line as JSON Lines (aka NDJSON) while it is being received from +\fBSTDIN\fP. This can significantly reduce the amount of memory required to +parse large amounts of command output (e.g. \fBls -lR /\fP) and can sometimes +process the data more quickly. Streaming parsers have slightly different +behavior than standard parsers as outlined below. .RS Note: Streaming parsers cannot be used with the "magic" syntax @@ -1209,7 +1240,14 @@ Note: Streaming parsers cannot be used with the "magic" syntax \fBIgnoring Errors\fP -You may want to ignore parsing errors when using streaming parsers since these may be used in long-lived processing pipelines and errors can break the pipe. To ignore parsing errors, use the \fB-qq\fP cli option. This will add a \fB_jc_meta\fP object to the JSON output with a \fBsuccess\fP attribute. If \fBsuccess\fP is \fBtrue\fP, then there were no issues parsing the line. If \fBsuccess\fP is \fBfalse\fP, then a parsing issue was found and \fBerror\fP and \fBline\fP fields will be added to include a short error description and the contents of the unparsable line, respectively: +You may want to ignore parsing errors when using streaming parsers since these +may be used in long-lived processing pipelines and errors can break the pipe. To +ignore parsing errors, use the \fB-qq\fP cli option. This will add a +\fB_jc_meta\fP object to the JSON output with a \fBsuccess\fP attribute. If +\fBsuccess\fP is \fBtrue\fP, then there were no issues parsing the line. If +\fBsuccess\fP is \fBfalse\fP, then a parsing issue was found and \fBerror\fP and +\fBline\fP fields will be added to include a short error description and the +contents of the unparsable line, respectively: .RS Successfully parsed line with \fB-qq\fP option: @@ -1240,7 +1278,11 @@ Unsuccessfully parsed line with \fB-qq\fP option: .RE \fBUnbuffering Output\fP -Most operating systems will buffer output that is being piped from process to process. The buffer is usually around 4KB. When viewing the output in the terminal the OS buffer is not engaged so output is immediately displayed on the screen. When piping multiple processes together, though, it may seem as if the output is hanging when the input data is very slow (e.g. \fBping\fP): +Most operating systems will buffer output that is being piped from process to +process. The buffer is usually around 4KB. When viewing the output in the +terminal the OS buffer is not engaged so output is immediately displayed on the +screen. When piping multiple processes together, though, it may seem as if the +output is hanging when the input data is very slow (e.g. \fBping\fP): .RS .nf @@ -1249,7 +1291,9 @@ $ ping 1.1.1.1 | jc \fB--ping-s\fP | jq .fi .RE -This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in this example. To display the data on the terminal in realtime, you can disable the buffer with the \fB-u\fP (unbuffer) cli option: +This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in +this example. To display the data on the terminal in realtime, you can disable +the buffer with the \fB-u\fP (unbuffer) cli option: .RS .nf @@ -1263,7 +1307,8 @@ Note: Unbuffered output can be slower for large data streams. .RE .SH CUSTOM PARSERS -Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": +Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your +local "App data directory": .RS .nf @@ -1273,11 +1318,16 @@ Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your .fi .RE -Local parser plugins are standard python module files. Use the \fBjc/parsers/foo.py\fP or \fBjc/parsers/foo_s.py\fP (streaming) parser as a template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. +Local parser plugins are standard python module files. Use the +\fBjc/parsers/foo.py\fP or \fBjc/parsers/foo_s.py\fP (streaming) parser as a +template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. -Local plugin filenames must be valid python module names and therefore must start with a letter and consist entirely of alphanumerics and underscores. Local plugins may override default parsers. +Local plugin filenames must be valid python module names and therefore must +start with a letter and consist entirely of alphanumerics and underscores. Local +plugins may override default parsers. -Note: The application data directory follows the XDG Base Directory Specification +Note: The application data directory follows the \fBXDG Base Directory +Specification\fP .SH CAVEATS \fBLocale\fP @@ -1302,9 +1352,13 @@ escape sequences if the \fBC\fP locale does not support UTF-8 encoding. \fBTimezones\fP -Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on). +Some parsers have calculated epoch timestamp fields added to the output. Unless +a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. +based on the local timezone of the system the \fBjc\fP parser was run on). -If a UTC timezone can be detected in the text of the command output, the timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. (e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps. +If a UTC timezone can be detected in the text of the command output, the +timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. +(e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps. .SH EXAMPLES Standard Syntax: diff --git a/templates/manpage_template b/templates/manpage_template index a2839a33..05b0f782 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -1,6 +1,7 @@ .TH jc 1 {{ today }} {{ jc.version}} "JSON Convert" .SH NAME -\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings +\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, +and strings .SH SYNOPSIS Standard syntax: @@ -22,7 +23,13 @@ Magic syntax: .RE .SH DESCRIPTION -\fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) +\fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings +for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and +outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. +Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the +command to be converted. Options can be passed to \fBjc\fP immediately before +the command is given. (Note: "Magic" syntax does not support shell builtins or +command aliases) .SH OPTIONS .B @@ -49,7 +56,8 @@ About \fBjc\fP (JSON or YAML output) .TP .B \fB-C\fP, \fB--force-color\fP -Force color output even when using pipes (overrides \fB-m\fP and the \fBNO_COLOR\fP env variable) +Force color output even when using pipes (overrides \fB-m\fP and the +\fBNO_COLOR\fP env variable) .TP .B \fB-d\fP, \fB--debug\fP @@ -57,7 +65,8 @@ Debug - show traceback (use \fB-dd\fP for verbose traceback) .TP .B \fB-h\fP, \fB--help\fP -Help (\fB--help --parser_name\fP for parser documentation). Use twice to show hidden parsers (e.g. \fB-hh\fP) +Help (\fB--help --parser_name\fP for parser documentation). Use twice to show +\hidden parsers (e.g. \fB-hh\fP) .TP .B \fB-m\fP, \fB--monochrome\fP @@ -65,7 +74,8 @@ Monochrome output .TP .B \fB-M\fP, \fB--meta-out\fP -Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. +Add metadata to output including timestamp, parser name, magic command, magic +command exit code, etc. .TP .B \fB-p\fP, \fB--pretty\fP @@ -73,11 +83,13 @@ Pretty print output .TP .B \fB-q\fP, \fB--quiet\fP -Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming parser errors) +Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming +parser errors) .TP .B \fB-r\fP, \fB--raw\fP -Raw output. Provides more literal output, typically with string values and no additional semantic processing +Raw output. Provides more literal output, typically with string values and no +additional semantic processing .TP .B \fB-u\fP, \fB--unbuffer\fP @@ -108,8 +120,8 @@ Line slicing is supported using the \fBSTART:STOP\fP syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the \fBSTDIN\fP input you would like \fBjc\fP to convert. -\fBSTART\fP and \fBSTOP\fP can be positive or negative integers or blank and allow -you to specify how many lines to skip and how many lines to process. +\fBSTART\fP and \fBSTOP\fP can be positive or negative integers or blank and +allow you to specify how many lines to skip and how many lines to process. Positive and blank slices are the most memory efficient. Any negative integers in the slice will use more memory. @@ -179,9 +191,13 @@ lines from the beginning through \fBSTOP\fP lines from the end all lines .SH EXIT CODES -Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. +Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, +otherwise the exit code will be \fB0\fP. -When using the "magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP. +When using the "magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store +the exit code of the program being parsed and add it to the \fBjc\fP exit code. +This way it is easier to determine if an error was from the parsed program or +\fBjc\fP. Consider the following examples using \fBifconfig\fP: @@ -196,9 +212,9 @@ ifconfig exit code = \fB1\fP, jc exit code = \fB100\fP, combined exit code = \fB .RE When using the "magic" syntax you can also retrieve the exit code of the called -program by using the \fB--meta-out\fP or \fB-M\fP option. This will append a \fB_jc_meta\fP -object to the output that will include the magic command information, including -the exit code. +program by using the \fB--meta-out\fP or \fB-M\fP option. This will append a +\fB_jc_meta\fP object to the output that will include the magic command +information, including the exit code. Here is an example with \fBping\fP: .RS @@ -240,11 +256,16 @@ $ echo $? \fBCustom Colors\fP -You can specify custom colors via the \fBJC_COLORS\fP environment variable. The \fBJC_COLORS\fP environment variable takes four comma separated string values in the following format: +You can specify custom colors via the \fBJC_COLORS\fP environment variable. The +\fBJC_COLORS\fP environment variable takes four comma separated string values in +the following format: JC_COLORS=,,, -Where colors are: \fBblack\fP, \fBred\fP, \fBgreen\fP, \fByellow\fP, \fBblue\fP, \fBmagenta\fP, \fBcyan\fP, \fBgray\fP, \fBbrightblack\fP, \fBbrightred\fP, \fBbrightgreen\fP, \fBbrightyellow\fP, \fBbrightblue\fP, \fBbrightmagenta\fP, \fBbrightcyan\fP, \fBwhite\fP, or \fBdefault\fP +Where colors are: \fBblack\fP, \fBred\fP, \fBgreen\fP, \fByellow\fP, \fBblue\fP, +\fBmagenta\fP, \fBcyan\fP, \fBgray\fP, \fBbrightblack\fP, \fBbrightred\fP, +\fBbrightgreen\fP, \fBbrightyellow\fP, \fBbrightblue\fP, \fBbrightmagenta\fP, +\fBbrightcyan\fP, \fBwhite\fP, or \fBdefault\fP For example, to set to the default colors: @@ -258,10 +279,20 @@ JC_COLORS=default,default,default,default \fBDisable Color Output\fP -You can set the \fBNO_COLOR\fP environment variable to any value to disable color output in \fBjc\fP. Note that using the \fB-C\fP option to force color output will override both the \fBNO_COLOR\fP environment variable and the \fB-m\fP option. +You can set the \fBNO_COLOR\fP environment variable to any value to disable +color output in \fBjc\fP. Note that using the \fB-C\fP option to force color +output will override both the \fBNO_COLOR\fP environment variable and the +\fB-m\fP option. .SH STREAMING PARSERS -Most parsers load all of the data from \fBSTDIN\fP, parse it, then output the entire JSON document serially. There are some streaming parsers (e.g. \fBls-s\fP, \fBping-s\fP, etc.) that immediately start processing and outputting the data line-by-line as JSON Lines (aka NDJSON) while it is being received from \fBSTDIN\fP. This can significantly reduce the amount of memory required to parse large amounts of command output (e.g. \fBls -lR /\fP) and can sometimes process the data more quickly. Streaming parsers have slightly different behavior than standard parsers as outlined below. +Most parsers load all of the data from \fBSTDIN\fP, parse it, then output the +entire JSON document serially. There are some streaming parsers (e.g. +\fBls-s\fP, \fBping-s\fP, etc.) that immediately start processing and outputting +the data line-by-line as JSON Lines (aka NDJSON) while it is being received from +\fBSTDIN\fP. This can significantly reduce the amount of memory required to +parse large amounts of command output (e.g. \fBls -lR /\fP) and can sometimes +process the data more quickly. Streaming parsers have slightly different +behavior than standard parsers as outlined below. .RS Note: Streaming parsers cannot be used with the "magic" syntax @@ -269,7 +300,14 @@ Note: Streaming parsers cannot be used with the "magic" syntax \fBIgnoring Errors\fP -You may want to ignore parsing errors when using streaming parsers since these may be used in long-lived processing pipelines and errors can break the pipe. To ignore parsing errors, use the \fB-qq\fP cli option. This will add a \fB_jc_meta\fP object to the JSON output with a \fBsuccess\fP attribute. If \fBsuccess\fP is \fBtrue\fP, then there were no issues parsing the line. If \fBsuccess\fP is \fBfalse\fP, then a parsing issue was found and \fBerror\fP and \fBline\fP fields will be added to include a short error description and the contents of the unparsable line, respectively: +You may want to ignore parsing errors when using streaming parsers since these +may be used in long-lived processing pipelines and errors can break the pipe. To +ignore parsing errors, use the \fB-qq\fP cli option. This will add a +\fB_jc_meta\fP object to the JSON output with a \fBsuccess\fP attribute. If +\fBsuccess\fP is \fBtrue\fP, then there were no issues parsing the line. If +\fBsuccess\fP is \fBfalse\fP, then a parsing issue was found and \fBerror\fP and +\fBline\fP fields will be added to include a short error description and the +contents of the unparsable line, respectively: .RS Successfully parsed line with \fB-qq\fP option: @@ -300,7 +338,11 @@ Unsuccessfully parsed line with \fB-qq\fP option: .RE \fBUnbuffering Output\fP -Most operating systems will buffer output that is being piped from process to process. The buffer is usually around 4KB. When viewing the output in the terminal the OS buffer is not engaged so output is immediately displayed on the screen. When piping multiple processes together, though, it may seem as if the output is hanging when the input data is very slow (e.g. \fBping\fP): +Most operating systems will buffer output that is being piped from process to +process. The buffer is usually around 4KB. When viewing the output in the +terminal the OS buffer is not engaged so output is immediately displayed on the +screen. When piping multiple processes together, though, it may seem as if the +output is hanging when the input data is very slow (e.g. \fBping\fP): .RS .nf @@ -309,7 +351,9 @@ $ ping 1.1.1.1 | jc \fB--ping-s\fP | jq .fi .RE -This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in this example. To display the data on the terminal in realtime, you can disable the buffer with the \fB-u\fP (unbuffer) cli option: +This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in +this example. To display the data on the terminal in realtime, you can disable +the buffer with the \fB-u\fP (unbuffer) cli option: .RS .nf @@ -323,7 +367,8 @@ Note: Unbuffered output can be slower for large data streams. .RE .SH CUSTOM PARSERS -Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": +Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your +local "App data directory": .RS .nf @@ -333,11 +378,16 @@ Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your .fi .RE -Local parser plugins are standard python module files. Use the \fBjc/parsers/foo.py\fP or \fBjc/parsers/foo_s.py\fP (streaming) parser as a template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. +Local parser plugins are standard python module files. Use the +\fBjc/parsers/foo.py\fP or \fBjc/parsers/foo_s.py\fP (streaming) parser as a +template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. -Local plugin filenames must be valid python module names and therefore must start with a letter and consist entirely of alphanumerics and underscores. Local plugins may override default parsers. +Local plugin filenames must be valid python module names and therefore must +start with a letter and consist entirely of alphanumerics and underscores. Local +plugins may override default parsers. -Note: The application data directory follows the XDG Base Directory Specification +Note: The application data directory follows the \fBXDG Base Directory +Specification\fP .SH CAVEATS \fBLocale\fP @@ -362,9 +412,13 @@ escape sequences if the \fBC\fP locale does not support UTF-8 encoding. \fBTimezones\fP -Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on). +Some parsers have calculated epoch timestamp fields added to the output. Unless +a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. +based on the local timezone of the system the \fBjc\fP parser was run on). -If a UTC timezone can be detected in the text of the command output, the timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. (e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps. +If a UTC timezone can be detected in the text of the command output, the +timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. +(e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps. .SH EXAMPLES Standard Syntax: diff --git a/templates/readme_template b/templates/readme_template index c6daa75e..b3d72609 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -44,8 +44,8 @@ $ jc dig example.com | jq -r '.[].answer[].data' 93.184.216.34 ``` -`jc` can also be used as a python library. In this case the output will be -a python dictionary, a list of dictionaries, or even a +`jc` can also be used as a python library. In this case the returned value +will be a python dictionary, a list of dictionaries, or even a [lazy iterable of dictionaries](#using-streaming-parsers-as-python-modules) instead of JSON: ```python From 0679951d4a3c355b6f89499ef971e25d15039b88 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 15:17:41 -0800 Subject: [PATCH 37/81] add tag --- README.md | 2 ++ templates/readme_template | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index a31bcdb5..b1ecc390 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ option. ### Parsers + | Argument | Command or Filetype | Documentation | |-------------------|---------------------------------------------------------|----------------------------------------------------------------------------| | ` --acpi` | `acpi` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/acpi) | @@ -312,6 +313,7 @@ option. | `-y` | `--yaml-out` | YAML output | | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | + ### Slice Line slicing is supported using the `START:STOP` syntax similar to Python diff --git a/templates/readme_template b/templates/readme_template index b3d72609..a8ba7568 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -152,6 +152,7 @@ option. ### Parsers + | Argument | Command or Filetype | Documentation | |-------------------|---------------------------------------------------------|----------------------------------------------------------------------------|{% for parser in parsers %} | `{{ "{:>15}".format(parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<74}".format("[details](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} @@ -174,6 +175,7 @@ option. | `-y` | `--yaml-out` | YAML output | | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | + ### Slice Line slicing is supported using the `START:STOP` syntax similar to Python From 1c76caf59b68f0ac6797bc9227371d2a3283e849 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 15:22:36 -0800 Subject: [PATCH 38/81] formatting --- README.md | 2 ++ templates/readme_template | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index b1ecc390..39ebb2a4 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ option. ### Parsers + | Argument | Command or Filetype | Documentation | |-------------------|---------------------------------------------------------|----------------------------------------------------------------------------| | ` --acpi` | `acpi` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/acpi) | @@ -313,6 +314,7 @@ option. | `-y` | `--yaml-out` | YAML output | | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | + ### Slice diff --git a/templates/readme_template b/templates/readme_template index a8ba7568..aba62583 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -153,6 +153,7 @@ option. ### Parsers + | Argument | Command or Filetype | Documentation | |-------------------|---------------------------------------------------------|----------------------------------------------------------------------------|{% for parser in parsers %} | `{{ "{:>15}".format(parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<74}".format("[details](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} @@ -175,6 +176,7 @@ option. | `-y` | `--yaml-out` | YAML output | | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | + ### Slice From bc4738e9006743324c46227c2e34c20cbf751917 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 15:58:01 -0800 Subject: [PATCH 39/81] don't quote spaces in command arguments --- README.md | 274 +++++++++++++++++++------------------- templates/readme_template | 6 +- 2 files changed, 136 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 39ebb2a4..3e3b6058 100644 --- a/README.md +++ b/README.md @@ -152,149 +152,147 @@ option. ### Parsers - - | Argument | Command or Filetype | Documentation | |-------------------|---------------------------------------------------------|----------------------------------------------------------------------------| -| ` --acpi` | `acpi` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/acpi) | -| ` --airport` | `airport -I` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/airport) | -| ` --airport-s` | `airport -s` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/airport_s) | -| ` --arp` | `arp` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/arp) | -| ` --asciitable` | ASCII and Unicode table parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable) | -| ` --asciitable-m` | multi-line ASCII and Unicode table parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m) | -| ` --blkid` | `blkid` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid) | -| ` --cbt` | `cbt` (Google Bigtable) command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cbt) | -| ` --cef` | CEF string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef) | -| ` --cef-s` | CEF string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef_s) | -| ` --chage` | `chage --list` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/chage) | -| ` --cksum` | `cksum` and `sum` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum) | -| ` --clf` | Common and Combined Log Format file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/clf) | -| ` --clf-s` | Common and Combined Log Format file streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/clf_s) | -| ` --crontab` | `crontab` command and file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab) | -| ` --crontab-u` | `crontab` file parser with user support | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab_u) | -| ` --csv` | CSV file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/csv) | -| ` --csv-s` | CSV file streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/csv_s) | -| ` --date` | `date` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/date) | -| ` --datetime-iso` | ISO 8601 Datetime string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/datetime_iso) | -| ` --df` | `df` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/df) | -| ` --dig` | `dig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dig) | -| ` --dir` | `dir` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dir) | -| ` --dmidecode` | `dmidecode` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dmidecode) | -| ` --dpkg-l` | `dpkg -l` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dpkg_l) | -| ` --du` | `du` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/du) | +| `--acpi` | `acpi` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/acpi) | +| `--airport` | `airport -I` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/airport) | +| `--airport-s` | `airport -s` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/airport_s) | +| `--arp` | `arp` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/arp) | +| `--asciitable` | ASCII and Unicode table parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable) | +| `--asciitable-m` | multi-line ASCII and Unicode table parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m) | +| `--blkid` | `blkid` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid) | +| `--cbt` | `cbt` (Google Bigtable) command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cbt) | +| `--cef` | CEF string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef) | +| `--cef-s` | CEF string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef_s) | +| `--chage` | `chage --list` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/chage) | +| `--cksum` | `cksum` and `sum` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum) | +| `--clf` | Common and Combined Log Format file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/clf) | +| `--clf-s` | Common and Combined Log Format file streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/clf_s) | +| `--crontab` | `crontab` command and file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab) | +| `--crontab-u` | `crontab` file parser with user support | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab_u) | +| `--csv` | CSV file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/csv) | +| `--csv-s` | CSV file streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/csv_s) | +| `--date` | `date` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/date) | +| `--datetime-iso` | ISO 8601 Datetime string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/datetime_iso) | +| `--df` | `df` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/df) | +| `--dig` | `dig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dig) | +| `--dir` | `dir` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dir) | +| `--dmidecode` | `dmidecode` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dmidecode) | +| `--dpkg-l` | `dpkg -l` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/dpkg_l) | +| `--du` | `du` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/du) | | `--email-address` | Email Address string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/email_address) | -| ` --env` | `env` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/env) | -| ` --file` | `file` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/file) | -| ` --findmnt` | `findmnt` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/findmnt) | -| ` --finger` | `finger` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/finger) | -| ` --free` | `free` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/free) | -| ` --fstab` | `/etc/fstab` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/fstab) | -| ` --git-log` | `git log` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/git_log) | -| ` --git-log-s` | `git log` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/git_log_s) | +| `--env` | `env` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/env) | +| `--file` | `file` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/file) | +| `--findmnt` | `findmnt` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/findmnt) | +| `--finger` | `finger` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/finger) | +| `--free` | `free` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/free) | +| `--fstab` | `/etc/fstab` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/fstab) | +| `--git-log` | `git log` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/git_log) | +| `--git-log-s` | `git log` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/git_log_s) | | `--git-ls-remote` | `git ls-remote` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/git_ls_remote) | -| ` --gpg` | `gpg --with-colons` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/gpg) | -| ` --group` | `/etc/group` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/group) | -| ` --gshadow` | `/etc/gshadow` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/gshadow) | -| ` --hash` | `hash` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hash) | -| ` --hashsum` | hashsum command parser (`md5sum`, `shasum`, etc.) | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hashsum) | -| ` --hciconfig` | `hciconfig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hciconfig) | -| ` --history` | `history` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/history) | -| ` --hosts` | `/etc/hosts` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hosts) | -| ` --id` | `id` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/id) | -| ` --ifconfig` | `ifconfig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ifconfig) | -| ` --ini` | INI file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini) | -| ` --ini-dup` | INI with duplicate key file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini_dup) | -| ` --iostat` | `iostat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat) | -| ` --iostat-s` | `iostat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat_s) | -| ` --ip-address` | IPv4 and IPv6 Address string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address) | -| ` --iptables` | `iptables` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables) | -| ` --iw-scan` | `iw dev [device] scan` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan) | -| ` --iwconfig` | `iwconfig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iwconfig) | -| ` --jar-manifest` | Java MANIFEST.MF file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest) | -| ` --jobs` | `jobs` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jobs) | -| ` --jwt` | JWT string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jwt) | -| ` --kv` | Key/Value file and string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv) | -| ` --last` | `last` and `lastb` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/last) | -| ` --ls` | `ls` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls) | -| ` --ls-s` | `ls` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls_s) | -| ` --lsblk` | `lsblk` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsblk) | -| ` --lsmod` | `lsmod` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsmod) | -| ` --lsof` | `lsof` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsof) | -| ` --lspci` | `lspci -mmv` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lspci) | -| ` --lsusb` | `lsusb` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsusb) | -| ` --m3u` | M3U and M3U8 file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/m3u) | -| ` --mdadm` | `mdadm` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mdadm) | -| ` --mount` | `mount` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mount) | -| ` --mpstat` | `mpstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat) | -| ` --mpstat-s` | `mpstat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat_s) | -| ` --netstat` | `netstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/netstat) | -| ` --nmcli` | `nmcli` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/nmcli) | -| ` --ntpq` | `ntpq -p` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq) | -| ` --openvpn` | openvpn-status.log file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/openvpn) | -| ` --os-prober` | `os-prober` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/os_prober) | -| ` --passwd` | `/etc/passwd` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/passwd) | -| ` --pci-ids` | `pci.ids` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pci_ids) | -| ` --pgpass` | PostgreSQL password file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pgpass) | -| ` --pidstat` | `pidstat -H` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat) | -| ` --pidstat-s` | `pidstat -H` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat_s) | -| ` --ping` | `ping` and `ping6` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ping) | -| ` --ping-s` | `ping` and `ping6` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s) | -| ` --pip-list` | `pip list` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list) | -| ` --pip-show` | `pip show` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_show) | -| ` --plist` | PLIST file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/plist) | -| ` --postconf` | `postconf -M` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/postconf) | -| ` --proc` | `/proc/` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/proc) | -| ` --ps` | `ps` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ps) | -| ` --route` | `route` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/route) | -| ` --rpm-qi` | `rpm -qi` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/rpm_qi) | -| ` --rsync` | `rsync` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync) | -| ` --rsync-s` | `rsync` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync_s) | -| ` --semver` | Semantic Version string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/semver) | -| ` --sfdisk` | `sfdisk` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sfdisk) | -| ` --shadow` | `/etc/shadow` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/shadow) | -| ` --ss` | `ss` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ss) | -| ` --ssh-conf` | `ssh` config file and `ssh -G` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ssh_conf) | -| ` --sshd-conf` | `sshd` config file and `sshd -T` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sshd_conf) | -| ` --stat` | `stat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat) | -| ` --stat-s` | `stat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s) | -| ` --sysctl` | `sysctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sysctl) | -| ` --syslog` | Syslog RFC 5424 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog) | -| ` --syslog-s` | Syslog RFC 5424 string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_s) | -| ` --syslog-bsd` | Syslog RFC 3164 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd) | -| ` --syslog-bsd-s` | Syslog RFC 3164 string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd_s) | -| ` --systemctl` | `systemctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl) | -| ` --systemctl-lj` | `systemctl list-jobs` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_lj) | -| ` --systemctl-ls` | `systemctl list-sockets` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_ls) | +| `--gpg` | `gpg --with-colons` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/gpg) | +| `--group` | `/etc/group` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/group) | +| `--gshadow` | `/etc/gshadow` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/gshadow) | +| `--hash` | `hash` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hash) | +| `--hashsum` | hashsum command parser (`md5sum`, `shasum`, etc.) | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hashsum) | +| `--hciconfig` | `hciconfig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hciconfig) | +| `--history` | `history` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/history) | +| `--hosts` | `/etc/hosts` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/hosts) | +| `--id` | `id` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/id) | +| `--ifconfig` | `ifconfig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ifconfig) | +| `--ini` | INI file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini) | +| `--ini-dup` | INI with duplicate key file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini_dup) | +| `--iostat` | `iostat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat) | +| `--iostat-s` | `iostat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat_s) | +| `--ip-address` | IPv4 and IPv6 Address string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address) | +| `--iptables` | `iptables` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables) | +| `--iw-scan` | `iw dev [device] scan` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan) | +| `--iwconfig` | `iwconfig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iwconfig) | +| `--jar-manifest` | Java MANIFEST.MF file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest) | +| `--jobs` | `jobs` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jobs) | +| `--jwt` | JWT string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jwt) | +| `--kv` | Key/Value file and string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv) | +| `--last` | `last` and `lastb` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/last) | +| `--ls` | `ls` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls) | +| `--ls-s` | `ls` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls_s) | +| `--lsblk` | `lsblk` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsblk) | +| `--lsmod` | `lsmod` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsmod) | +| `--lsof` | `lsof` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsof) | +| `--lspci` | `lspci -mmv` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lspci) | +| `--lsusb` | `lsusb` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/lsusb) | +| `--m3u` | M3U and M3U8 file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/m3u) | +| `--mdadm` | `mdadm` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mdadm) | +| `--mount` | `mount` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mount) | +| `--mpstat` | `mpstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat) | +| `--mpstat-s` | `mpstat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat_s) | +| `--netstat` | `netstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/netstat) | +| `--nmcli` | `nmcli` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/nmcli) | +| `--ntpq` | `ntpq -p` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq) | +| `--openvpn` | openvpn-status.log file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/openvpn) | +| `--os-prober` | `os-prober` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/os_prober) | +| `--passwd` | `/etc/passwd` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/passwd) | +| `--pci-ids` | `pci.ids` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pci_ids) | +| `--pgpass` | PostgreSQL password file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pgpass) | +| `--pidstat` | `pidstat -H` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat) | +| `--pidstat-s` | `pidstat -H` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat_s) | +| `--ping` | `ping` and `ping6` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ping) | +| `--ping-s` | `ping` and `ping6` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s) | +| `--pip-list` | `pip list` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list) | +| `--pip-show` | `pip show` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_show) | +| `--plist` | PLIST file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/plist) | +| `--postconf` | `postconf -M` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/postconf) | +| `--proc` | `/proc/` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/proc) | +| `--ps` | `ps` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ps) | +| `--route` | `route` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/route) | +| `--rpm-qi` | `rpm -qi` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/rpm_qi) | +| `--rsync` | `rsync` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync) | +| `--rsync-s` | `rsync` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync_s) | +| `--semver` | Semantic Version string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/semver) | +| `--sfdisk` | `sfdisk` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sfdisk) | +| `--shadow` | `/etc/shadow` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/shadow) | +| `--ss` | `ss` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ss) | +| `--ssh-conf` | `ssh` config file and `ssh -G` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ssh_conf) | +| `--sshd-conf` | `sshd` config file and `sshd -T` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sshd_conf) | +| `--stat` | `stat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat) | +| `--stat-s` | `stat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s) | +| `--sysctl` | `sysctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/sysctl) | +| `--syslog` | Syslog RFC 5424 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog) | +| `--syslog-s` | Syslog RFC 5424 string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_s) | +| `--syslog-bsd` | Syslog RFC 3164 string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd) | +| `--syslog-bsd-s` | Syslog RFC 3164 string streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/syslog_bsd_s) | +| `--systemctl` | `systemctl` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl) | +| `--systemctl-lj` | `systemctl list-jobs` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_lj) | +| `--systemctl-ls` | `systemctl list-sockets` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_ls) | | `--systemctl-luf` | `systemctl list-unit-files` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_luf) | -| ` --systeminfo` | `systeminfo` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systeminfo) | -| ` --time` | `/usr/bin/time` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/time) | -| ` --timedatectl` | `timedatectl status` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timedatectl) | -| ` --timestamp` | Unix Epoch Timestamp string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timestamp) | -| ` --toml` | TOML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/toml) | -| ` --top` | `top -b` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top) | -| ` --top-s` | `top -b` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top_s) | -| ` --tracepath` | `tracepath` and `tracepath6` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/tracepath) | -| ` --traceroute` | `traceroute` and `traceroute6` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/traceroute) | -| ` --udevadm` | `udevadm info` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/udevadm) | -| ` --ufw` | `ufw status` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ufw) | -| ` --ufw-appinfo` | `ufw app info [application]` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ufw_appinfo) | -| ` --uname` | `uname -a` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/uname) | +| `--systeminfo` | `systeminfo` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/systeminfo) | +| `--time` | `/usr/bin/time` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/time) | +| `--timedatectl` | `timedatectl status` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timedatectl) | +| `--timestamp` | Unix Epoch Timestamp string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timestamp) | +| `--toml` | TOML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/toml) | +| `--top` | `top -b` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top) | +| `--top-s` | `top -b` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top_s) | +| `--tracepath` | `tracepath` and `tracepath6` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/tracepath) | +| `--traceroute` | `traceroute` and `traceroute6` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/traceroute) | +| `--udevadm` | `udevadm info` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/udevadm) | +| `--ufw` | `ufw status` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ufw) | +| `--ufw-appinfo` | `ufw app info [application]` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ufw_appinfo) | +| `--uname` | `uname -a` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/uname) | | `--update-alt-gs` | `update-alternatives --get-selections` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/update_alt_gs) | -| ` --update-alt-q` | `update-alternatives --query` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/update_alt_q) | -| ` --upower` | `upower` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/upower) | -| ` --uptime` | `uptime` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/uptime) | -| ` --url` | URL string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/url) | -| ` --ver` | Version string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ver) | -| ` --vmstat` | `vmstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/vmstat) | -| ` --vmstat-s` | `vmstat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/vmstat_s) | -| ` --w` | `w` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/w) | -| ` --wc` | `wc` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/wc) | -| ` --who` | `who` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/who) | -| ` --x509-cert` | X.509 PEM and DER certificate file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/x509_cert) | -| ` --xml` | XML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/xml) | -| ` --xrandr` | `xrandr` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/xrandr) | -| ` --yaml` | YAML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml) | -| ` --zipinfo` | `zipinfo` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo) | +| `--update-alt-q` | `update-alternatives --query` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/update_alt_q) | +| `--upower` | `upower` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/upower) | +| `--uptime` | `uptime` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/uptime) | +| `--url` | URL string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/url) | +| `--ver` | Version string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ver) | +| `--vmstat` | `vmstat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/vmstat) | +| `--vmstat-s` | `vmstat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/vmstat_s) | +| `--w` | `w` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/w) | +| `--wc` | `wc` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/wc) | +| `--who` | `who` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/who) | +| `--x509-cert` | X.509 PEM and DER certificate file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/x509_cert) | +| `--xml` | XML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/xml) | +| `--xrandr` | `xrandr` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/xrandr) | +| `--yaml` | YAML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml) | +| `--zipinfo` | `zipinfo` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo) | ### Options @@ -315,8 +313,6 @@ option. | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | - - ### Slice Line slicing is supported using the `START:STOP` syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the diff --git a/templates/readme_template b/templates/readme_template index aba62583..2434b9f7 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -152,11 +152,9 @@ option. ### Parsers - - | Argument | Command or Filetype | Documentation | |-------------------|---------------------------------------------------------|----------------------------------------------------------------------------|{% for parser in parsers %} -| `{{ "{:>15}".format(parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<74}".format("[details](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} +| {{ "{:>16}".format("`" + parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<74}".format("[details](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} ### Options @@ -177,8 +175,6 @@ option. | `-B` | `--bash-comp` | Generate Bash shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | | `-Z` | `--zsh-comp` | Generate Zsh shell completion script ([more info](https://github.com/kellyjonbrazil/jc/wiki/Shell-Completions)) | - - ### Slice Line slicing is supported using the `START:STOP` syntax similar to Python slicing. This allows you to skip lines at the beginning and/or end of the From c0239a771cfc0c3b2839aa6afb45bdb58341fea9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 27 Jan 2023 16:28:55 -0800 Subject: [PATCH 40/81] clarify parser argument quote wrapping code --- templates/readme_template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/readme_template b/templates/readme_template index 2434b9f7..d6325585 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -154,7 +154,7 @@ option. | Argument | Command or Filetype | Documentation | |-------------------|---------------------------------------------------------|----------------------------------------------------------------------------|{% for parser in parsers %} -| {{ "{:>16}".format("`" + parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<74}".format("[details](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} +| {{ "{:>17}".format("`" + parser.argument + "`") }} | {{ "{:<55}".format(parser.description) }} | {{ "{:<74}".format("[details](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} ### Options From 4b55f49e9912bd6a522e5718e6a188028096501e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 30 Jan 2023 08:11:21 -0800 Subject: [PATCH 41/81] formatting --- jc/cli_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/cli_data.py b/jc/cli_data.py index 736ba512..48422209 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -89,7 +89,7 @@ Examples: $ jc --pretty /proc/meminfo Line Slicing: - $ cat file.csv | jc :102 --csv # parse first 100 records + $ cat file.csv | jc :101 --csv # parse first 100 lines Parser Documentation: $ jc --help --dig From 2beb26f3e3324aa520c103e50cccb203a82cb748 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 30 Jan 2023 08:52:22 -0800 Subject: [PATCH 42/81] add slice info to Metadata output --- jc/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jc/cli.py b/jc/cli.py index 36936370..66b808c9 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -610,7 +610,9 @@ class JcCli(): if self.run_timestamp: meta_obj: JSONDictType = { 'parser': self.parser_name, - 'timestamp': self.run_timestamp.timestamp() + 'timestamp': self.run_timestamp.timestamp(), + 'slice_start': self.slice_start, + 'slice_end': self.slice_end } if self.magic_run_command: From 7a43ba631bce139a0e12cbdd5462ca90b794f60c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 30 Jan 2023 08:59:54 -0800 Subject: [PATCH 43/81] fix tests for slice info metadata --- tests/test_jc_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_jc_cli.py b/tests/test_jc_cli.py index dee91761..6981ebc6 100644 --- a/tests/test_jc_cli.py +++ b/tests/test_jc_cli.py @@ -292,7 +292,7 @@ class MyTests(unittest.TestCase): cli.magic_returncode = 2 cli.magic_run_command = ['ping', '-c3', '192.168.1.123'] cli.parser_name = 'ping' - expected = {'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}} + expected = {'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349, 'slice_start': None, 'slice_end': None}} cli.add_metadata_to_output() self.assertEqual(cli.data_out, expected) @@ -303,7 +303,7 @@ class MyTests(unittest.TestCase): cli.magic_returncode = 2 cli.magic_run_command = ['ping', '-c3', '192.168.1.123'] cli.parser_name = 'ping' - expected = [{'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}] + expected = [{'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349, 'slice_start': None, 'slice_end': None}}, {'a': 3, 'b': 4, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349, 'slice_start': None, 'slice_end': None}}] cli.add_metadata_to_output() self.assertEqual(cli.data_out, expected) @@ -314,7 +314,7 @@ class MyTests(unittest.TestCase): cli.data_out = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar'}} cli.run_timestamp = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) cli.parser_name = 'ping' - expected = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}} + expected = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349, 'slice_start': None, 'slice_end': None}} cli.add_metadata_to_output() self.assertEqual(cli.data_out, expected) @@ -325,7 +325,7 @@ class MyTests(unittest.TestCase): cli.magic_returncode = 2 cli.magic_run_command = ['ping', '-c3', '192.168.1.123'] cli.parser_name = 'ping' - expected = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}] + expected = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349, 'slice_start': None, 'slice_end': None}}, {'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349, 'slice_start': None, 'slice_end': None}}] cli.add_metadata_to_output() self.assertEqual(cli.data_out, expected) From 23ff19fdb503f9c890f79263d8f866367e294c45 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 06:48:37 -0800 Subject: [PATCH 44/81] add slicer tests --- tests/test_jc_cli.py | 124 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/tests/test_jc_cli.py b/tests/test_jc_cli.py index 6981ebc6..5bfc13d2 100644 --- a/tests/test_jc_cli.py +++ b/tests/test_jc_cli.py @@ -329,5 +329,129 @@ class MyTests(unittest.TestCase): cli.add_metadata_to_output() self.assertEqual(cli.data_out, expected) + def test_slice_none_str(self): + cli = JcCli() + cli.slice_start = None + cli.slice_end = None + cli.data_in = '''\ + row0 + row1 + row2 + row3 + row4 + row5''' + expected = '''\ + row0 + row1 + row2 + row3 + row4 + row5''' + cli.slicer() + self.assertEqual(cli.data_in, expected) + + def test_slice_positive_str(self): + cli = JcCli() + cli.slice_start = 1 + cli.slice_end = 5 + cli.data_in = '''\ + row0 + row1 + row2 + row3 + row4 + row5''' + expected = '''\ + row1 + row2 + row3 + row4''' + cli.slicer() + self.assertEqual(cli.data_in, expected) + + def test_slice_negative_str(self): + cli = JcCli() + cli.slice_start = 1 + cli.slice_end = -1 + cli.data_in = '''\ + row0 + row1 + row2 + row3 + row4 + row5''' + expected = '''\ + row1 + row2 + row3 + row4''' + cli.slicer() + self.assertEqual(cli.data_in, expected) + + def test_slice_none_iter(self): + cli = JcCli() + cli.slice_start = None + cli.slice_end = None + cli.data_in = [ + 'row0', + 'row1', + 'row2', + 'row3', + 'row4', + 'row5' + ] + expected = [ + 'row0', + 'row1', + 'row2', + 'row3', + 'row4', + 'row5' + ] + cli.slicer() + self.assertEqual(cli.data_in, expected) + + def test_slice_positive_iter(self): + cli = JcCli() + cli.slice_start = 1 + cli.slice_end = 5 + cli.data_in = [ + 'row0', + 'row1', + 'row2', + 'row3', + 'row4', + 'row5' + ] + expected = [ + 'row1', + 'row2', + 'row3', + 'row4' + ] + cli.slicer() + self.assertEqual(list(cli.data_in), expected) + + def test_slice_negative_iter(self): + cli = JcCli() + cli.slice_start = 1 + cli.slice_end = -1 + cli.data_in = [ + 'row0', + 'row1', + 'row2', + 'row3', + 'row4', + 'row5' + ] + expected = [ + 'row1', + 'row2', + 'row3', + 'row4' + ] + cli.slicer() + self.assertEqual(list(cli.data_in), expected) + if __name__ == '__main__': unittest.main() \ No newline at end of file From b134c53f33d33dd21952597bfc8df43166613fd2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 08:29:43 -0800 Subject: [PATCH 45/81] add ssh_conf tests --- jc/parsers/ssh_conf.py | 48 ++++++++++- tests/fixtures/generic/ssh_config1 | 45 ++++++++++ tests/fixtures/generic/ssh_config1.json | 1 + tests/fixtures/generic/ssh_config2 | 21 +++++ tests/fixtures/generic/ssh_config2.json | 1 + tests/fixtures/generic/ssh_config3 | 33 ++++++++ tests/fixtures/generic/ssh_config3.json | 1 + tests/fixtures/generic/ssh_config4 | 105 ++++++++++++++++++++++++ tests/fixtures/generic/ssh_config4.json | 1 + tests/fixtures/generic/ssh_config5 | 14 ++++ tests/fixtures/generic/ssh_config5.json | 1 + tests/test_ssh_conf.py | 95 +++++++++++++++++++++ 12 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/generic/ssh_config1 create mode 100644 tests/fixtures/generic/ssh_config1.json create mode 100644 tests/fixtures/generic/ssh_config2 create mode 100644 tests/fixtures/generic/ssh_config2.json create mode 100644 tests/fixtures/generic/ssh_config3 create mode 100644 tests/fixtures/generic/ssh_config3.json create mode 100644 tests/fixtures/generic/ssh_config4 create mode 100644 tests/fixtures/generic/ssh_config4.json create mode 100644 tests/fixtures/generic/ssh_config5 create mode 100644 tests/fixtures/generic/ssh_config5.json create mode 100644 tests/test_ssh_conf.py diff --git a/jc/parsers/ssh_conf.py b/jc/parsers/ssh_conf.py index 45ed2cdd..95b2c7ec 100644 --- a/jc/parsers/ssh_conf.py +++ b/jc/parsers/ssh_conf.py @@ -26,6 +26,9 @@ Schema: [ { "host": string, + "host_list": [ + string + ], "addkeystoagent": string, "addressfamily": string, "batchmode": string, @@ -364,6 +367,9 @@ Examples: [ { "host": "server1", + "host_list": [ + "server1" + ], "hostname": "server1.cyberciti.biz", "user": "nixcraft", "port": 4242, @@ -373,6 +379,9 @@ Examples: }, { "host": "nas01", + "host_list": [ + "nas01" + ], "hostname": "192.168.1.100", "user": "root", "identityfile": [ @@ -381,6 +390,9 @@ Examples: }, { "host": "aws.apache", + "host_list": [ + "aws.apache" + ], "hostname": "1.2.3.4", "user": "wwwdata", "identityfile": [ @@ -389,12 +401,19 @@ Examples: }, { "host": "uk.gw.lan uk.lan", + "host_list": [ + "uk.gw.lan", + "uk.lan" + ], "hostname": "192.168.0.251", "user": "nixcraft", "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" }, { "host": "proxyus", + "host_list": [ + "proxyus" + ], "hostname": "vps1.cyberciti.biz", "user": "breakfree", "identityfile": [ @@ -406,6 +425,9 @@ Examples: }, { "host": "*", + "host_list": [ + "*" + ], "forwardagent": "no", "forwardx11": "no", "forwardx11trusted": "yes", @@ -421,6 +443,9 @@ Examples: [ { "host": "server1", + "host_list": [ + "server1" + ], "hostname": "server1.cyberciti.biz", "user": "nixcraft", "port": "4242", @@ -430,6 +455,9 @@ Examples: }, { "host": "nas01", + "host_list": [ + "nas01" + ], "hostname": "192.168.1.100", "user": "root", "identityfile": [ @@ -438,6 +466,9 @@ Examples: }, { "host": "aws.apache", + "host_list": [ + "aws.apache" + ], "hostname": "1.2.3.4", "user": "wwwdata", "identityfile": [ @@ -446,12 +477,19 @@ Examples: }, { "host": "uk.gw.lan uk.lan", + "host_list": [ + "uk.gw.lan", + "uk.lan" + ], "hostname": "192.168.0.251", "user": "nixcraft", "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" }, { "host": "proxyus", + "host_list": [ + "proxyus" + ], "hostname": "vps1.cyberciti.biz", "user": "breakfree", "identityfile": [ @@ -463,6 +501,9 @@ Examples: }, { "host": "*", + "host_list": [ + "*" + ], "forwardagent": "no", "forwardx11": "no", "forwardx11trusted": "yes", @@ -602,7 +643,12 @@ def parse( if line.strip().startswith('Host '): if host: raw_output.append(host) - host = {'host': line.split()[1]} + + hostnames = line.split(maxsplit=1)[1] + host = { + 'host': hostnames, + 'host_list': hostnames.split() + } # support configuration file by ignoring all lines between # Match xxx and Match any diff --git a/tests/fixtures/generic/ssh_config1 b/tests/fixtures/generic/ssh_config1 new file mode 100644 index 00000000..719cc3ba --- /dev/null +++ b/tests/fixtures/generic/ssh_config1 @@ -0,0 +1,45 @@ +## override as per host ## +Host server1 + HostName server1.cyberciti.biz + User nixcraft + Port 4242 + IdentityFile /nfs/shared/users/nixcraft/keys/server1/id_rsa + +## Home nas server ## +Host nas01 + HostName 192.168.1.100 + User root + IdentityFile ~/.ssh/nas01.key + +## Login AWS Cloud ## +Host aws.apache + HostName 1.2.3.4 + User wwwdata + IdentityFile ~/.ssh/aws.apache.key + +## Login to internal lan server at 192.168.0.251 via our public uk office ssh based gateway using ## +## $ ssh uk.gw.lan ## +Host uk.gw.lan uk.lan + HostName 192.168.0.251 + User nixcraft + ProxyCommand ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null + +## Our Us Proxy Server ## +## Forward all local port 3128 traffic to port 3128 on the remote vps1.cyberciti.biz server ## +## $ ssh -f -N proxyus ## +Host proxyus + HostName vps1.cyberciti.biz + User breakfree + IdentityFile ~/.ssh/vps1.cyberciti.biz.key + LocalForward 3128 127.0.0.1:3128 + +### default for all ## +Host * + ForwardAgent no + ForwardX11 no + ForwardX11Trusted yes + User nixcraft + Port 22 + Protocol 2 + ServerAliveInterval 60 + ServerAliveCountMax 30 diff --git a/tests/fixtures/generic/ssh_config1.json b/tests/fixtures/generic/ssh_config1.json new file mode 100644 index 00000000..99cd39cf --- /dev/null +++ b/tests/fixtures/generic/ssh_config1.json @@ -0,0 +1 @@ +[{"host":"server1","host_list":["server1"],"hostname":"server1.cyberciti.biz","user":"nixcraft","port":4242,"identityfile":["/nfs/shared/users/nixcraft/keys/server1/id_rsa"]},{"host":"nas01","host_list":["nas01"],"hostname":"192.168.1.100","user":"root","identityfile":["~/.ssh/nas01.key"]},{"host":"aws.apache","host_list":["aws.apache"],"hostname":"1.2.3.4","user":"wwwdata","identityfile":["~/.ssh/aws.apache.key"]},{"host":"uk.gw.lan uk.lan","host_list":["uk.gw.lan","uk.lan"],"hostname":"192.168.0.251","user":"nixcraft","proxycommand":"ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null"},{"host":"proxyus","host_list":["proxyus"],"hostname":"vps1.cyberciti.biz","user":"breakfree","identityfile":["~/.ssh/vps1.cyberciti.biz.key"],"localforward":["3128 127.0.0.1:3128"]},{"host":"*","host_list":["*"],"forwardagent":"no","forwardx11":"no","forwardx11trusted":"yes","user":"nixcraft","port":22,"protocol":2,"serveraliveinterval":60,"serveralivecountmax":30}] diff --git a/tests/fixtures/generic/ssh_config2 b/tests/fixtures/generic/ssh_config2 new file mode 100644 index 00000000..15261f36 --- /dev/null +++ b/tests/fixtures/generic/ssh_config2 @@ -0,0 +1,21 @@ +Host targaryen + HostName 192.168.1.10 + User daenerys + Port 7654 + IdentityFile ~/.ssh/targaryen.key + +Host tyrell + HostName 192.168.10.20 + +Host martell + HostName 192.168.10.50 + +Host *ell + user oberyn + +Host * !martell + LogLevel INFO + +Host * + User root + Compression yes diff --git a/tests/fixtures/generic/ssh_config2.json b/tests/fixtures/generic/ssh_config2.json new file mode 100644 index 00000000..3f2683a3 --- /dev/null +++ b/tests/fixtures/generic/ssh_config2.json @@ -0,0 +1 @@ +[{"host":"targaryen","host_list":["targaryen"],"hostname":"192.168.1.10","user":"daenerys","port":7654,"identityfile":["~/.ssh/targaryen.key"]},{"host":"tyrell","host_list":["tyrell"],"hostname":"192.168.10.20"},{"host":"martell","host_list":["martell"],"hostname":"192.168.10.50"},{"host":"*ell","host_list":["*ell"],"user":"oberyn"},{"host":"* !martell","host_list":["*","!martell"],"loglevel":"INFO"},{"host":"*","host_list":["*"],"user":"root","compression":"yes"}] diff --git a/tests/fixtures/generic/ssh_config3 b/tests/fixtures/generic/ssh_config3 new file mode 100644 index 00000000..8f2e41ed --- /dev/null +++ b/tests/fixtures/generic/ssh_config3 @@ -0,0 +1,33 @@ +Host server1 + HostName server1.cyberciti.biz + User nixcraft + Port 4242 + IdentityFile /nfs/shared/users/nixcraft/keys/server1/id_rsa + +## Home nas server ## +Host nas01 + HostName 192.168.1.100 + User root + IdentityFile ~/.ssh/nas01.key + +## Login AWS Cloud ## +Host aws.apache + HostName 1.2.3.4 + User wwwdata + IdentityFile ~/.ssh/aws.apache.key + +## Login to internal lan server at 192.168.0.251 via our public uk office ssh based gateway using ## +## $ ssh uk.gw.lan ## +Host uk.gw.lan uk.lan + HostName 192.168.0.251 + User nixcraft + ProxyCommand ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null + +## Our Us Proxy Server ## +## Forward all local port 3128 traffic to port 3128 on the remote vps1.cyberciti.biz server ## +## $ ssh -f -N proxyus ## +Host proxyus + HostName vps1.cyberciti.biz + User breakfree + IdentityFile ~/.ssh/vps1.cyberciti.biz.key + LocalForward 3128 127.0.0.1:3128 diff --git a/tests/fixtures/generic/ssh_config3.json b/tests/fixtures/generic/ssh_config3.json new file mode 100644 index 00000000..22d75a5f --- /dev/null +++ b/tests/fixtures/generic/ssh_config3.json @@ -0,0 +1 @@ +[{"host":"server1","host_list":["server1"],"hostname":"server1.cyberciti.biz","user":"nixcraft","port":4242,"identityfile":["/nfs/shared/users/nixcraft/keys/server1/id_rsa"]},{"host":"nas01","host_list":["nas01"],"hostname":"192.168.1.100","user":"root","identityfile":["~/.ssh/nas01.key"]},{"host":"aws.apache","host_list":["aws.apache"],"hostname":"1.2.3.4","user":"wwwdata","identityfile":["~/.ssh/aws.apache.key"]},{"host":"uk.gw.lan uk.lan","host_list":["uk.gw.lan","uk.lan"],"hostname":"192.168.0.251","user":"nixcraft","proxycommand":"ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null"},{"host":"proxyus","host_list":["proxyus"],"hostname":"vps1.cyberciti.biz","user":"breakfree","identityfile":["~/.ssh/vps1.cyberciti.biz.key"],"localforward":["3128 127.0.0.1:3128"]}] diff --git a/tests/fixtures/generic/ssh_config4 b/tests/fixtures/generic/ssh_config4 new file mode 100644 index 00000000..318d5cb9 --- /dev/null +++ b/tests/fixtures/generic/ssh_config4 @@ -0,0 +1,105 @@ +Host * + AddKeysToAgent ask + AddressFamily inet + BatchMode no + BindAddress 1.1.1.1 + BindInterface en0 + CanonicalDomains abc.com xyz.com + CanonicalizeFallbackLocal yes + CanonicalizeHostname none + CanonicalizeMaxDots 2 + CanonicalizePermittedCNAMEs *.a.example.com:*.b.example.com,*.c.example.com + CASignatureAlgorithms ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com + CertificateFile ~/certificates/cert1.pem + CertificateFile ~/certificates/cert2.pem + CheckHostIP yes + Ciphers 3des-cbc,aes128-cbc,aes192-cbc + ClearAllForwardings yes + Compression yes + ConnectionAttempts 9 + ConnectTimeout 30 + ControlMaster ask + ControlPath none + ControlPersist yes + DynamicForward 1.1.1.1:443 + EnableEscapeCommandline no + EnableSSHKeysign yes + EscapeChar none + ExitOnForwardFailure yes + FingerprintHash md5 + ForkAfterAuthentication yes + ForwardAgent $mypath + ForwardX11 no + ForwardX11Timeout 500 + ForwardX11Trusted yes + GatewayPorts yes + GlobalKnownHostsFile /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts2 + GSSAPIAuthentication yes + GSSAPIDelegateCredentials yes + HashKnownHosts yes + HostbasedAcceptedAlgorithms ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com + HostbasedAuthentication yes + HostKeyAlgorithms ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com + HostKeyAlias foobar + Hostname localhost + IdentitiesOnly yes + IdentityAgent SSH_AUTH_SOCK + IdentityFile ~/.ssh/vps1.cyberciti.biz.key + IdentityFile ~/.ssh/vps2.cyberciti.biz.key + IgnoreUnknown helloworld + Include ~/.ssh/config-extras ~/foo/bar + Include ~/.ssh/config-extra-extras + IPQoS af11 af12 + KbdInteractiveAuthentication yes + KbdInteractiveDevices bsdauth,pam,skey + KexAlgorithms +sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org + KnownHostsCommand ~/checkknownhosts + LocalCommand ~/mycommand + LocalForward 3128 127.0.0.1:3128 + LocalForward 3129 127.0.0.1:3129 + LogLevel INFO + LogVerbose kex.c:*:1000,*:kex_exchange_identification():*,packet.c:* + MACs ^umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com + NoHostAuthenticationForLocalhost yes + NumberOfPasswordPrompts 3 + PasswordAuthentication yes + PermitLocalCommand yes + PermitRemoteOpen 1.1.1.1:443 2.2.2.2:443 + PKCS11Provider ~/pkcs11provider + Port 22 + PreferredAuthentications gssapi-with-mic,hostbased,publickey,keyboard-interactive,password + Protocol 2 + ProxyCommand ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null + ProxyJump 1.1.1.1:22,2.2.2.2:22 + ProxyUseFdpass yes + PubkeyAcceptedAlgorithms -ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com + PubkeyAuthentication unbound + RekeyLimit 4G + RemoteCommand ~/mycommand + RemoteForward 1.1.1.1:22 2.2.2.2:22 + RequestTTY force + RequiredRSASize 2048 + RevokedHostKeys ~/revokedkeyfile + SecurityKeyProvider ~/keyprovider + SendEnv ENV1 ENV2 + SendEnv ENV3 + ServerAliveCountMax 3 + ServerAliveInterval 3 + SessionType none + SetEnv ENV1 ENV2 + SetEnv ENV3 + StdinNull yes + StreamLocalBindMask 0000 + StreamLocalBindUnlink yes + StrictHostKeyChecking ask + SyslogFacility USER + TCPKeepAlive yes + Tunnel ethernet + TunnelDevice tun1:tun2 + UpdateHostKeys ask + User nixcraft + UserKnownHostsFile ~/.ssh/knownhosts1 ~/.ssh/knownhosts2 + VerifyHostKeyDNS ask + VisualHostKey yes + XAuthLocation /usr/X11R6/bin/xauth + diff --git a/tests/fixtures/generic/ssh_config4.json b/tests/fixtures/generic/ssh_config4.json new file mode 100644 index 00000000..d3a8560e --- /dev/null +++ b/tests/fixtures/generic/ssh_config4.json @@ -0,0 +1 @@ +[{"host":"*","host_list":["*"],"addkeystoagent":"ask","addressfamily":"inet","batchmode":"no","bindaddress":"1.1.1.1","bindinterface":"en0","canonicaldomains":["abc.com","xyz.com"],"canonicalizefallbacklocal":"yes","canonicalizehostname":"none","canonicalizemaxdots":2,"canonicalizepermittedcnames":["*.a.example.com:*.b.example.com","*.c.example.com"],"casignaturealgorithms":["ssh-ed25519","ecdsa-sha2-nistp256","ecdsa-sha2-nistp384","ecdsa-sha2-nistp521","sk-ssh-ed25519@openssh.com"],"certificatefile":["~/certificates/cert1.pem","~/certificates/cert2.pem"],"checkhostip":"yes","ciphers":["3des-cbc","aes128-cbc","aes192-cbc"],"clearallforwardings":"yes","compression":"yes","connectionattempts":9,"connecttimeout":30,"controlmaster":"ask","controlpath":"none","controlpersist":"yes","dynamicforward":"1.1.1.1:443","enableescapecommandline":"no","enablesshkeysign":"yes","escapechar":"none","exitonforwardfailure":"yes","fingerprinthash":"md5","forkafterauthentication":"yes","forwardagent":"$mypath","forwardx11":"no","forwardx11timeout":500,"forwardx11trusted":"yes","gatewayports":"yes","globalknownhostsfile":["/etc/ssh/ssh_known_hosts","/etc/ssh/ssh_known_hosts2"],"gssapiauthentication":"yes","gssapidelegatecredentials":"yes","hashknownhosts":"yes","hostbasedacceptedalgorithms":["ssh-ed25519-cert-v01@openssh.com","ecdsa-sha2-nistp256-cert-v01@openssh.com"],"hostbasedauthentication":"yes","hostkeyalgorithms":["ssh-ed25519-cert-v01@openssh.com","ecdsa-sha2-nistp256-cert-v01@openssh.com"],"hostkeyalias":"foobar","hostname":"localhost","identitiesonly":"yes","identityagent":"SSH_AUTH_SOCK","identityfile":["~/.ssh/vps1.cyberciti.biz.key","~/.ssh/vps2.cyberciti.biz.key"],"ignoreunknown":"helloworld","include":["~/.ssh/config-extras","~/foo/bar","~/.ssh/config-extra-extras"],"ipqos":["af11","af12"],"kbdinteractiveauthentication":"yes","kbdinteractivedevices":["bsdauth","pam","skey"],"kexalgorithms":["sntrup761x25519-sha512@openssh.com","curve25519-sha256","curve25519-sha256@libssh.org"],"kexalgorithms_strategy":"+","knownhostscommand":"~/checkknownhosts","localcommand":"~/mycommand","localforward":["3128 127.0.0.1:3128","3129 127.0.0.1:3129"],"loglevel":"INFO","logverbose":["kex.c:*:1000","*:kex_exchange_identification():*","packet.c:*"],"macs":["umac-64-etm@openssh.com","umac-128-etm@openssh.com","hmac-sha2-256-etm@openssh.com","hmac-sha2-512-etm@openssh.com"],"macs_strategy":"^","nohostauthenticationforlocalhost":"yes","numberofpasswordprompts":3,"passwordauthentication":"yes","permitlocalcommand":"yes","permitremoteopen":["1.1.1.1:443","2.2.2.2:443"],"pkcs11provider":"~/pkcs11provider","port":22,"preferredauthentications":["gssapi-with-mic","hostbased","publickey","keyboard-interactive","password"],"protocol":2,"proxycommand":"ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null","proxyjump":["1.1.1.1:22","2.2.2.2:22"],"proxyusefdpass":"yes","pubkeyacceptedalgorithms":["ssh-ed25519-cert-v01@openssh.com","ecdsa-sha2-nistp256-cert-v01@openssh.com"],"pubkeyacceptedalgorithms_strategy":"-","pubkeyauthentication":"unbound","rekeylimit":"4G","remotecommand":"~/mycommand","remoteforward":"1.1.1.1:22 2.2.2.2:22","requesttty":"force","requiredrsasize":2048,"revokedhostkeys":"~/revokedkeyfile","securitykeyprovider":"~/keyprovider","sendenv":["ENV1","ENV2","ENV3"],"serveralivecountmax":3,"serveraliveinterval":3,"sessiontype":"none","setenv":["ENV1","ENV2","ENV3"],"stdinnull":"yes","streamlocalbindmask":"0000","streamlocalbindunlink":"yes","stricthostkeychecking":"ask","syslogfacility":"USER","tcpkeepalive":"yes","tunnel":"ethernet","tunneldevice":"tun1:tun2","updatehostkeys":"ask","user":"nixcraft","userknownhostsfile":["~/.ssh/knownhosts1","~/.ssh/knownhosts2"],"verifyhostkeydns":"ask","visualhostkey":"yes","xauthlocation":"/usr/X11R6/bin/xauth"}] diff --git a/tests/fixtures/generic/ssh_config5 b/tests/fixtures/generic/ssh_config5 new file mode 100644 index 00000000..f24ac73d --- /dev/null +++ b/tests/fixtures/generic/ssh_config5 @@ -0,0 +1,14 @@ + +# comment +Host * + User something + +# comment 2 +Host svu + Hostname www.svuniversity.ac.in + # within-host-comment + Port 22 + ProxyCommand nc -w 300 -x localhost:9050 %h %p + +# another comment +# bla bla diff --git a/tests/fixtures/generic/ssh_config5.json b/tests/fixtures/generic/ssh_config5.json new file mode 100644 index 00000000..35db7059 --- /dev/null +++ b/tests/fixtures/generic/ssh_config5.json @@ -0,0 +1 @@ +[{"host":"*","host_list":["*"],"user":"something"},{"host":"svu","host_list":["svu"],"hostname":"www.svuniversity.ac.in","port":22,"proxycommand":"nc -w 300 -x localhost:9050 %h %p"}] diff --git a/tests/test_ssh_conf.py b/tests/test_ssh_conf.py new file mode 100644 index 00000000..aa6647b1 --- /dev/null +++ b/tests/test_ssh_conf.py @@ -0,0 +1,95 @@ +import os +import unittest +import json +from typing import Dict +from jc.parsers.ssh_conf import parse + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + f_in: Dict = {} + f_json: Dict = {} + + @classmethod + def setUpClass(cls): + fixtures = { + 'ssh_config1': ( + 'fixtures/generic/ssh_config1', + 'fixtures/generic/ssh_config1.json'), + 'ssh_config2': ( + 'fixtures/generic/ssh_config2', + 'fixtures/generic/ssh_config2.json'), + 'ssh_config3': ( + 'fixtures/generic/ssh_config3', + 'fixtures/generic/ssh_config3.json'), + 'ssh_config4': ( + 'fixtures/generic/ssh_config4', + 'fixtures/generic/ssh_config4.json'), + 'ssh_config5': ( + 'fixtures/generic/ssh_config5', + 'fixtures/generic/ssh_config5.json') + } + + for file, filepaths in fixtures.items(): + with open(os.path.join(THIS_DIR, filepaths[0]), 'r', encoding='utf-8') as a, \ + open(os.path.join(THIS_DIR, filepaths[1]), 'r', encoding='utf-8') as b: + cls.f_in[file] = a.read() + cls.f_json[file] = json.loads(b.read()) + + + def test_ssh_nodata(self): + """ + Test 'ssh' with no data + """ + self.assertEqual(parse('', quiet=True), []) + + + def test_ssh_config1(self): + """ + Test 'ssh' config 1 + """ + self.assertEqual( + parse(self.f_in['ssh_config1'], quiet=True), + self.f_json['ssh_config1'] + ) + + def test_ssh_config2(self): + """ + Test 'ssh' config 2 + """ + self.assertEqual( + parse(self.f_in['ssh_config2'], quiet=True), + self.f_json['ssh_config2'] + ) + + def test_ssh_config3(self): + """ + Test 'ssh' config 3 + """ + self.assertEqual( + parse(self.f_in['ssh_config3'], quiet=True), + self.f_json['ssh_config3'] + ) + + def test_ssh_config4(self): + """ + Test 'ssh' config 4 + """ + self.assertEqual( + parse(self.f_in['ssh_config4'], quiet=True), + self.f_json['ssh_config4'] + ) + + def test_ssh_config5(self): + """ + Test 'ssh' config 5 + """ + self.assertEqual( + parse(self.f_in['ssh_config5'], quiet=True), + self.f_json['ssh_config5'] + ) + + +if __name__ == '__main__': + unittest.main() From 12c4419c6a56f261a5fd3e43f53e8fa3293e4000 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 08:30:37 -0800 Subject: [PATCH 46/81] doc update --- docs/parsers/ssh_conf.md | 41 ++++++++++++++++++++++++++++++++++++++++ man/jc.1 | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docs/parsers/ssh_conf.md b/docs/parsers/ssh_conf.md index 77acfeaa..2502f1cd 100644 --- a/docs/parsers/ssh_conf.md +++ b/docs/parsers/ssh_conf.md @@ -31,6 +31,9 @@ Schema: [ { "host": string, + "host_list": [ + string + ], "addkeystoagent": string, "addressfamily": string, "batchmode": string, @@ -369,6 +372,9 @@ Examples: [ { "host": "server1", + "host_list": [ + "server1" + ], "hostname": "server1.cyberciti.biz", "user": "nixcraft", "port": 4242, @@ -378,6 +384,9 @@ Examples: }, { "host": "nas01", + "host_list": [ + "nas01" + ], "hostname": "192.168.1.100", "user": "root", "identityfile": [ @@ -386,6 +395,9 @@ Examples: }, { "host": "aws.apache", + "host_list": [ + "aws.apache" + ], "hostname": "1.2.3.4", "user": "wwwdata", "identityfile": [ @@ -394,12 +406,19 @@ Examples: }, { "host": "uk.gw.lan uk.lan", + "host_list": [ + "uk.gw.lan", + "uk.lan" + ], "hostname": "192.168.0.251", "user": "nixcraft", "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" }, { "host": "proxyus", + "host_list": [ + "proxyus" + ], "hostname": "vps1.cyberciti.biz", "user": "breakfree", "identityfile": [ @@ -411,6 +430,9 @@ Examples: }, { "host": "*", + "host_list": [ + "*" + ], "forwardagent": "no", "forwardx11": "no", "forwardx11trusted": "yes", @@ -426,6 +448,9 @@ Examples: [ { "host": "server1", + "host_list": [ + "server1" + ], "hostname": "server1.cyberciti.biz", "user": "nixcraft", "port": "4242", @@ -435,6 +460,9 @@ Examples: }, { "host": "nas01", + "host_list": [ + "nas01" + ], "hostname": "192.168.1.100", "user": "root", "identityfile": [ @@ -443,6 +471,9 @@ Examples: }, { "host": "aws.apache", + "host_list": [ + "aws.apache" + ], "hostname": "1.2.3.4", "user": "wwwdata", "identityfile": [ @@ -451,12 +482,19 @@ Examples: }, { "host": "uk.gw.lan uk.lan", + "host_list": [ + "uk.gw.lan", + "uk.lan" + ], "hostname": "192.168.0.251", "user": "nixcraft", "proxycommand": "ssh nixcraft@gateway.uk.cyberciti.biz nc %h %p 2> /dev/null" }, { "host": "proxyus", + "host_list": [ + "proxyus" + ], "hostname": "vps1.cyberciti.biz", "user": "breakfree", "identityfile": [ @@ -468,6 +506,9 @@ Examples: }, { "host": "*", + "host_list": [ + "*" + ], "forwardagent": "no", "forwardx11": "no", "forwardx11trusted": "yes", diff --git a/man/jc.1 b/man/jc.1 index 61cffdb7..c159e1d9 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-01-27 1.23.0 "JSON Convert" +.TH jc 1 2023-01-31 1.23.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings From 1d8f83b8c69232c8ff6471a2b04eac171635a410 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 09:44:23 -0800 Subject: [PATCH 47/81] add ver parser tests --- tests/test_ver.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/test_ver.py diff --git a/tests/test_ver.py b/tests/test_ver.py new file mode 100644 index 00000000..b1729761 --- /dev/null +++ b/tests/test_ver.py @@ -0,0 +1,45 @@ +import unittest +from jc.parsers.ver import parse + + +class MyTests(unittest.TestCase): + + def test_ver_nodata(self): + """ + Test 'ver' with no data + """ + self.assertEqual(parse('', quiet=True), {}) + + + def test_ver_strict_strings(self): + strict_strings = { + '0.4': {'major': 0, 'minor': 4, 'patch': 0, 'prerelease': None, 'prerelease_num': None, 'strict': True}, + '0.4.0': {'major': 0, 'minor': 4, 'patch': 0, 'prerelease': None, 'prerelease_num': None, 'strict': True}, + '0.4.1': {'major': 0, 'minor': 4, 'patch': 1, 'prerelease': None, 'prerelease_num': None, 'strict': True}, + '0.5a1': {'major': 0, 'minor': 5, 'patch': 0, 'prerelease': 'a', 'prerelease_num': 1, 'strict': True}, + '0.5b3': {'major': 0, 'minor': 5, 'patch': 0, 'prerelease': 'b', 'prerelease_num': 3, 'strict': True}, + '0.5': {'major': 0, 'minor': 5, 'patch': 0, 'prerelease': None, 'prerelease_num': None, 'strict': True}, + '0.9.6': {'major': 0, 'minor': 9, 'patch': 6, 'prerelease': None, 'prerelease_num': None, 'strict': True}, + '1.0': {'major': 1, 'minor': 0, 'patch': 0, 'prerelease': None, 'prerelease_num': None, 'strict': True}, + '1.0.4a3': {'major': 1, 'minor': 0, 'patch': 4, 'prerelease': 'a', 'prerelease_num': 3, 'strict': True}, + '1.0.4b1': {'major': 1, 'minor': 0, 'patch': 4, 'prerelease': 'b', 'prerelease_num': 1, 'strict': True}, + '1.0.4': {'major': 1, 'minor': 0, 'patch': 4, 'prerelease': None, 'prerelease_num': None, 'strict': True} + } + + for ver_string, expected in strict_strings.items(): + self.assertEqual(parse(ver_string, quiet=True), expected) + + def test_ver_loose_strings(self): + loose_strings = { + '1': {'components': [1], 'strict': False}, + '2.7.2.2': {'components': [2, 7, 2, 2], 'strict': False}, + '1.3.a4': {'components': [1, 3, 'a', 4], 'strict': False}, + '1.3pl1': {'components': [1, 3, 'pl', 1], 'strict': False}, + '1.3c4': {'components': [1, 3, 'c', 4], 'strict': False} + } + + for ver_string, expected in loose_strings.items(): + self.assertEqual(parse(ver_string, quiet=True), expected) + +if __name__ == '__main__': + unittest.main() From ec29b8bbc6be1c2d85137f62d7ed6875330b1ede Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 09:51:23 -0800 Subject: [PATCH 48/81] add test for acpi fix for never fully discharge state --- .../generic/acpi-V-never-fully-discharge.json | 1 + .../generic/acpi-V-never-fully-discharge.out | 21 +++++++++++++++++++ tests/test_acpi.py | 12 +++++++++++ 3 files changed, 34 insertions(+) create mode 100644 tests/fixtures/generic/acpi-V-never-fully-discharge.json create mode 100644 tests/fixtures/generic/acpi-V-never-fully-discharge.out diff --git a/tests/fixtures/generic/acpi-V-never-fully-discharge.json b/tests/fixtures/generic/acpi-V-never-fully-discharge.json new file mode 100644 index 00000000..bce4bd3f --- /dev/null +++ b/tests/fixtures/generic/acpi-V-never-fully-discharge.json @@ -0,0 +1 @@ +[{"type":"Battery","id":0,"state":"Discharging","charge_percent":87,"design_capacity_mah":2110,"last_full_capacity":2271,"last_full_capacity_percent":100},{"type":"Battery","id":1,"state":"Discharging","charge_percent":98,"charge_remaining":"01:43:14","design_capacity_mah":4400,"last_full_capacity":3013,"last_full_capacity_percent":68,"charge_remaining_hours":1,"charge_remaining_minutes":43,"charge_remaining_seconds":14,"charge_remaining_total_seconds":6194},{"type":"Battery","id":2,"state":"Discharging","charge_percent":0},{"type":"Battery","id":3,"state":"Full","charge_percent":100},{"type":"Adapter","id":0,"on-line":true},{"type":"Adapter","id":1,"on-line":false},{"type":"Thermal","id":0,"mode":"ok","temperature":46.0,"temperature_unit":"C","trip_points":[{"id":0,"switches_to_mode":"critical","temperature":127.0,"temperature_unit":"C"},{"id":1,"switches_to_mode":"hot","temperature":127.0,"temperature_unit":"C"}]},{"type":"Thermal","id":1,"mode":"ok","temperature":55.0,"temperature_unit":"C","trip_points":[{"id":0,"switches_to_mode":"critical","temperature":130.0,"temperature_unit":"C"},{"id":1,"switches_to_mode":"hot","temperature":100.0,"temperature_unit":"C"}]},{"type":"Cooling","id":0,"messages":["Processor 0 of 10"]},{"type":"Cooling","id":1,"messages":["Processor 0 of 10"]},{"type":"Cooling","id":2,"messages":["x86_pkg_temp no state information available"]},{"type":"Cooling","id":3,"messages":["Processor 0 of 10"]},{"type":"Cooling","id":4,"messages":["intel_powerclamp no state information available","another message"]},{"type":"Cooling","id":5,"messages":["Processor 0 of 10"]}] diff --git a/tests/fixtures/generic/acpi-V-never-fully-discharge.out b/tests/fixtures/generic/acpi-V-never-fully-discharge.out new file mode 100644 index 00000000..c7c9b445 --- /dev/null +++ b/tests/fixtures/generic/acpi-V-never-fully-discharge.out @@ -0,0 +1,21 @@ +Battery 0: Discharging, 87%, discharging at zero rate - will never fully discharge. +Battery 0: design capacity 2110 mAh, last full capacity 2271 mAh = 100% +Battery 1: Discharging, 98%, 01:43:14 remaining +Battery 1: design capacity 4400 mAh, last full capacity 3013 mAh = 68% +Battery 2: Discharging, 0%, rate information unavailable +Battery 3: Full, 100% +Adapter 0: on-line +Adapter 1: off-line +Thermal 0: ok, 46.0 degrees C +Thermal 0: trip point 0 switches to mode critical at temperature 127.0 degrees C +Thermal 0: trip point 1 switches to mode hot at temperature 127.0 degrees C +Thermal 1: ok, 55.0 degrees C +Thermal 1: trip point 0 switches to mode critical at temperature 130.0 degrees C +Thermal 1: trip point 1 switches to mode hot at temperature 100.0 degrees C +Cooling 0: Processor 0 of 10 +Cooling 1: Processor 0 of 10 +Cooling 2: x86_pkg_temp no state information available +Cooling 3: Processor 0 of 10 +Cooling 4: intel_powerclamp no state information available +Cooling 4: another message +Cooling 5: Processor 0 of 10 diff --git a/tests/test_acpi.py b/tests/test_acpi.py index a795f88a..ab60049b 100644 --- a/tests/test_acpi.py +++ b/tests/test_acpi.py @@ -24,6 +24,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/acpi-V.out'), 'r', encoding='utf-8') as f: ubuntu_18_04_acpi_V = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/acpi-V-never-fully-discharge.out'), 'r', encoding='utf-8') as f: + acpi_V_never_fully_discharge = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/acpi-V.json'), 'r', encoding='utf-8') as f: generic_acpi_V_json = json.loads(f.read()) @@ -40,6 +43,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/acpi-V.json'), 'r', encoding='utf-8') as f: ubuntu_18_04_acpi_V_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/acpi-V-never-fully-discharge.json'), 'r', encoding='utf-8') as f: + acpi_V_never_fully_discharge_json = json.loads(f.read()) + def test_acpi_nodata(self): """ Test 'acpi' with no data @@ -76,6 +82,12 @@ class MyTests(unittest.TestCase): """ self.assertEqual(jc.parsers.acpi.parse(self.ubuntu_18_04_acpi_V, quiet=True), self.ubuntu_18_04_acpi_V_json) + def test_acpi_V_never_fully_discharge(self): + """ + Test 'acpi -V' with "never fully discharge" message + """ + self.assertEqual(jc.parsers.acpi.parse(self.acpi_V_never_fully_discharge, quiet=True), self.acpi_V_never_fully_discharge_json) + if __name__ == '__main__': unittest.main() From 098e8dbef6d0fdbaa8d2d6f0560d9ccd8e637d50 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 11:59:06 -0800 Subject: [PATCH 49/81] doc update --- jc/parsers/universal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index b9f392e2..c476803a 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -28,8 +28,7 @@ def simple_table_parse(data: Iterable[str]) -> List[Dict]: underscore '_'. You should also ensure headers are lowercase by using .lower(). - Also, ensure there are no blank lines (list items) - in the data. + Also, ensure there are no blank rows in the data. Returns: From d0b8a91f9451cd90fbc781f5453be28b6b5dee78 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 11:59:19 -0800 Subject: [PATCH 50/81] add zpool-iostat parser --- jc/lib.py | 3 +- jc/parsers/zpool_iostat.py | 131 +++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 jc/parsers/zpool_iostat.py diff --git a/jc/lib.py b/jc/lib.py index 6139acea..748099f3 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -200,7 +200,8 @@ parsers: List[str] = [ 'xml', 'xrandr', 'yaml', - 'zipinfo' + 'zipinfo', + 'zpool-iostat' ] def _cliname_to_modname(parser_cli_name: str) -> str: diff --git a/jc/parsers/zpool_iostat.py b/jc/parsers/zpool_iostat.py new file mode 100644 index 00000000..33f9bae8 --- /dev/null +++ b/jc/parsers/zpool_iostat.py @@ -0,0 +1,131 @@ +"""jc - JSON Convert `zpool iostat` command output parser + +<> + +Usage (cli): + + $ zpool iostat | jc --zpool-iostat + +or + + $ jc zpool iostat + +Usage (module): + + import jc + result = jc.parse('zpool_iostat', zpool_iostat_command_output) + +Schema: + + [ + { + "zpool": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ zpool iostat | jc --zpool-iostat -p + [] + + $ zpool iostat | jc --zpool-iostat -p -r + [] +""" +from typing import List, Dict +from jc.jc_types import JSONDictType +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`zpool iostat` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'freebsd'] + tags = ['command'] + magic_commands = ['zpool iostat'] + + +__version__ = info.version + + +def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[JSONDictType]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List[Dict] = [] + output_line: Dict = {} + pool_parent = '' + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + # skip non-data lines + if '---' in line or \ + line.strip().endswith('bandwidth') or \ + line.strip().endswith('write'): + continue + + # data lines + line_list = line.strip().split() + if line.startswith(' '): + output_line = { + "pool": line_list[0], + "parent": pool_parent + } + + else: + pool_parent = line_list[0] + output_line = { + "pool": pool_parent + } + + output_line.update( + { + 'cap_alloc': line_list[1], + 'cap_free': line_list[2], + 'ops_read': line_list[3], + 'ops_write': line_list[4], + 'bw_read': line_list[5], + 'bw_write': line_list[6] + } + ) + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) From 00274c15df57615968172b8cfe51c11287ba5b61 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 16:52:56 -0800 Subject: [PATCH 51/81] add process conversions --- jc/parsers/zpool_iostat.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/jc/parsers/zpool_iostat.py b/jc/parsers/zpool_iostat.py index 33f9bae8..851889e8 100644 --- a/jc/parsers/zpool_iostat.py +++ b/jc/parsers/zpool_iostat.py @@ -1,6 +1,6 @@ """jc - JSON Convert `zpool iostat` command output parser -<> +Supports with or without the `-v` flag. Usage (cli): @@ -19,9 +19,18 @@ Schema: [ { - "zpool": string, - "bar": boolean, - "baz": integer + "pool": string, + "parent": string, + "cap_alloc": float, + "cap_alloc_unit": string, + "cap_free": float, + "cap_free_unit": string, + "ops_read": integer, + "ops_write": integer, + "bw_read": float, + "bw_read_unit": string, + "bw_write": float, + "bw_write_unit": string } ] @@ -64,6 +73,18 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: List of Dictionaries. Structured to conform to the schema. """ + unit_values = {'cap_alloc', 'cap_free', 'bw_read', 'bw_write'} + int_list = {'ops_read', 'ops_write'} + + for obj in proc_data: + for k, v in obj.copy().items(): + if k in unit_values: + obj[k + '_unit'] = v[-1] + obj[k] = jc.utils.convert_to_float(v[:-1]) + + if k in int_list: + obj[k] = jc.utils.convert_to_int(v) + return proc_data From 9d41f0a93884a1f5b23fc27e83f6fa4bc37e86fe Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 17:00:42 -0800 Subject: [PATCH 52/81] doc update --- README.md | 1 + completions/jc_bash_completion.sh | 4 +- completions/jc_zsh_completion.sh | 6 +- docs/parsers/universal.md | 3 +- docs/parsers/zpool_iostat.md | 125 ++++++++++++++++++++++++++++++ jc/parsers/zpool_iostat.py | 60 ++++++++++++-- man/jc.1 | 5 ++ 7 files changed, 193 insertions(+), 11 deletions(-) create mode 100644 docs/parsers/zpool_iostat.md diff --git a/README.md b/README.md index 3e3b6058..9a03f9b9 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,7 @@ option. | `--xrandr` | `xrandr` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/xrandr) | | `--yaml` | YAML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml) | | `--zipinfo` | `zipinfo` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo) | +| `--zpool-iostat` | `zpool iostat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/zpool_iostat) | ### Options diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 18b826b7..e1e2e8c8 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -3,8 +3,8 @@ _jc() local cur prev words cword jc_commands jc_parsers jc_options \ jc_about_options jc_about_mod_options jc_help_options jc_special_options - jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo zpool) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo --zpool-iostat) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 0e199db7..4521acf0 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -9,7 +9,7 @@ _jc() { jc_help_options jc_help_options_describe \ jc_special_options jc_special_options_describe - jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) + jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo zpool) jc_commands_describe=( 'acpi:run "acpi" command with magic syntax.' 'airport:run "airport" command with magic syntax.' @@ -102,8 +102,9 @@ _jc() { 'who:run "who" command with magic syntax.' 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' + 'zpool:run "zpool" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo --zpool-iostat) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -293,6 +294,7 @@ _jc() { '--xrandr:`xrandr` command parser' '--yaml:YAML file parser' '--zipinfo:`zipinfo` command parser' + '--zpool-iostat:`zpool iostat` command parser' ) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_options_describe=( diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index da8eddd0..ab8b1266 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -42,8 +42,7 @@ Parameters: underscore '_'. You should also ensure headers are lowercase by using .lower(). - Also, ensure there are no blank lines (list items) - in the data. + Also, ensure there are no blank rows in the data. Returns: diff --git a/docs/parsers/zpool_iostat.md b/docs/parsers/zpool_iostat.md new file mode 100644 index 00000000..45f3fd78 --- /dev/null +++ b/docs/parsers/zpool_iostat.md @@ -0,0 +1,125 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.zpool\_iostat + +jc - JSON Convert `zpool iostat` command output parser + +Supports with or without the `-v` flag. + +Usage (cli): + + $ zpool iostat | jc --zpool-iostat + +or + + $ jc zpool iostat + +Usage (module): + + import jc + result = jc.parse('zpool_iostat', zpool_iostat_command_output) + +Schema: + + [ + { + "pool": string, + "parent": string, + "cap_alloc": float, + "cap_alloc_unit": string, + "cap_free": float, + "cap_free_unit": string, + "ops_read": integer, + "ops_write": integer, + "bw_read": float, + "bw_read_unit": string, + "bw_write": float, + "bw_write_unit": string + } + ] + +Examples: + + $ zpool iostat -v | jc --zpool-iostat -p + [ + { + "pool": "zhgstera6", + "cap_alloc": 2.89, + "cap_free": 2.2, + "ops_read": 0, + "ops_write": 2, + "bw_read": 349.0, + "bw_write": 448.0, + "cap_alloc_unit": "T", + "cap_free_unit": "T", + "bw_read_unit": "K", + "bw_write_unit": "K" + }, + { + "pool": "726060ALE614-K8JAPRGN:10", + "parent": "zhgstera6", + "cap_alloc": 2.89, + "cap_free": 2.2, + "ops_read": 0, + "ops_write": 2, + "bw_read": 349.0, + "bw_write": 448.0, + "cap_alloc_unit": "T", + "cap_free_unit": "T", + "bw_read_unit": "K", + "bw_write_unit": "K" + }, + ... + ] + + $ zpool iostat | jc --zpool-iostat -p -r + [ + { + "pool": "zhgstera6", + "cap_alloc": "2.89T", + "cap_free": "2.20T", + "ops_read": "0", + "ops_write": "2", + "bw_read": "349K", + "bw_write": "448K" + }, + { + "pool": "726060ALE614-K8JAPRGN:10", + "parent": "zhgstera6", + "cap_alloc": "2.89T", + "cap_free": "2.20T", + "ops_read": "0", + "ops_write": "2", + "bw_read": "349K", + "bw_write": "448K" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, + raw: bool = False, + quiet: bool = False) -> List[JSONDictType] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/zpool_iostat.py b/jc/parsers/zpool_iostat.py index 851889e8..0f2cba91 100644 --- a/jc/parsers/zpool_iostat.py +++ b/jc/parsers/zpool_iostat.py @@ -36,11 +36,61 @@ Schema: Examples: - $ zpool iostat | jc --zpool-iostat -p - [] + $ zpool iostat -v | jc --zpool-iostat -p + [ + { + "pool": "zhgstera6", + "cap_alloc": 2.89, + "cap_free": 2.2, + "ops_read": 0, + "ops_write": 2, + "bw_read": 349.0, + "bw_write": 448.0, + "cap_alloc_unit": "T", + "cap_free_unit": "T", + "bw_read_unit": "K", + "bw_write_unit": "K" + }, + { + "pool": "726060ALE614-K8JAPRGN:10", + "parent": "zhgstera6", + "cap_alloc": 2.89, + "cap_free": 2.2, + "ops_read": 0, + "ops_write": 2, + "bw_read": 349.0, + "bw_write": 448.0, + "cap_alloc_unit": "T", + "cap_free_unit": "T", + "bw_read_unit": "K", + "bw_write_unit": "K" + }, + ... + ] $ zpool iostat | jc --zpool-iostat -p -r - [] + [ + { + "pool": "zhgstera6", + "cap_alloc": "2.89T", + "cap_free": "2.20T", + "ops_read": "0", + "ops_write": "2", + "bw_read": "349K", + "bw_write": "448K" + }, + { + "pool": "726060ALE614-K8JAPRGN:10", + "parent": "zhgstera6", + "cap_alloc": "2.89T", + "cap_free": "2.20T", + "ops_read": "0", + "ops_write": "2", + "bw_read": "349K", + "bw_write": "448K" + }, + ... + ] """ from typing import List, Dict from jc.jc_types import JSONDictType @@ -127,8 +177,8 @@ def parse( line_list = line.strip().split() if line.startswith(' '): output_line = { - "pool": line_list[0], - "parent": pool_parent + "pool": line_list[0], + "parent": pool_parent } else: diff --git a/man/jc.1 b/man/jc.1 index c159e1d9..a4ea3316 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -982,6 +982,11 @@ YAML file parser \fB--zipinfo\fP `zipinfo` command parser +.TP +.B +\fB--zpool-iostat\fP +`zpool iostat` command parser + .RE .PP From 7361eac1a452412db932fbac54b7eb16d1f31985 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 31 Jan 2023 17:02:04 -0800 Subject: [PATCH 53/81] formatting --- docs/parsers/zpool_iostat.md | 2 +- jc/parsers/zpool_iostat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parsers/zpool_iostat.md b/docs/parsers/zpool_iostat.md index 45f3fd78..1d4f3711 100644 --- a/docs/parsers/zpool_iostat.md +++ b/docs/parsers/zpool_iostat.md @@ -73,7 +73,7 @@ Examples: ... ] - $ zpool iostat | jc --zpool-iostat -p -r + $ zpool iostat -v | jc --zpool-iostat -p -r [ { "pool": "zhgstera6", diff --git a/jc/parsers/zpool_iostat.py b/jc/parsers/zpool_iostat.py index 0f2cba91..bc838094 100644 --- a/jc/parsers/zpool_iostat.py +++ b/jc/parsers/zpool_iostat.py @@ -68,7 +68,7 @@ Examples: ... ] - $ zpool iostat | jc --zpool-iostat -p -r + $ zpool iostat -v | jc --zpool-iostat -p -r [ { "pool": "zhgstera6", From cd8d43446bcf3efa5dc9e09e591254e201c468f8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 2 Feb 2023 15:57:15 -0800 Subject: [PATCH 54/81] add zpool-status parser --- CHANGELOG | 2 + jc/lib.py | 3 +- jc/parsers/zpool_status.py | 154 +++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 jc/parsers/zpool_status.py diff --git a/CHANGELOG b/CHANGELOG index 982bbe9e..ab7d7642 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,8 @@ jc changelog - Add input slicing as a `jc` command-line option - Add `ssh` configuration file parser - Add `ver` Version string parser +- Add `zpool iostat` command parser +- Add `zpool status` command parser - Fix `acpi` command parser for "will never fully discharge" battery state 20230111 v1.22.5 diff --git a/jc/lib.py b/jc/lib.py index 748099f3..bc97d64e 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -201,7 +201,8 @@ parsers: List[str] = [ 'xrandr', 'yaml', 'zipinfo', - 'zpool-iostat' + 'zpool-iostat', + 'zpool-status' ] def _cliname_to_modname(parser_cli_name: str) -> str: diff --git a/jc/parsers/zpool_status.py b/jc/parsers/zpool_status.py new file mode 100644 index 00000000..ef3cfadd --- /dev/null +++ b/jc/parsers/zpool_status.py @@ -0,0 +1,154 @@ +"""jc - JSON Convert `zpool status` command output parser + +<> + +Usage (cli): + + $ zpool status | jc --zpool_status + +or + + $ jc zpool status + +Usage (module): + + import jc + result = jc.parse('zpool_status', zpool_status_command_output) + +Schema: + + [ + { + "zpool status": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ zpool status | jc --zpool status -p + [] + + $ zpool status | jc --zpool status -p -r + [] +""" +from typing import List, Dict +from jc.jc_types import JSONDictType +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`zpool status` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'freebsd'] + tags = ['command'] + magic_commands = ['zpool status'] + + +__version__ = info.version + + +def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[JSONDictType]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List[Dict] = [] + pool_obj: Dict = {} + in_config: bool = False + parent: str = '' + config: List[Dict] = [] + config_obj: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + line_list = line.strip().split(maxsplit=1) + + if line.startswith(' pool: '): + if pool_obj: + if config: + pool_obj['config'] = config + raw_output.append(pool_obj) + + config_obj = {} + config = [] + parent = '' + in_config = False + pool_obj = { + "pool": line_list[1] + } + continue + + if line.startswith(' state: ') \ + or line.startswith(' scan: ') \ + or line.startswith('errors: '): + pool_obj[line_list[0][:-1]] = line_list[1] + in_config = False + continue + + if line.startswith('config:'): + in_config = True + continue + + if in_config and line.strip().endswith('READ WRITE CKSUM'): + continue + + if in_config: + config_line = line.rstrip().split() + config_obj = {} + if line.startswith(' '): + config_obj['parent'] = parent + config_obj['name'] = config_line[0] + else: + parent = config_line[0] + config_obj['name'] = parent + + config_obj['state'] = config_line[1] + config_obj['read'] = config_line[2] + config_obj['write'] = config_line[3] + config_obj['checksum'] = config_line[4] + + config.append(config_obj) + + if pool_obj: + if config: + pool_obj['config'] = config + raw_output.append(pool_obj) + + return raw_output if raw else _process(raw_output) From 64f442d7437bee5d56d14971dd23c5c4c168aa15 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 4 Feb 2023 11:20:48 -0800 Subject: [PATCH 55/81] fix for is_current? --- jc/parsers/xrandr.py | 53 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index 60b51802..89f55981 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -330,8 +330,8 @@ def _parse_mode(line: str) -> Optional[Mode]: for match in result: d = match.groupdict() frequency = float(d["frequency"]) - is_current = len(d["star"]) > 0 - is_preferred = len(d["plus"]) > 0 + is_current = len(d["star"].strip()) > 0 + is_preferred = len(d["plus"].strip()) > 0 f: Frequency = { "frequency": frequency, "is_current": is_current, @@ -376,3 +376,52 @@ def parse(data: str, raw: bool =False, quiet: bool =False) -> Dict: return {} return result + +if __name__ == '__main__': + data = '''\ +Screen 0: minimum 320 x 200, current 2806 x 900, maximum 8192 x 8192 +LVDS-1 connected primary 1366x768+0+0 (normal left inverted right x axis y axis) 344mm x 194mm + 1366x768 60.00*+ + 1280x720 60.00 59.99 59.86 59.74 + 1024x768 60.04 60.00 + 960x720 60.00 + 928x696 60.05 + 896x672 60.01 + 1024x576 59.95 59.96 59.90 59.82 + 960x600 59.93 60.00 + 960x540 59.96 59.99 59.63 59.82 + 800x600 60.00 60.32 56.25 + 840x525 60.01 59.88 + 864x486 59.92 59.57 + 700x525 59.98 + 800x450 59.95 59.82 + 640x512 60.02 + 700x450 59.96 59.88 + 640x480 60.00 59.94 + 720x405 59.51 58.99 + 684x384 59.88 59.85 + 640x400 59.88 59.98 + 640x360 59.86 59.83 59.84 59.32 + 512x384 60.00 + 512x288 60.00 59.92 + 480x270 59.63 59.82 + 400x300 60.32 56.34 + 432x243 59.92 59.57 + 320x240 60.05 + 360x202 59.51 59.13 + 320x180 59.84 59.32 +VGA-1 connected 1440x900+1366+0 normal Y axis (normal left inverted right x axis y axis) 408mm x 255mm + 1440x900 59.89*+ 74.98 + 1280x1024 75.02 60.02 + 1280x960 60.00 + 1280x800 74.93 59.81 + 1152x864 75.00 + 1024x768 75.03 70.07 60.00 + 832x624 74.55 + 800x600 72.19 75.00 60.32 56.25 + 640x480 75.00 72.81 66.67 59.94 + 720x400 70.08 +HDMI-1 disconnected (normal left inverted right x axis y axis) +DP-1 disconnected (normal left inverted right x axis y axis)''' + + parse(data) \ No newline at end of file From 1c09c95c712a3259c3c704baaa7e51d2112e0408 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 4 Feb 2023 11:26:41 -0800 Subject: [PATCH 56/81] doc update --- CHANGELOG | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index ab7d7642..6edeb9d4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,8 @@ jc changelog - Add `zpool iostat` command parser - Add `zpool status` command parser - Fix `acpi` command parser for "will never fully discharge" battery state +- Fix `xrandr` command parser for proper `is_current` output +- Fix `xrandr` command parser for infinite loop with some device configurations 20230111 v1.22.5 - Add TOML file parser From adf5f403ae10248328857954d909422ddcae1a5d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 4 Feb 2023 14:37:03 -0800 Subject: [PATCH 57/81] regex fix for infinite loop? --- jc/parsers/xrandr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index 89f55981..1a799d46 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -252,7 +252,7 @@ _device_pattern = ( + r"(?P primary)? ?" + r"((?P\d+)x(?P\d+)" + r"\+(?P\d+)\+(?P\d+))? " - + r"(?P(inverted|left|right))? ?" + + r"(?P.*?)? ?" + r"\(normal left inverted right x axis y axis\)" + r"( ((?P\d+)mm x (?P\d+)mm)?)?" ) From 96cb01f57a9454a11620f1d4ce03732410826115 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 4 Feb 2023 17:37:48 -0800 Subject: [PATCH 58/81] fix zpool-status for multi-line fields --- jc/parsers/zpool_status.py | 83 ++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/jc/parsers/zpool_status.py b/jc/parsers/zpool_status.py index ef3cfadd..3215dbe3 100644 --- a/jc/parsers/zpool_status.py +++ b/jc/parsers/zpool_status.py @@ -36,6 +36,7 @@ Examples: from typing import List, Dict from jc.jc_types import JSONDictType import jc.utils +from jc.parsers.kv import parse as kv_parse class info(): @@ -67,6 +68,25 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: return proc_data +def _build_config_list(string: str) -> List[Dict]: + config_list: List = [] + for line in filter(None, string.splitlines()): + if line.strip().endswith('READ WRITE CKSUM'): + continue + + line_list = line.strip().split(maxsplit=5) + config_obj: Dict = {} + config_obj['name'] = line_list[0] + config_obj['state'] = line_list[1] + config_obj['read'] = line_list[2] + config_obj['write'] = line_list[3] + config_obj['checksum'] = line_list[4] + if len(line_list) == 6: + config_obj['errors'] = line_list[5] + config_list.append(config_obj) + + return config_list + def parse( data: str, raw: bool = False, @@ -89,66 +109,33 @@ def parse( jc.utils.input_type_check(data) raw_output: List[Dict] = [] + pool_str: str = '' pool_obj: Dict = {} - in_config: bool = False - parent: str = '' - config: List[Dict] = [] - config_obj: Dict = {} if jc.utils.has_data(data): for line in filter(None, data.splitlines()): - line_list = line.strip().split(maxsplit=1) - if line.startswith(' pool: '): - if pool_obj: - if config: - pool_obj['config'] = config + if line.lstrip().startswith('pool: '): + if pool_str: + pool_obj = kv_parse(pool_str) + if 'config' in pool_obj: + pool_obj['config'] = _build_config_list(pool_obj['config']) raw_output.append(pool_obj) - - config_obj = {} - config = [] - parent = '' - in_config = False - pool_obj = { - "pool": line_list[1] - } + pool_str = '' + pool_str += line + '\n' continue - if line.startswith(' state: ') \ - or line.startswith(' scan: ') \ - or line.startswith('errors: '): - pool_obj[line_list[0][:-1]] = line_list[1] - in_config = False + if line.startswith(' '): + pool_str += line + '\n' continue - if line.startswith('config:'): - in_config = True - continue + pool_str += line.strip() + '\n' - if in_config and line.strip().endswith('READ WRITE CKSUM'): - continue - - if in_config: - config_line = line.rstrip().split() - config_obj = {} - if line.startswith(' '): - config_obj['parent'] = parent - config_obj['name'] = config_line[0] - else: - parent = config_line[0] - config_obj['name'] = parent - - config_obj['state'] = config_line[1] - config_obj['read'] = config_line[2] - config_obj['write'] = config_line[3] - config_obj['checksum'] = config_line[4] - - config.append(config_obj) - - if pool_obj: - if config: - pool_obj['config'] = config + if pool_str: + pool_obj = kv_parse(pool_str) + if 'config' in pool_obj: + pool_obj['config'] = _build_config_list(pool_obj['config']) raw_output.append(pool_obj) return raw_output if raw else _process(raw_output) From 00b74be540d634ed4af6ababf3728c89082f569a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 4 Feb 2023 17:37:56 -0800 Subject: [PATCH 59/81] version bump --- jc/parsers/xrandr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index 1a799d46..ba2606be 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -141,7 +141,7 @@ import jc.utils class info: """Provides parser metadata (version, author, etc.)""" - version = "1.1" + version = "1.2" description = "`xrandr` command parser" author = "Kevin Lyter" author_email = "lyter_git at sent.com" From 5c7bf363a6634ac9f93198c477d69fe9c7522817 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 4 Feb 2023 18:51:31 -0800 Subject: [PATCH 60/81] add schema and docs --- jc/parsers/zpool_status.py | 113 ++++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 8 deletions(-) diff --git a/jc/parsers/zpool_status.py b/jc/parsers/zpool_status.py index 3215dbe3..1912e2c2 100644 --- a/jc/parsers/zpool_status.py +++ b/jc/parsers/zpool_status.py @@ -1,6 +1,6 @@ """jc - JSON Convert `zpool status` command output parser -<> +Works with or without the `-v` option. Usage (cli): @@ -19,19 +19,116 @@ Schema: [ { - "zpool status": string, - "bar": boolean, - "baz": integer + "pool": string, + "state": string, + "status": string, + "action": string, + "see": string, + "scan": string, + "scrub": string, + "config": [ + { + "name": string, + "state": string, + "read": integer, + "write": integer, + "checksum": integer, + "errors": string, + } + ], + "errors": string } ] Examples: - $ zpool status | jc --zpool status -p - [] + $ zpool status -v | jc --zpool status -p + [ + { + "pool": "tank", + "state": "DEGRADED", + "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", + "action": "Attach the missing device and online it using 'zpool online'.", + "see": "http://www.sun.com/msg/ZFS-8000-2Q", + "scrub": "none requested", + "config": [ + { + "name": "tank", + "state": "DEGRADED", + "read": 0, + "write": 0, + "checksum": 0 + }, + { + "name": "mirror-0", + "state": "DEGRADED", + "read": 0, + "write": 0, + "checksum": 0 + }, + { + "name": "c1t0d0", + "state": "ONLINE", + "read": 0, + "write": 0, + "checksum": 0 + }, + { + "name": "c1t1d0", + "state": "UNAVAIL", + "read": 0, + "write": 0, + "checksum": 0, + "errors": "cannot open" + } + ], + "errors": "No known data errors" + } + ] - $ zpool status | jc --zpool status -p -r - [] + $ zpool status -v | jc --zpool status -p -r + [ + { + "pool": "tank", + "state": "DEGRADED", + "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", + "action": "Attach the missing device and online it using 'zpool online'.", + "see": "http://www.sun.com/msg/ZFS-8000-2Q", + "scrub": "none requested", + "config": [ + { + "name": "tank", + "state": "DEGRADED", + "read": "0", + "write": "0", + "checksum": "0" + }, + { + "name": "mirror-0", + "state": "DEGRADED", + "read": "0", + "write": "0", + "checksum": "0" + }, + { + "name": "c1t0d0", + "state": "ONLINE", + "read": "0", + "write": "0", + "checksum": "0" + }, + { + "name": "c1t1d0", + "state": "UNAVAIL", + "read": "0", + "write": "0", + "checksum": "0", + "errors": "cannot open" + } + ], + "errors": "No known data errors" + } + ] """ from typing import List, Dict from jc.jc_types import JSONDictType From aada5f0794416b7e8e90beb0c5f8dd8546a0549d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 4 Feb 2023 19:04:43 -0800 Subject: [PATCH 61/81] add int conversions --- jc/parsers/zpool_status.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/jc/parsers/zpool_status.py b/jc/parsers/zpool_status.py index 1912e2c2..78f0baa9 100644 --- a/jc/parsers/zpool_status.py +++ b/jc/parsers/zpool_status.py @@ -162,6 +162,15 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]: List of Dictionaries. Structured to conform to the schema. """ + int_list = {'read', 'write', 'checksum'} + + for obj in proc_data: + if 'config' in obj: + for conf in obj['config']: + for k, v in conf.items(): + if k in int_list: + conf[k] = jc.utils.convert_to_int(v) + return proc_data @@ -223,10 +232,17 @@ def parse( pool_str += line + '\n' continue + # preserve indentation in continuation lines if line.startswith(' '): pool_str += line + '\n' continue + # indent path lines for errors field + if line.startswith('/'): + pool_str += ' ' + line + '\n' + continue + + # remove initial spaces from field start lines so we don't confuse line continuation pool_str += line.strip() + '\n' if pool_str: From c46fe9816c8b6542c4c9e4870a93068b7dafe221 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 5 Feb 2023 09:49:32 -0800 Subject: [PATCH 62/81] doc update --- README.md | 1 + completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 3 +- docs/parsers/xrandr.md | 2 +- docs/parsers/zpool_status.md | 163 ++++++++++++++++++++++++++++++ man/jc.1 | 22 +++- templates/manpage_template | 15 ++- 7 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 docs/parsers/zpool_status.md diff --git a/README.md b/README.md index 9a03f9b9..571a2c85 100644 --- a/README.md +++ b/README.md @@ -294,6 +294,7 @@ option. | `--yaml` | YAML file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml) | | `--zipinfo` | `zipinfo` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo) | | `--zpool-iostat` | `zpool iostat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/zpool_iostat) | +| `--zpool-status` | `zpool status` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/zpool_status) | ### Options diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index e1e2e8c8..8383a66b 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss ssh sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo zpool) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo --zpool-iostat) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo --zpool-iostat --zpool-status) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index 4521acf0..caf0b730 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -104,7 +104,7 @@ _jc() { 'zipinfo:run "zipinfo" command with magic syntax.' 'zpool:run "zpool" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo --zpool-iostat) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --ssh-conf --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --ver --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo --zpool-iostat --zpool-status) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -295,6 +295,7 @@ _jc() { '--yaml:YAML file parser' '--zipinfo:`zipinfo` command parser' '--zpool-iostat:`zpool iostat` command parser' + '--zpool-status:`zpool status` command parser' ) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_options_describe=( diff --git a/docs/parsers/xrandr.md b/docs/parsers/xrandr.md index 924b5bf5..6359fedc 100644 --- a/docs/parsers/xrandr.md +++ b/docs/parsers/xrandr.md @@ -162,4 +162,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, cygwin, aix, freebsd -Version 1.1 by Kevin Lyter (lyter_git at sent.com) +Version 1.2 by Kevin Lyter (lyter_git at sent.com) diff --git a/docs/parsers/zpool_status.md b/docs/parsers/zpool_status.md new file mode 100644 index 00000000..85494c7c --- /dev/null +++ b/docs/parsers/zpool_status.md @@ -0,0 +1,163 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.zpool\_status + +jc - JSON Convert `zpool status` command output parser + +Works with or without the `-v` option. + +Usage (cli): + + $ zpool status | jc --zpool_status + +or + + $ jc zpool status + +Usage (module): + + import jc + result = jc.parse('zpool_status', zpool_status_command_output) + +Schema: + + [ + { + "pool": string, + "state": string, + "status": string, + "action": string, + "see": string, + "scan": string, + "scrub": string, + "config": [ + { + "name": string, + "state": string, + "read": integer, + "write": integer, + "checksum": integer, + "errors": string, + } + ], + "errors": string + } + ] + +Examples: + + $ zpool status -v | jc --zpool status -p + [ + { + "pool": "tank", + "state": "DEGRADED", + "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", + "action": "Attach the missing device and online it using 'zpool online'.", + "see": "http://www.sun.com/msg/ZFS-8000-2Q", + "scrub": "none requested", + "config": [ + { + "name": "tank", + "state": "DEGRADED", + "read": 0, + "write": 0, + "checksum": 0 + }, + { + "name": "mirror-0", + "state": "DEGRADED", + "read": 0, + "write": 0, + "checksum": 0 + }, + { + "name": "c1t0d0", + "state": "ONLINE", + "read": 0, + "write": 0, + "checksum": 0 + }, + { + "name": "c1t1d0", + "state": "UNAVAIL", + "read": 0, + "write": 0, + "checksum": 0, + "errors": "cannot open" + } + ], + "errors": "No known data errors" + } + ] + + $ zpool status -v | jc --zpool status -p -r + [ + { + "pool": "tank", + "state": "DEGRADED", + "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", + "action": "Attach the missing device and online it using 'zpool online'.", + "see": "http://www.sun.com/msg/ZFS-8000-2Q", + "scrub": "none requested", + "config": [ + { + "name": "tank", + "state": "DEGRADED", + "read": "0", + "write": "0", + "checksum": "0" + }, + { + "name": "mirror-0", + "state": "DEGRADED", + "read": "0", + "write": "0", + "checksum": "0" + }, + { + "name": "c1t0d0", + "state": "ONLINE", + "read": "0", + "write": "0", + "checksum": "0" + }, + { + "name": "c1t1d0", + "state": "UNAVAIL", + "read": "0", + "write": "0", + "checksum": "0", + "errors": "cannot open" + } + ], + "errors": "No known data errors" + } + ] + + + +### parse + +```python +def parse(data: str, + raw: bool = False, + quiet: bool = False) -> List[JSONDictType] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/man/jc.1 b/man/jc.1 index a4ea3316..85f33dcd 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-01-31 1.23.0 "JSON Convert" +.TH jc 1 2023-02-05 1.23.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings @@ -987,6 +987,11 @@ YAML file parser \fB--zpool-iostat\fP `zpool iostat` command parser +.TP +.B +\fB--zpool-status\fP +`zpool status` command parser + .RE .PP @@ -1011,7 +1016,7 @@ Debug - show traceback (use \fB-dd\fP for verbose traceback) .B \fB-h\fP, \fB--help\fP Help (\fB--help --parser_name\fP for parser documentation). Use twice to show -\hidden parsers (e.g. \fB-hh\fP) +hidden parsers (e.g. \fB-hh\fP) .TP .B \fB-m\fP, \fB--monochrome\fP @@ -1380,10 +1385,23 @@ $ jc \fB--pretty\fP dig www.google.com $ jc \fB--pretty\fP /proc/meminfo .RE +Line Slicing: +.RS +$ cat file.csv | jc \fB:101\fP \fB--csv\fP # parse first 100 lines +.RE + For parser documentation: .RS $ jc \fB--help\fP \fB--dig\fP .RE + +More Help: +.RS +$ jc \fB-hh\fP # show hidden parsers + +$ jc \fB-hhh\fP # list parsers by category tags +.RE + .SH AUTHOR Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/templates/manpage_template b/templates/manpage_template index 05b0f782..9630e8cb 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -66,7 +66,7 @@ Debug - show traceback (use \fB-dd\fP for verbose traceback) .B \fB-h\fP, \fB--help\fP Help (\fB--help --parser_name\fP for parser documentation). Use twice to show -\hidden parsers (e.g. \fB-hh\fP) +hidden parsers (e.g. \fB-hh\fP) .TP .B \fB-m\fP, \fB--monochrome\fP @@ -435,10 +435,23 @@ $ jc \fB--pretty\fP dig www.google.com $ jc \fB--pretty\fP /proc/meminfo .RE +Line Slicing: +.RS +$ cat file.csv | jc \fB:101\fP \fB--csv\fP # parse first 100 lines +.RE + For parser documentation: .RS $ jc \fB--help\fP \fB--dig\fP .RE + +More Help: +.RS +$ jc \fB-hh\fP # show hidden parsers + +$ jc \fB-hhh\fP # list parsers by category tags +.RE + .SH AUTHOR {{ jc.author }} ({{ jc.author_email }}) From 193ddf71f8c8a74598ca9e7d2b20bcf172d629dc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 5 Feb 2023 09:52:59 -0800 Subject: [PATCH 63/81] fix typos --- docs/parsers/zpool_status.md | 6 +++--- jc/parsers/zpool_status.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/parsers/zpool_status.md b/docs/parsers/zpool_status.md index 85494c7c..bd320050 100644 --- a/docs/parsers/zpool_status.md +++ b/docs/parsers/zpool_status.md @@ -9,7 +9,7 @@ Works with or without the `-v` option. Usage (cli): - $ zpool status | jc --zpool_status + $ zpool status | jc --zpool-status or @@ -47,7 +47,7 @@ Schema: Examples: - $ zpool status -v | jc --zpool status -p + $ zpool status -v | jc --zpool-status -p [ { "pool": "tank", @@ -91,7 +91,7 @@ Examples: } ] - $ zpool status -v | jc --zpool status -p -r + $ zpool status -v | jc --zpool-status -p -r [ { "pool": "tank", diff --git a/jc/parsers/zpool_status.py b/jc/parsers/zpool_status.py index 78f0baa9..a16a6d98 100644 --- a/jc/parsers/zpool_status.py +++ b/jc/parsers/zpool_status.py @@ -4,7 +4,7 @@ Works with or without the `-v` option. Usage (cli): - $ zpool status | jc --zpool_status + $ zpool status | jc --zpool-status or @@ -42,7 +42,7 @@ Schema: Examples: - $ zpool status -v | jc --zpool status -p + $ zpool status -v | jc --zpool-status -p [ { "pool": "tank", @@ -86,7 +86,7 @@ Examples: } ] - $ zpool status -v | jc --zpool status -p -r + $ zpool status -v | jc --zpool-status -p -r [ { "pool": "tank", From 5c7a520a0b59b433c9effe65f119aa70c9ce1be5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 5 Feb 2023 13:04:32 -0800 Subject: [PATCH 64/81] formatting --- docs/parsers/zpool_status.md | 8 ++++---- jc/parsers/zpool_status.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/parsers/zpool_status.md b/docs/parsers/zpool_status.md index bd320050..b0b84a52 100644 --- a/docs/parsers/zpool_status.md +++ b/docs/parsers/zpool_status.md @@ -52,8 +52,8 @@ Examples: { "pool": "tank", "state": "DEGRADED", - "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", - "action": "Attach the missing device and online it using 'zpool online'.", + "status": "One or more devices could not be opened. Suffic...", + "action": "Attach the missing device and online it using 'zpool...", "see": "http://www.sun.com/msg/ZFS-8000-2Q", "scrub": "none requested", "config": [ @@ -96,8 +96,8 @@ Examples: { "pool": "tank", "state": "DEGRADED", - "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", - "action": "Attach the missing device and online it using 'zpool online'.", + "status": "One or more devices could not be opened. Sufficient...", + "action": "Attach the missing device and online it using 'zpool...", "see": "http://www.sun.com/msg/ZFS-8000-2Q", "scrub": "none requested", "config": [ diff --git a/jc/parsers/zpool_status.py b/jc/parsers/zpool_status.py index a16a6d98..27c831b1 100644 --- a/jc/parsers/zpool_status.py +++ b/jc/parsers/zpool_status.py @@ -47,8 +47,8 @@ Examples: { "pool": "tank", "state": "DEGRADED", - "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", - "action": "Attach the missing device and online it using 'zpool online'.", + "status": "One or more devices could not be opened. Suffic...", + "action": "Attach the missing device and online it using 'zpool...", "see": "http://www.sun.com/msg/ZFS-8000-2Q", "scrub": "none requested", "config": [ @@ -91,8 +91,8 @@ Examples: { "pool": "tank", "state": "DEGRADED", - "status": "One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.", - "action": "Attach the missing device and online it using 'zpool online'.", + "status": "One or more devices could not be opened. Sufficient...", + "action": "Attach the missing device and online it using 'zpool...", "see": "http://www.sun.com/msg/ZFS-8000-2Q", "scrub": "none requested", "config": [ From 7486b0c7cd83b922ccc30ed797598433f2b7733f Mon Sep 17 00:00:00 2001 From: Jake Ob Date: Tue, 7 Feb 2023 12:39:44 +0200 Subject: [PATCH 65/81] Move the reflect value in its own key (xrandr) Currently the reflect value of a device is included in the rotation key as a combination of both rotation and reflection (e.g. normal X axis). With this modification we move the reflection value to its own key in the JSON document like so: ```json { ... "rotation": "...", "reflection": "..., ... } ``` Fixes: issue #360 --- jc/parsers/xrandr.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index ba2606be..283cf19b 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -50,7 +50,8 @@ Schema: "offset_height": integer, "dimension_width": integer, "dimension_height": integer, - "rotation": string + "rotation": string, + "reflection": string } ], "unassociated_devices": [ @@ -127,7 +128,8 @@ Examples: "offset_height": 0, "dimension_width": 310, "dimension_height": 170, - "rotation": "normal" + "rotation": "normal", + "reflection": "normal" } } ], @@ -185,6 +187,8 @@ try: "dimension_width": int, "dimension_height": int, "associated_modes": List[Mode], + "rotation": str, + "reflection": str, }, ) Screen = TypedDict( @@ -252,7 +256,8 @@ _device_pattern = ( + r"(?P primary)? ?" + r"((?P\d+)x(?P\d+)" + r"\+(?P\d+)\+(?P\d+))? " - + r"(?P.*?)? ?" + + r"(?P(normal|right|left|inverted)?) ?" + + r"(?P(X axis|Y axis|X and Y axis)?) ?" + r"\(normal left inverted right x axis y axis\)" + r"( ((?P\d+)mm x (?P\d+)mm)?)?" ) @@ -277,9 +282,10 @@ def _parse_device(next_lines: List[str], quiet: bool = False) -> Optional[Device and len(matches["is_primary"]) > 0, "device_name": matches["device_name"], "rotation": matches["rotation"] or "normal", + "reflection": matches["reflection"] or "normal", } for k, v in matches.items(): - if k not in {"is_connected", "is_primary", "device_name", "rotation"}: + if k not in {"is_connected", "is_primary", "device_name", "rotation", "reflection"}: try: if v: device[k] = int(v) From 9dde65c25c05fa68de413f8273969c46e01adb9e Mon Sep 17 00:00:00 2001 From: Jean-Pierre Matsumoto Date: Tue, 7 Feb 2023 14:57:15 +0100 Subject: [PATCH 66/81] Improve accepted formats in proc_pid_smaps --- jc/parsers/proc_pid_smaps.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jc/parsers/proc_pid_smaps.py b/jc/parsers/proc_pid_smaps.py index 921b4a2c..e20c3882 100644 --- a/jc/parsers/proc_pid_smaps.py +++ b/jc/parsers/proc_pid_smaps.py @@ -109,7 +109,8 @@ Examples: "mw", "me", "dw", - "sd" + "sd", + "mp" ], "VmFlags_pretty": [ "readable", @@ -211,6 +212,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: 'mw': 'may write', 'me': 'may execute', 'ms': 'may share', + 'mp': 'MPX-specific VMA', 'gd': 'stack segment growns down', 'pf': 'pure PFN range', 'dw': 'disabled write to the mapped file', @@ -274,10 +276,10 @@ def parse( if jc.utils.has_data(data): map_line = re.compile(r''' - ^(?P[0-9a-f]{12,16})- - (?P[0-9a-f]{12,16})\s + ^(?P[0-9a-f]{8,16})- + (?P[0-9a-f]{8,16})\s (?P[rwxsp\-]{4})\s - (?P[0-9a-f]{8})\s + (?P[0-9a-f]{8,9})\s (?P[0-9a-f]{2}): (?P[0-9a-f]{2})\s (?P\d+)\s+ From 80fb4d40a506a51a44551e02d6fc53120e6f1e13 Mon Sep 17 00:00:00 2001 From: Jake Ob Date: Thu, 9 Feb 2023 21:22:34 +0200 Subject: [PATCH 67/81] Add unit test to assert devices in reflect mode --- tests/fixtures/generic/xrandr_simple.json | 1 + tests/test_xrandr.py | 26 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/fixtures/generic/xrandr_simple.json b/tests/fixtures/generic/xrandr_simple.json index e03e79c8..0f1ebe49 100644 --- a/tests/fixtures/generic/xrandr_simple.json +++ b/tests/fixtures/generic/xrandr_simple.json @@ -44,6 +44,7 @@ "is_primary": true, "device_name": "eDP1", "rotation": "normal", + "reflection": "normal", "resolution_width": 1920, "resolution_height": 1080, "offset_width": 0, diff --git a/tests/test_xrandr.py b/tests/test_xrandr.py index 8fce69b7..dea94f71 100644 --- a/tests/test_xrandr.py +++ b/tests/test_xrandr.py @@ -32,6 +32,8 @@ class XrandrTests(unittest.TestCase): "eDP1 connected primary 1920x1080+0+0 (normal left inverted right x axis y axis) 310mm x 170mm", "eDP-1 connected primary 1920x1080+0+0 (normal left inverted right x axis y axis) 309mm x 174mm", "HDMI-0 connected 2160x3840+3840+0 right (normal left inverted right x axis y axis) 609mm x 349mm", + "LVDS-1 connected primary 1366x768+0+0 normal X axis (normal left inverted right x axis y axis) 609mm x 349mm", + "VGA-1 connected 1280x1024+0+0 left X and Y axis (normal left inverted right x axis y axis) 609mm x 349mm", ] for device in devices: self.assertIsNotNone(re.match(_device_pattern, device)) @@ -118,6 +120,30 @@ class XrandrTests(unittest.TestCase): 59.94, device["associated_modes"][12]["frequencies"][4]["frequency"] ) + def test_device_with_reflect(self): + sample = "VGA-1 connected primary 1920x1080+0+0 left X and Y axis (normal left inverted right x axis y axis) 310mm x 170mm" + actual: Optional[Device] = _parse_device([sample]) + + expected = { + "device_name": "VGA-1", + "is_connected": True, + "is_primary": True, + "resolution_width": 1920, + "resolution_height": 1080, + "offset_width": 0, + "offset_height": 0, + "dimension_width": 310, + "dimension_height": 170, + "rotation": "left", + "reflection": "X and Y axis", + } + + self.assertIsNotNone(actual) + + if actual: + for k, v in expected.items(): + self.assertEqual(v, actual[k], f"Devices regex failed on {k}") + def test_mode(self): sample_1 = "1920x1080 60.03*+ 59.93" expected = { From 81982a9f79eb157b0abae56109c456055d552333 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 12 Feb 2023 18:30:31 -0800 Subject: [PATCH 68/81] doc updagte --- CHANGELOG | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 6edeb9d4..d2061e35 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,8 @@ jc changelog - Fix `acpi` command parser for "will never fully discharge" battery state - Fix `xrandr` command parser for proper `is_current` output - Fix `xrandr` command parser for infinite loop with some device configurations +- Add `reflection` key to `xrandr` parser schema +- Add `MPX-specific VMA` support for VM Flags in `/proc//smaps` parser 20230111 v1.22.5 - Add TOML file parser From 8536514bafc1e8c363543d75497d94ab70288de3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 13 Feb 2023 18:09:10 -0800 Subject: [PATCH 69/81] add test for output with extra spaces that caused infinite loops --- tests/fixtures/generic/xrandr_fix_spaces.json | 1 + tests/fixtures/generic/xrandr_fix_spaces.out | 44 +++++++++++++++++++ tests/test_xrandr.py | 13 ++++-- 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/generic/xrandr_fix_spaces.json create mode 100644 tests/fixtures/generic/xrandr_fix_spaces.out diff --git a/tests/fixtures/generic/xrandr_fix_spaces.json b/tests/fixtures/generic/xrandr_fix_spaces.json new file mode 100644 index 00000000..98ee6634 --- /dev/null +++ b/tests/fixtures/generic/xrandr_fix_spaces.json @@ -0,0 +1 @@ +{"screens":[{"screen_number":0,"minimum_width":320,"minimum_height":200,"current_width":1846,"current_height":768,"maximum_width":8192,"maximum_height":8192,"associated_device":{"associated_modes":[{"resolution_width":1366,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":true,"is_preferred":true}]},{"resolution_width":1280,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.74,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.04,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":928,"resolution_height":696,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":896,"resolution_height":672,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":576,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.9,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":59.93,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":540,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":840,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":864,"resolution_height":486,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":512,"is_high_resolution":false,"frequencies":[{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":405,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":58.99,"is_current":false,"is_preferred":false}]},{"resolution_width":684,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.85,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":360,"is_high_resolution":false,"frequencies":[{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.83,"is_current":false,"is_preferred":false},{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":288,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.92,"is_current":false,"is_preferred":false}]},{"resolution_width":480,"resolution_height":270,"is_high_resolution":false,"frequencies":[{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":400,"resolution_height":300,"is_high_resolution":false,"frequencies":[{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.34,"is_current":false,"is_preferred":false}]},{"resolution_width":432,"resolution_height":243,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":240,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":360,"resolution_height":202,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":59.13,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":180,"is_high_resolution":false,"frequencies":[{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":true,"device_name":"LVDS-1","rotation":"normal","reflection":"normal","resolution_width":1366,"resolution_height":768,"offset_width":0,"offset_height":0,"dimension_width":344,"dimension_height":194}}],"unassociated_devices":[{"associated_modes":[{"resolution_width":1440,"resolution_height":900,"is_high_resolution":false,"frequencies":[{"frequency":59.89,"is_current":false,"is_preferred":true},{"frequency":74.98,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":1024,"is_high_resolution":false,"frequencies":[{"frequency":75.02,"is_current":false,"is_preferred":false},{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":960,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":800,"is_high_resolution":false,"frequencies":[{"frequency":74.93,"is_current":false,"is_preferred":false},{"frequency":59.81,"is_current":false,"is_preferred":false}]},{"resolution_width":1152,"resolution_height":864,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":75.03,"is_current":false,"is_preferred":false},{"frequency":70.07,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":832,"resolution_height":624,"is_high_resolution":false,"frequencies":[{"frequency":74.55,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":72.19,"is_current":false,"is_preferred":false},{"frequency":75.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":true,"is_preferred":false},{"frequency":72.81,"is_current":false,"is_preferred":false},{"frequency":66.67,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":70.08,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":false,"device_name":"VGA-1","rotation":"left","reflection":"normal","resolution_width":480,"resolution_height":640,"offset_width":1366,"offset_height":0,"dimension_width":408,"dimension_height":255},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"HDMI-1","rotation":"normal","reflection":"normal"},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"DP-1","rotation":"normal","reflection":"normal"}]} diff --git a/tests/fixtures/generic/xrandr_fix_spaces.out b/tests/fixtures/generic/xrandr_fix_spaces.out new file mode 100644 index 00000000..452a6462 --- /dev/null +++ b/tests/fixtures/generic/xrandr_fix_spaces.out @@ -0,0 +1,44 @@ +Screen 0: minimum 320 x 200, current 1846 x 768, maximum 8192 x 8192 +LVDS-1 connected primary 1366x768+0+0 (normal left inverted right x axis y axis) 344mm x 194mm + 1366x768 60.00*+ + 1280x720 60.00 59.99 59.86 59.74 + 1024x768 60.04 60.00 + 960x720 60.00 + 928x696 60.05 + 896x672 60.01 + 1024x576 59.95 59.96 59.90 59.82 + 960x600 59.93 60.00 + 960x540 59.96 59.99 59.63 59.82 + 800x600 60.00 60.32 56.25 + 840x525 60.01 59.88 + 864x486 59.92 59.57 + 700x525 59.98 + 800x450 59.95 59.82 + 640x512 60.02 + 700x450 59.96 59.88 + 640x480 60.00 59.94 + 720x405 59.51 58.99 + 684x384 59.88 59.85 + 640x400 59.88 59.98 + 640x360 59.86 59.83 59.84 59.32 + 512x384 60.00 + 512x288 60.00 59.92 + 480x270 59.63 59.82 + 400x300 60.32 56.34 + 432x243 59.92 59.57 + 320x240 60.05 + 360x202 59.51 59.13 + 320x180 59.84 59.32 +VGA-1 connected 480x640+1366+0 left (normal left inverted right x axis y axis) 408mm x 255mm + 1440x900 59.89 + 74.98 + 1280x1024 75.02 60.02 + 1280x960 60.00 + 1280x800 74.93 59.81 + 1152x864 75.00 + 1024x768 75.03 70.07 60.00 + 832x624 74.55 + 800x600 72.19 75.00 60.32 56.25 + 640x480 75.00* 72.81 66.67 59.94 + 720x400 70.08 +HDMI-1 disconnected (normal left inverted right x axis y axis) +DP-1 disconnected (normal left inverted right x axis y axis) diff --git a/tests/test_xrandr.py b/tests/test_xrandr.py index dea94f71..1d1e927c 100644 --- a/tests/test_xrandr.py +++ b/tests/test_xrandr.py @@ -196,14 +196,21 @@ class XrandrTests(unittest.TestCase): txt = f.read() actual = parse(txt, quiet=True) - with open("tests/fixtures/generic/xrandr_simple.json", "w") as f: - json.dump(actual, f, indent=True) - self.assertEqual(1, len(actual["screens"])) self.assertEqual(0, len(actual["unassociated_devices"])) self.assertEqual( 2, len(actual["screens"][0]["associated_device"]["associated_modes"]) ) + def test_infinite_loop_fix(self): + with open("tests/fixtures/generic/xrandr_fix_spaces.out", "r") as f: + txt = f.read() + actual = parse(txt, quiet=True) + + with open("tests/fixtures/generic/xrandr_fix_spaces.json", "r") as f: + json_dict = json.loads(f.read()) + + self.assertEqual(actual, json_dict) + if __name__ == '__main__': unittest.main() From ae19183803269453a5764ae0473cab54afc72b3f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 13 Feb 2023 18:17:14 -0800 Subject: [PATCH 70/81] add test for is_current fix --- tests/fixtures/generic/xrandr_fix_spaces.json | 2 +- tests/fixtures/generic/xrandr_fix_spaces.out | 8 ++-- .../generic/xrandr_is_current_fix.json | 1 + .../generic/xrandr_is_current_fix.out | 44 +++++++++++++++++++ tests/test_xrandr.py | 10 +++++ 5 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/generic/xrandr_is_current_fix.json create mode 100644 tests/fixtures/generic/xrandr_is_current_fix.out diff --git a/tests/fixtures/generic/xrandr_fix_spaces.json b/tests/fixtures/generic/xrandr_fix_spaces.json index 98ee6634..aac9c63a 100644 --- a/tests/fixtures/generic/xrandr_fix_spaces.json +++ b/tests/fixtures/generic/xrandr_fix_spaces.json @@ -1 +1 @@ -{"screens":[{"screen_number":0,"minimum_width":320,"minimum_height":200,"current_width":1846,"current_height":768,"maximum_width":8192,"maximum_height":8192,"associated_device":{"associated_modes":[{"resolution_width":1366,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":true,"is_preferred":true}]},{"resolution_width":1280,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.74,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.04,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":928,"resolution_height":696,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":896,"resolution_height":672,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":576,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.9,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":59.93,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":540,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":840,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":864,"resolution_height":486,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":512,"is_high_resolution":false,"frequencies":[{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":405,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":58.99,"is_current":false,"is_preferred":false}]},{"resolution_width":684,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.85,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":360,"is_high_resolution":false,"frequencies":[{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.83,"is_current":false,"is_preferred":false},{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":288,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.92,"is_current":false,"is_preferred":false}]},{"resolution_width":480,"resolution_height":270,"is_high_resolution":false,"frequencies":[{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":400,"resolution_height":300,"is_high_resolution":false,"frequencies":[{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.34,"is_current":false,"is_preferred":false}]},{"resolution_width":432,"resolution_height":243,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":240,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":360,"resolution_height":202,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":59.13,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":180,"is_high_resolution":false,"frequencies":[{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":true,"device_name":"LVDS-1","rotation":"normal","reflection":"normal","resolution_width":1366,"resolution_height":768,"offset_width":0,"offset_height":0,"dimension_width":344,"dimension_height":194}}],"unassociated_devices":[{"associated_modes":[{"resolution_width":1440,"resolution_height":900,"is_high_resolution":false,"frequencies":[{"frequency":59.89,"is_current":false,"is_preferred":true},{"frequency":74.98,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":1024,"is_high_resolution":false,"frequencies":[{"frequency":75.02,"is_current":false,"is_preferred":false},{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":960,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":800,"is_high_resolution":false,"frequencies":[{"frequency":74.93,"is_current":false,"is_preferred":false},{"frequency":59.81,"is_current":false,"is_preferred":false}]},{"resolution_width":1152,"resolution_height":864,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":75.03,"is_current":false,"is_preferred":false},{"frequency":70.07,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":832,"resolution_height":624,"is_high_resolution":false,"frequencies":[{"frequency":74.55,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":72.19,"is_current":false,"is_preferred":false},{"frequency":75.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":true,"is_preferred":false},{"frequency":72.81,"is_current":false,"is_preferred":false},{"frequency":66.67,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":70.08,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":false,"device_name":"VGA-1","rotation":"left","reflection":"normal","resolution_width":480,"resolution_height":640,"offset_width":1366,"offset_height":0,"dimension_width":408,"dimension_height":255},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"HDMI-1","rotation":"normal","reflection":"normal"},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"DP-1","rotation":"normal","reflection":"normal"}]} +{"screens":[{"screen_number":0,"minimum_width":320,"minimum_height":200,"current_width":2806,"current_height":900,"maximum_width":8192,"maximum_height":8192,"associated_device":{"associated_modes":[{"resolution_width":1366,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":true,"is_preferred":true}]},{"resolution_width":1280,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.74,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.04,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":928,"resolution_height":696,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":896,"resolution_height":672,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":576,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.9,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":59.93,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":540,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":840,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":864,"resolution_height":486,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":512,"is_high_resolution":false,"frequencies":[{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":405,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":58.99,"is_current":false,"is_preferred":false}]},{"resolution_width":684,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.85,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":360,"is_high_resolution":false,"frequencies":[{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.83,"is_current":false,"is_preferred":false},{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":288,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.92,"is_current":false,"is_preferred":false}]},{"resolution_width":480,"resolution_height":270,"is_high_resolution":false,"frequencies":[{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":400,"resolution_height":300,"is_high_resolution":false,"frequencies":[{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.34,"is_current":false,"is_preferred":false}]},{"resolution_width":432,"resolution_height":243,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":240,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":360,"resolution_height":202,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":59.13,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":180,"is_high_resolution":false,"frequencies":[{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":true,"device_name":"LVDS-1","rotation":"normal","reflection":"normal","resolution_width":1366,"resolution_height":768,"offset_width":0,"offset_height":0,"dimension_width":344,"dimension_height":194}}],"unassociated_devices":[{"associated_modes":[{"resolution_width":1440,"resolution_height":900,"is_high_resolution":false,"frequencies":[{"frequency":59.89,"is_current":true,"is_preferred":true},{"frequency":74.98,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":1024,"is_high_resolution":false,"frequencies":[{"frequency":75.02,"is_current":false,"is_preferred":false},{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":960,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":800,"is_high_resolution":false,"frequencies":[{"frequency":74.93,"is_current":false,"is_preferred":false},{"frequency":59.81,"is_current":false,"is_preferred":false}]},{"resolution_width":1152,"resolution_height":864,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":75.03,"is_current":false,"is_preferred":false},{"frequency":70.07,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":832,"resolution_height":624,"is_high_resolution":false,"frequencies":[{"frequency":74.55,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":72.19,"is_current":false,"is_preferred":false},{"frequency":75.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":false,"is_preferred":false},{"frequency":72.81,"is_current":false,"is_preferred":false},{"frequency":66.67,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":70.08,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":false,"device_name":"VGA-1","rotation":"normal","reflection":"Y axis","resolution_width":1440,"resolution_height":900,"offset_width":1366,"offset_height":0,"dimension_width":408,"dimension_height":255},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"HDMI-1","rotation":"normal","reflection":"normal"},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"DP-1","rotation":"normal","reflection":"normal"}]} diff --git a/tests/fixtures/generic/xrandr_fix_spaces.out b/tests/fixtures/generic/xrandr_fix_spaces.out index 452a6462..f45bb098 100644 --- a/tests/fixtures/generic/xrandr_fix_spaces.out +++ b/tests/fixtures/generic/xrandr_fix_spaces.out @@ -1,4 +1,4 @@ -Screen 0: minimum 320 x 200, current 1846 x 768, maximum 8192 x 8192 +Screen 0: minimum 320 x 200, current 2806 x 900, maximum 8192 x 8192 LVDS-1 connected primary 1366x768+0+0 (normal left inverted right x axis y axis) 344mm x 194mm 1366x768 60.00*+ 1280x720 60.00 59.99 59.86 59.74 @@ -29,8 +29,8 @@ LVDS-1 connected primary 1366x768+0+0 (normal left inverted right x axis y axis) 320x240 60.05 360x202 59.51 59.13 320x180 59.84 59.32 -VGA-1 connected 480x640+1366+0 left (normal left inverted right x axis y axis) 408mm x 255mm - 1440x900 59.89 + 74.98 +VGA-1 connected 1440x900+1366+0 normal Y axis (normal left inverted right x axis y axis) 408mm x 255mm + 1440x900 59.89*+ 74.98 1280x1024 75.02 60.02 1280x960 60.00 1280x800 74.93 59.81 @@ -38,7 +38,7 @@ VGA-1 connected 480x640+1366+0 left (normal left inverted right x axis y axis) 4 1024x768 75.03 70.07 60.00 832x624 74.55 800x600 72.19 75.00 60.32 56.25 - 640x480 75.00* 72.81 66.67 59.94 + 640x480 75.00 72.81 66.67 59.94 720x400 70.08 HDMI-1 disconnected (normal left inverted right x axis y axis) DP-1 disconnected (normal left inverted right x axis y axis) diff --git a/tests/fixtures/generic/xrandr_is_current_fix.json b/tests/fixtures/generic/xrandr_is_current_fix.json new file mode 100644 index 00000000..98ee6634 --- /dev/null +++ b/tests/fixtures/generic/xrandr_is_current_fix.json @@ -0,0 +1 @@ +{"screens":[{"screen_number":0,"minimum_width":320,"minimum_height":200,"current_width":1846,"current_height":768,"maximum_width":8192,"maximum_height":8192,"associated_device":{"associated_modes":[{"resolution_width":1366,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":true,"is_preferred":true}]},{"resolution_width":1280,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.74,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":60.04,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":720,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":928,"resolution_height":696,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":896,"resolution_height":672,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":576,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.9,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":59.93,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":960,"resolution_height":540,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.99,"is_current":false,"is_preferred":false},{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":840,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":60.01,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":864,"resolution_height":486,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":525,"is_high_resolution":false,"frequencies":[{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.95,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":512,"is_high_resolution":false,"frequencies":[{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":700,"resolution_height":450,"is_high_resolution":false,"frequencies":[{"frequency":59.96,"is_current":false,"is_preferred":false},{"frequency":59.88,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":405,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":58.99,"is_current":false,"is_preferred":false}]},{"resolution_width":684,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.85,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":59.88,"is_current":false,"is_preferred":false},{"frequency":59.98,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":360,"is_high_resolution":false,"frequencies":[{"frequency":59.86,"is_current":false,"is_preferred":false},{"frequency":59.83,"is_current":false,"is_preferred":false},{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":384,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":512,"resolution_height":288,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false},{"frequency":59.92,"is_current":false,"is_preferred":false}]},{"resolution_width":480,"resolution_height":270,"is_high_resolution":false,"frequencies":[{"frequency":59.63,"is_current":false,"is_preferred":false},{"frequency":59.82,"is_current":false,"is_preferred":false}]},{"resolution_width":400,"resolution_height":300,"is_high_resolution":false,"frequencies":[{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.34,"is_current":false,"is_preferred":false}]},{"resolution_width":432,"resolution_height":243,"is_high_resolution":false,"frequencies":[{"frequency":59.92,"is_current":false,"is_preferred":false},{"frequency":59.57,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":240,"is_high_resolution":false,"frequencies":[{"frequency":60.05,"is_current":false,"is_preferred":false}]},{"resolution_width":360,"resolution_height":202,"is_high_resolution":false,"frequencies":[{"frequency":59.51,"is_current":false,"is_preferred":false},{"frequency":59.13,"is_current":false,"is_preferred":false}]},{"resolution_width":320,"resolution_height":180,"is_high_resolution":false,"frequencies":[{"frequency":59.84,"is_current":false,"is_preferred":false},{"frequency":59.32,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":true,"device_name":"LVDS-1","rotation":"normal","reflection":"normal","resolution_width":1366,"resolution_height":768,"offset_width":0,"offset_height":0,"dimension_width":344,"dimension_height":194}}],"unassociated_devices":[{"associated_modes":[{"resolution_width":1440,"resolution_height":900,"is_high_resolution":false,"frequencies":[{"frequency":59.89,"is_current":false,"is_preferred":true},{"frequency":74.98,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":1024,"is_high_resolution":false,"frequencies":[{"frequency":75.02,"is_current":false,"is_preferred":false},{"frequency":60.02,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":960,"is_high_resolution":false,"frequencies":[{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1280,"resolution_height":800,"is_high_resolution":false,"frequencies":[{"frequency":74.93,"is_current":false,"is_preferred":false},{"frequency":59.81,"is_current":false,"is_preferred":false}]},{"resolution_width":1152,"resolution_height":864,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":false,"is_preferred":false}]},{"resolution_width":1024,"resolution_height":768,"is_high_resolution":false,"frequencies":[{"frequency":75.03,"is_current":false,"is_preferred":false},{"frequency":70.07,"is_current":false,"is_preferred":false},{"frequency":60.0,"is_current":false,"is_preferred":false}]},{"resolution_width":832,"resolution_height":624,"is_high_resolution":false,"frequencies":[{"frequency":74.55,"is_current":false,"is_preferred":false}]},{"resolution_width":800,"resolution_height":600,"is_high_resolution":false,"frequencies":[{"frequency":72.19,"is_current":false,"is_preferred":false},{"frequency":75.0,"is_current":false,"is_preferred":false},{"frequency":60.32,"is_current":false,"is_preferred":false},{"frequency":56.25,"is_current":false,"is_preferred":false}]},{"resolution_width":640,"resolution_height":480,"is_high_resolution":false,"frequencies":[{"frequency":75.0,"is_current":true,"is_preferred":false},{"frequency":72.81,"is_current":false,"is_preferred":false},{"frequency":66.67,"is_current":false,"is_preferred":false},{"frequency":59.94,"is_current":false,"is_preferred":false}]},{"resolution_width":720,"resolution_height":400,"is_high_resolution":false,"frequencies":[{"frequency":70.08,"is_current":false,"is_preferred":false}]}],"is_connected":true,"is_primary":false,"device_name":"VGA-1","rotation":"left","reflection":"normal","resolution_width":480,"resolution_height":640,"offset_width":1366,"offset_height":0,"dimension_width":408,"dimension_height":255},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"HDMI-1","rotation":"normal","reflection":"normal"},{"associated_modes":[],"is_connected":false,"is_primary":false,"device_name":"DP-1","rotation":"normal","reflection":"normal"}]} diff --git a/tests/fixtures/generic/xrandr_is_current_fix.out b/tests/fixtures/generic/xrandr_is_current_fix.out new file mode 100644 index 00000000..452a6462 --- /dev/null +++ b/tests/fixtures/generic/xrandr_is_current_fix.out @@ -0,0 +1,44 @@ +Screen 0: minimum 320 x 200, current 1846 x 768, maximum 8192 x 8192 +LVDS-1 connected primary 1366x768+0+0 (normal left inverted right x axis y axis) 344mm x 194mm + 1366x768 60.00*+ + 1280x720 60.00 59.99 59.86 59.74 + 1024x768 60.04 60.00 + 960x720 60.00 + 928x696 60.05 + 896x672 60.01 + 1024x576 59.95 59.96 59.90 59.82 + 960x600 59.93 60.00 + 960x540 59.96 59.99 59.63 59.82 + 800x600 60.00 60.32 56.25 + 840x525 60.01 59.88 + 864x486 59.92 59.57 + 700x525 59.98 + 800x450 59.95 59.82 + 640x512 60.02 + 700x450 59.96 59.88 + 640x480 60.00 59.94 + 720x405 59.51 58.99 + 684x384 59.88 59.85 + 640x400 59.88 59.98 + 640x360 59.86 59.83 59.84 59.32 + 512x384 60.00 + 512x288 60.00 59.92 + 480x270 59.63 59.82 + 400x300 60.32 56.34 + 432x243 59.92 59.57 + 320x240 60.05 + 360x202 59.51 59.13 + 320x180 59.84 59.32 +VGA-1 connected 480x640+1366+0 left (normal left inverted right x axis y axis) 408mm x 255mm + 1440x900 59.89 + 74.98 + 1280x1024 75.02 60.02 + 1280x960 60.00 + 1280x800 74.93 59.81 + 1152x864 75.00 + 1024x768 75.03 70.07 60.00 + 832x624 74.55 + 800x600 72.19 75.00 60.32 56.25 + 640x480 75.00* 72.81 66.67 59.94 + 720x400 70.08 +HDMI-1 disconnected (normal left inverted right x axis y axis) +DP-1 disconnected (normal left inverted right x axis y axis) diff --git a/tests/test_xrandr.py b/tests/test_xrandr.py index 1d1e927c..c4ff17f7 100644 --- a/tests/test_xrandr.py +++ b/tests/test_xrandr.py @@ -212,5 +212,15 @@ class XrandrTests(unittest.TestCase): self.assertEqual(actual, json_dict) + def test_is_current_fix(self): + with open("tests/fixtures/generic/xrandr_is_current_fix.out", "r") as f: + txt = f.read() + actual = parse(txt, quiet=True) + + with open("tests/fixtures/generic/xrandr_is_current_fix.json", "r") as f: + json_dict = json.loads(f.read()) + + self.assertEqual(actual, json_dict) + if __name__ == '__main__': unittest.main() From 98ced9616c3174f6fcfa65096bea685c1a020717 Mon Sep 17 00:00:00 2001 From: Jake Ob Date: Thu, 16 Feb 2023 12:33:29 +0200 Subject: [PATCH 71/81] Modify xrandr parser to extract model info from EDID (#364) This change will make the xrandr parser to work also with outputs from `xrandr --properties`, where the option `--properties` enables xrandr to return the EDID data of each output device. The parser is expected to remain backward compatible for regular xrandr outputs. The extra info that will be added to every device object is: ```json { "model_name": "str", "product_id": "str", "serial_number": "str" } ``` Because the EDID data come encoded in a hexadecimal string, we have to decode them by vendoring the pyedid (https://github.com/jojonas/pyedid) library. Fixes: #364 --- jc/parsers/pyedid/LICENSE | 18 +++ jc/parsers/pyedid/__init__.py | 0 jc/parsers/pyedid/edid.py | 171 +++++++++++++++++++++++ jc/parsers/pyedid/helpers/__init__.py | 0 jc/parsers/pyedid/helpers/edid_helper.py | 61 ++++++++ jc/parsers/pyedid/helpers/registry.py | 136 ++++++++++++++++++ jc/parsers/pyedid/main.py | 29 ++++ jc/parsers/xrandr.py | 135 +++++++++++++++++- 8 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 jc/parsers/pyedid/LICENSE create mode 100644 jc/parsers/pyedid/__init__.py create mode 100755 jc/parsers/pyedid/edid.py create mode 100644 jc/parsers/pyedid/helpers/__init__.py create mode 100644 jc/parsers/pyedid/helpers/edid_helper.py create mode 100644 jc/parsers/pyedid/helpers/registry.py create mode 100644 jc/parsers/pyedid/main.py diff --git a/jc/parsers/pyedid/LICENSE b/jc/parsers/pyedid/LICENSE new file mode 100644 index 00000000..f6616159 --- /dev/null +++ b/jc/parsers/pyedid/LICENSE @@ -0,0 +1,18 @@ +Copyright 2019-2020 Jonas Lieb, Davydov Denis + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/jc/parsers/pyedid/__init__.py b/jc/parsers/pyedid/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/jc/parsers/pyedid/edid.py b/jc/parsers/pyedid/edid.py new file mode 100755 index 00000000..fafe9c2c --- /dev/null +++ b/jc/parsers/pyedid/edid.py @@ -0,0 +1,171 @@ +""" +Edid module +""" + +import struct +from collections import namedtuple +from typing import ByteString + +__all__ = ["Edid"] + + +class Edid: + """Edid class + + Raises: + `ValueError`: if invalid edid data + """ + + _STRUCT_FORMAT = ( + "<" # little-endian + "8s" # constant header (8 bytes) + "H" # manufacturer id (2 bytes) + "H" # product id (2 bytes) + "I" # serial number (4 bytes) + "B" # manufactoring week (1 byte) + "B" # manufactoring year (1 byte) + "B" # edid version (1 byte) + "B" # edid revision (1 byte) + "B" # video input type (1 byte) + "B" # horizontal size in cm (1 byte) + "B" # vertical size in cm (1 byte) + "B" # display gamma (1 byte) + "B" # supported features (1 byte) + "10s" # color characteristics (10 bytes) + "H" # supported timings (2 bytes) + "B" # reserved timing (1 byte) + "16s" # EDID supported timings (16 bytes) + "18s" # detailed timing block 1 (18 bytes) + "18s" # detailed timing block 2 (18 bytes) + "18s" # detailed timing block 3 (18 bytes) + "18s" # detailed timing block 4 (18 bytes) + "B" # extension flag (1 byte) + "B" + ) # checksum (1 byte) + + _TIMINGS = { + 0: (1280, 1024, 75.0), + 1: (1024, 768, 75.0), + 2: (1024, 768, 70.0), + 3: (1024, 768, 60.0), + 4: (1024, 768, 87.0), + 5: (832, 624, 75.0), + 6: (800, 600, 75.0), + 7: (800, 600, 72.0), + 8: (800, 600, 60.0), + 9: (800, 600, 56.0), + 10: (640, 480, 75.0), + 11: (640, 480, 72.0), + 12: (640, 480, 67.0), + 13: (640, 480, 60.0), + 14: (720, 400, 88.0), + 15: (720, 400, 70.0), + } + + _ASPECT_RATIOS = { + 0b00: (16, 10), + 0b01: ( 4, 3), + 0b10: ( 5, 4), + 0b11: (16, 9), + } + + _RawEdid = namedtuple("RawEdid", + ("header", + "manu_id", + "prod_id", + "serial_no", + "manu_week", + "manu_year", + "edid_version", + "edid_revision", + "input_type", + "width", + "height", + "gamma", + "features", + "color", + "timings_supported", + "timings_reserved", + "timings_edid", + "timing_1", + "timing_2", + "timing_3", + "timing_4", + "extension", + "checksum") + ) + + def __init__(self, edid: ByteString): + self._parse_edid(edid) + + def _parse_edid(self, edid: ByteString): + """Convert edid byte string to edid object""" + if struct.calcsize(self._STRUCT_FORMAT) != 128: + raise ValueError("Wrong edid size.") + + if sum(map(int, edid)) % 256 != 0: + raise ValueError("Checksum mismatch.") + + unpacked = struct.unpack(self._STRUCT_FORMAT, edid) + raw_edid = self._RawEdid(*unpacked) + + if raw_edid.header != b'\x00\xff\xff\xff\xff\xff\xff\x00': + raise ValueError("Invalid header.") + + self.raw = edid + self.manufacturer_id = raw_edid.manu_id + self.product = raw_edid.prod_id + self.year = raw_edid.manu_year + 1990 + self.edid_version = "{:d}.{:d}".format(raw_edid.edid_version, raw_edid.edid_revision) + self.type = "digital" if (raw_edid.input_type & 0xFF) else "analog" + self.width = float(raw_edid.width) + self.height = float(raw_edid.height) + self.gamma = (raw_edid.gamma+100)/100 + self.dpms_standby = bool(raw_edid.features & 0xFF) + self.dpms_suspend = bool(raw_edid.features & 0x7F) + self.dpms_activeoff = bool(raw_edid.features & 0x3F) + + self.resolutions = [] + for i in range(16): + bit = raw_edid.timings_supported & (1 << i) + if bit: + self.resolutions.append(self._TIMINGS[i]) + + for i in range(8): + bytes_data = raw_edid.timings_edid[2*i:2*i+2] + if bytes_data == b'\x01\x01': + continue + byte1, byte2 = bytes_data + x_res = 8*(int(byte1)+31) + aspect_ratio = self._ASPECT_RATIOS[(byte2>>6) & 0b11] + y_res = int(x_res * aspect_ratio[1]/aspect_ratio[0]) + rate = (int(byte2) & 0b00111111) + 60.0 + self.resolutions.append((x_res, y_res, rate)) + + self.name = None + self.serial = None + + for timing_bytes in (raw_edid.timing_1, raw_edid.timing_2, raw_edid.timing_3, raw_edid.timing_4): + # "other" descriptor + if timing_bytes[0:2] == b'\x00\x00': + timing_type = timing_bytes[3] + if timing_type in (0xFF, 0xFE, 0xFC): + buffer = timing_bytes[5:] + buffer = buffer.partition(b"\x0a")[0] + text = buffer.decode("cp437") + if timing_type == 0xFF: + self.serial = text + elif timing_type == 0xFC: + self.name = text + + if not self.serial: + self.serial = raw_edid.serial_no + + def __repr__(self): + clsname = self.__class__.__name__ + attributes = [] + for name in dir(self): + if not name.startswith("_"): + value = getattr(self, name) + attributes.append("\t{}={}".format(name, value)) + return "{}(\n{}\n)".format(clsname, ", \n".join(attributes)) diff --git a/jc/parsers/pyedid/helpers/__init__.py b/jc/parsers/pyedid/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/jc/parsers/pyedid/helpers/edid_helper.py b/jc/parsers/pyedid/helpers/edid_helper.py new file mode 100644 index 00000000..b4165ca9 --- /dev/null +++ b/jc/parsers/pyedid/helpers/edid_helper.py @@ -0,0 +1,61 @@ +""" +EDID helper +""" + +from subprocess import CalledProcessError, check_output +from typing import ByteString, List + +__all__ = ["EdidHelper"] + + +class EdidHelper: + """Class for working with EDID data""" + + @staticmethod + def hex2bytes(hex_data: str) -> ByteString: + """Convert hex EDID string to bytes + + Args: + hex_data (str): hex edid string + + Returns: + ByteString: edid byte string + """ + # delete edid 1.3 additional block + if len(hex_data) > 256: + hex_data = hex_data[:256] + + numbers = [] + for i in range(0, len(hex_data), 2): + pair = hex_data[i : i + 2] + numbers.append(int(pair, 16)) + return bytes(numbers) + + @classmethod + def get_edids(cls) -> List[ByteString]: + """Get edids from xrandr + + Raises: + `RuntimeError`: if error with retrieving xrandr util data + + Returns: + List[ByteString]: list with edids + """ + try: + output = check_output(["xrandr", "--verbose"]) + except (CalledProcessError, FileNotFoundError) as err: + raise RuntimeError( + "Error retrieving xrandr util data: {}".format(err) + ) from None + + edids = [] + lines = output.splitlines() + for i, line in enumerate(lines): + line = line.decode().strip() + if line.startswith("EDID:"): + selection = lines[i + 1 : i + 9] + selection = list(s.decode().strip() for s in selection) + selection = "".join(selection) + bytes_section = cls.hex2bytes(selection) + edids.append(bytes_section) + return edids diff --git a/jc/parsers/pyedid/helpers/registry.py b/jc/parsers/pyedid/helpers/registry.py new file mode 100644 index 00000000..56ba05eb --- /dev/null +++ b/jc/parsers/pyedid/helpers/registry.py @@ -0,0 +1,136 @@ +""" +Module for working with PNP ID REGISTRY +""" + +import csv +import string +from html.parser import HTMLParser +from urllib import request + +__all__ = ["Registry"] + + +class WebPnpIdParser(HTMLParser): + """Parser pnp id from https://uefi.org/PNP_ID_List + + Examples: + p = WebPnpIdParser() + p.feed(html_data) + p.result + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._find_table = False + self._find_row = False + # first -- company name, second -- pnp id, third -- approved date + self._last_field = [] + # key -- pnp id, value -- tuple (company_name, approved_date) + self.result = {} + + def handle_starttag(self, tag, attrs): + if tag == "tbody": + self._find_table = True + elif self._find_table and tag == "tr": + self._find_row = True + + def handle_endtag(self, tag): + if tag == "tbody": + self._find_table = False + elif self._find_table and tag == "tr": + self._find_row = False + # add table row to result + self.result[self._last_field[1]] = ( + self._last_field[0], + self._last_field[-1], + ) + self._last_field.clear() + + def handle_data(self, data): + # skip processing until table is found + if not self._find_table: + return + + if self._find_row: + data = data.strip() + if data: + self._last_field.append(data) + + def error(self, message): + super().close() + + +class Registry(dict): + """Registry pnp id data dictionary + + key -- pnp_id + value -- company name + """ + + @classmethod + def from_web(cls, filter_by_id: str = None): + """Get registry from https://uefi.org/PNP_ID_List + + Args: + filter_by_id (str), optional: filter registry by id + + Raises: + + Returns: + + """ + url = "https://uefi.org/PNP_ID_List" + if filter_by_id: + url += "?search={}".format(filter_by_id) + + with request.urlopen(url) as req: + parse = WebPnpIdParser() + parse.feed(req.read().decode()) + + registry = cls() + for key, value in parse.result.items(): + # skip invalid search value + if filter_by_id and key != filter_by_id: + continue + registry[key] = value[0] + return registry + + @classmethod + def from_csv(cls, csv_path: str, filter_by_id: str = None): + """Get registry by csv local file + + Args: + csv_path (str): path to csv file + filter_by_id (str), optional: filter registry by id + + Raises: + + Returns: + + """ + registry = cls() + with open(csv_path, "r") as file: + reader = csv.reader(file) + for line in reader: + # filter + if filter_by_id and filter_by_id != line[0]: + continue + registry[line[0]] = line[1] + return registry + + def to_csv(self, csv_path: str): + """Dump registry to csv file""" + with open(csv_path, "w") as csv_file: + writer = csv.writer(csv_file) + writer.writerows(self.items()) + return self + + def get_company_from_id(self, pnp_id: str) -> str: + """Convert PNP id to company name""" + return self.get(pnp_id, "Unknown") + + def get_company_from_raw(self, raw: int) -> str: + """Convert raw edid value to company name""" + tmp = [(raw >> 10) & 31, (raw >> 5) & 31, raw & 31] + pnp_id = "".join(string.ascii_uppercase[n - 1] for n in tmp) + return self.get_company_from_id(pnp_id) diff --git a/jc/parsers/pyedid/main.py b/jc/parsers/pyedid/main.py new file mode 100644 index 00000000..4ad431f3 --- /dev/null +++ b/jc/parsers/pyedid/main.py @@ -0,0 +1,29 @@ +""" +Entrypoint +""" + +from pyedid.edid import Edid +from pyedid.helpers.edid_helper import EdidHelper +from pyedid.helpers.registry import Registry + + +def main(): + """Main func""" + + edid_csv_cache = "/tmp/pyedid-database.csv" + + try: + registry = Registry.from_csv(edid_csv_cache) + except FileNotFoundError: + print("Loading registry from web...") + registry = Registry.from_web() + print("Done!\n") + registry.to_csv(edid_csv_cache) + + for raw in EdidHelper.get_edids(): + edid = Edid(raw, registry) + print(edid) + + +if __name__ == "__main__": + main() diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index 283cf19b..0f1e364f 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -3,6 +3,7 @@ Usage (cli): $ xrandr | jc --xrandr + $ xrandr --properties | jc --xrandr or @@ -44,6 +45,9 @@ Schema: "is_connected": boolean, "is_primary": boolean, "device_name": string, + "model_name": string, + "product_id" string, + "serial_number": string, "resolution_width": integer, "resolution_height": integer, "offset_width": integer, @@ -135,10 +139,75 @@ Examples: ], "unassociated_devices": [] } + + $ xrandr --properties | jc --xrandr -p + { + "screens": [ + { + "screen_number": 0, + "minimum_width": 8, + "minimum_height": 8, + "current_width": 1920, + "current_height": 1080, + "maximum_width": 32767, + "maximum_height": 32767, + "associated_device": { + "associated_modes": [ + { + "resolution_width": 1920, + "resolution_height": 1080, + "is_high_resolution": false, + "frequencies": [ + { + "frequency": 60.03, + "is_current": true, + "is_preferred": true + }, + { + "frequency": 59.93, + "is_current": false, + "is_preferred": false + } + ] + }, + { + "resolution_width": 1680, + "resolution_height": 1050, + "is_high_resolution": false, + "frequencies": [ + { + "frequency": 59.88, + "is_current": false, + "is_preferred": false + } + ] + } + ], + "is_connected": true, + "is_primary": true, + "device_name": "eDP1", + "model_name": "ASUS VW193S", + "product_id": "54297", + "serial_number": "78L8021107", + "resolution_width": 1920, + "resolution_height": 1080, + "offset_width": 0, + "offset_height": 0, + "dimension_width": 310, + "dimension_height": 170, + "rotation": "normal", + "reflection": "normal" + } + } + ], + "unassociated_devices": [] + } """ import re from typing import Dict, List, Optional, Union import jc.utils +from jc.parsers.pyedid.edid import Edid +from jc.parsers.pyedid.helpers.edid_helper import EdidHelper class info: @@ -147,6 +216,7 @@ class info: description = "`xrandr` command parser" author = "Kevin Lyter" author_email = "lyter_git at sent.com" + details = 'Using parts of the pyedid library at https://github.com/jojonas/pyedid.' compatible = ["linux", "darwin", "cygwin", "aix", "freebsd"] magic_commands = ["xrandr"] tags = ['command'] @@ -174,10 +244,21 @@ try: "frequencies": List[Frequency], }, ) + Model = TypedDict( + "Model", + { + "name": str, + "product_id": str, + "serial_number": str, + }, + ) Device = TypedDict( "Device", { "device_name": str, + "model_name": str, + "product_id": str, + "serial_number": str, "is_connected": bool, "is_primary": bool, "resolution_width": int, @@ -294,15 +375,67 @@ def _parse_device(next_lines: List[str], quiet: bool = False) -> Optional[Device [f"{next_line} : {k} - {v} is not int-able"] ) + model: Optional[Model] = _parse_model(next_lines, quiet) + if model: + device["model_name"] = model["name"] + device["product_id"] = model["product_id"] + device["serial_number"] = model["serial_number"] + while next_lines: next_line = next_lines.pop() next_mode: Optional[Mode] = _parse_mode(next_line) if next_mode: device["associated_modes"].append(next_mode) else: + if re.match(_device_pattern, next_line): + next_lines.append(next_line) + break + return device + + +# EDID: +# 00ffffffffffff004ca3523100000000 +# 0014010380221378eac8959e57549226 +# 0f505400000001010101010101010101 +# 010101010101381d56d4500016303020 +# 250058c2100000190000000f00000000 +# 000000000025d9066a00000000fe0053 +# 414d53554e470a204ca34154000000fe +# 004c544e313536415432343430310018 +_edid_head_pattern = r"\s*EDID:\s*" +_edid_line_pattern = r"\s*(?P[0-9a-fA-F]{32})\s*" + + +def _parse_model(next_lines: List[str], quiet: bool = False) -> Optional[Model]: + if not next_lines: + return None + + next_line = next_lines.pop() + if not re.match(_edid_head_pattern, next_line): + next_lines.append(next_line) + return None + + edid_hex_value = "" + + while next_lines: + next_line = next_lines.pop() + result = re.match(_edid_line_pattern, next_line) + + if not result: next_lines.append(next_line) break - return device + + matches = result.groupdict() + edid_hex_value += matches["edid_line"] + + edid = Edid(EdidHelper.hex2bytes(edid_hex_value)) + + model: Model = { + "name": edid.name or "Generic", + "product_id": str(edid.product), + "serial_number": str(edid.serial), + } + return model # 1920x1080i 60.03*+ 59.93 From 6ea2d776ae4424d975256f43721dac6f6079875f Mon Sep 17 00:00:00 2001 From: Jake Ob Date: Fri, 17 Feb 2023 16:25:41 +0200 Subject: [PATCH 72/81] Add unit tests for xrandr --properties outputs --- tests/fixtures/generic/xrandr_properties.out | 110 +++++++++++++++++++ tests/test_xrandr.py | 92 ++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 tests/fixtures/generic/xrandr_properties.out diff --git a/tests/fixtures/generic/xrandr_properties.out b/tests/fixtures/generic/xrandr_properties.out new file mode 100644 index 00000000..e42d9e86 --- /dev/null +++ b/tests/fixtures/generic/xrandr_properties.out @@ -0,0 +1,110 @@ +Screen 0: minimum 320 x 200, current 2806 x 900, maximum 8192 x 8192 +LVDS-1 connected primary 1366x768+0+0 (normal left inverted right x axis y axis) 344mm x 194mm + EDID: + 00ffffffffffff004ca3523100000000 + 0014010380221378eac8959e57549226 + 0f505400000001010101010101010101 + 010101010101381d56d4500016303020 + 250058c2100000190000000f00000000 + 000000000025d9066a00000000fe0053 + 414d53554e470a204ca34154000000fe + 004c544e313536415432343430310018 + scaling mode: Full aspect + supported: Full, Center, Full aspect + link-status: Good + supported: Good, Bad + CONNECTOR_ID: 61 + supported: 61 + non-desktop: 0 + range: (0, 1) + 1366x768 60.00*+ + 1280x720 60.00 59.99 59.86 59.74 + 1024x768 60.04 60.00 + 960x720 60.00 + 928x696 60.05 + 896x672 60.01 + 1024x576 59.95 59.96 59.90 59.82 + 960x600 59.93 60.00 + 960x540 59.96 59.99 59.63 59.82 + 800x600 60.00 60.32 56.25 + 840x525 60.01 59.88 + 864x486 59.92 59.57 + 700x525 59.98 + 800x450 59.95 59.82 + 640x512 60.02 + 700x450 59.96 59.88 + 640x480 60.00 59.94 + 720x405 59.51 58.99 + 684x384 59.88 59.85 + 640x400 59.88 59.98 + 640x360 59.86 59.83 59.84 59.32 + 512x384 60.00 + 512x288 60.00 59.92 + 480x270 59.63 59.82 + 400x300 60.32 56.34 + 432x243 59.92 59.57 + 320x240 60.05 + 360x202 59.51 59.13 + 320x180 59.84 59.32 +VGA-1 connected 1440x900+1366+0 (normal left inverted right x axis y axis) 408mm x 255mm + EDID: + 00ffffffffffff000469d41901010101 + 2011010308291a78ea8585a6574a9c26 + 125054bfef80714f8100810f81408180 + 9500950f01019a29a0d0518422305098 + 360098ff1000001c000000fd00374b1e + 530f000a202020202020000000fc0041 + 535553205657313933530a20000000ff + 0037384c383032313130370a20200077 + link-status: Good + supported: Good, Bad + CONNECTOR_ID: 64 + supported: 64 + non-desktop: 0 + range: (0, 1) + 1440x900 59.89*+ 74.98 + 1280x1024 75.02 60.02 + 1280x960 60.00 + 1280x800 74.93 59.81 + 1152x864 75.00 + 1024x768 75.03 70.07 60.00 + 832x624 74.55 + 800x600 72.19 75.00 60.32 56.25 + 640x480 75.00 72.81 66.67 59.94 + 720x400 70.08 +HDMI-1 disconnected (normal left inverted right x axis y axis) + max bpc: 12 + range: (8, 12) + content type: No Data + supported: No Data, Graphics, Photo, Cinema, Game + Colorspace: Default + supported: Default, SMPTE_170M_YCC, BT709_YCC, XVYCC_601, XVYCC_709, SYCC_601, opYCC_601, opRGB, BT2020_CYCC, BT2020_RGB, BT2020_YCC, DCI-P3_RGB_D65, DCI-P3_RGB_Theater + aspect ratio: Automatic + supported: Automatic, 4:3, 16:9 + Broadcast RGB: Automatic + supported: Automatic, Full, Limited 16:235 + audio: auto + supported: force-dvi, off, auto, on + link-status: Good + supported: Good, Bad + CONNECTOR_ID: 68 + supported: 68 + non-desktop: 0 + range: (0, 1) +DP-1 disconnected (normal left inverted right x axis y axis) + Colorspace: Default + supported: Default, RGB_Wide_Gamut_Fixed_Point, RGB_Wide_Gamut_Floating_Point, opRGB, DCI-P3_RGB_D65, BT2020_RGB, BT601_YCC, BT709_YCC, XVYCC_601, XVYCC_709, SYCC_601, opYCC_601, BT2020_CYCC, BT2020_YCC + max bpc: 12 + range: (6, 12) + Broadcast RGB: Automatic + supported: Automatic, Full, Limited 16:235 + audio: auto + supported: force-dvi, off, auto, on + subconnector: Unknown + supported: Unknown, VGA, DVI-D, HDMI, DP, Wireless, Native + link-status: Good + supported: Good, Bad + CONNECTOR_ID: 76 + supported: 76 + non-desktop: 0 + range: (0, 1) diff --git a/tests/test_xrandr.py b/tests/test_xrandr.py index c4ff17f7..2866dc32 100644 --- a/tests/test_xrandr.py +++ b/tests/test_xrandr.py @@ -7,12 +7,16 @@ from jc.parsers.xrandr import ( _parse_screen, _parse_device, _parse_mode, + _parse_model, _device_pattern, _screen_pattern, _mode_pattern, _frequencies_pattern, + _edid_head_pattern, + _edid_line_pattern, parse, Mode, + Model, Device, Screen, ) @@ -59,6 +63,27 @@ class XrandrTests(unittest.TestCase): if match: rest = match.groupdict()["rest"] self.assertIsNotNone(re.match(_frequencies_pattern, rest)) + + edid_lines = [ + " EDID: ", + " 00ffffffffffff000469d41901010101 ", + " 2011010308291a78ea8585a6574a9c26 ", + " 125054bfef80714f8100810f81408180 ", + " 9500950f01019a29a0d0518422305098 ", + " 360098ff1000001c000000fd00374b1e ", + " 530f000a202020202020000000fc0041 ", + " 535553205657313933530a20000000ff ", + " 0037384c383032313130370a20200077 " + ] + + for i in range(len(edid_lines)): + line = edid_lines[i] + if i == 0: + match = re.match(_edid_head_pattern, line) + else: + match = re.match(_edid_line_pattern, line) + + self.assertIsNotNone(match) def test_screens(self): sample = "Screen 0: minimum 8 x 8, current 1920 x 1080, maximum 32767 x 32767" @@ -202,6 +227,16 @@ class XrandrTests(unittest.TestCase): 2, len(actual["screens"][0]["associated_device"]["associated_modes"]) ) + with open("tests/fixtures/generic/xrandr_properties.out", "r") as f: + txt = f.read() + actual = parse(txt, quiet=True) + + self.assertEqual(1, len(actual["screens"])) + self.assertEqual(3, len(actual["unassociated_devices"])) + self.assertEqual( + 29, len(actual["screens"][0]["associated_device"]["associated_modes"]) + ) + def test_infinite_loop_fix(self): with open("tests/fixtures/generic/xrandr_fix_spaces.out", "r") as f: txt = f.read() @@ -222,5 +257,62 @@ class XrandrTests(unittest.TestCase): self.assertEqual(actual, json_dict) + def test_model(self): + asus_edid = [ + " EDID: ", + " 00ffffffffffff000469d41901010101", + " 2011010308291a78ea8585a6574a9c26", + " 125054bfef80714f8100810f81408180", + " 9500950f01019a29a0d0518422305098", + " 360098ff1000001c000000fd00374b1e", + " 530f000a202020202020000000fc0041", + " 535553205657313933530a20000000ff", + " 0037384c383032313130370a20200077" + ] + asus_edid.reverse() + + expected = { + "name": "ASUS VW193S", + "product_id": "6612", + "serial_number": "78L8021107", + } + + actual: Optional[Model] = _parse_model(asus_edid) + self.assertIsNotNone(actual) + + if actual: + for k, v in expected.items(): + self.assertEqual(v, actual[k], f"mode regex failed on {k}") + + generic_edid = [ + " EDID: ", + " 00ffffffffffff004ca3523100000000", + " 0014010380221378eac8959e57549226", + " 0f505400000001010101010101010101", + " 010101010101381d56d4500016303020", + " 250058c2100000190000000f00000000", + " 000000000025d9066a00000000fe0053", + " 414d53554e470a204ca34154000000fe", + " 004c544e313536415432343430310018" + ] + generic_edid.reverse() + + expected = { + "name": "Generic", + "product_id": "12626", + "serial_number": "0", + } + + actual: Optional[Model] = _parse_model(generic_edid) + self.assertIsNotNone(actual) + + if actual: + for k, v in expected.items(): + self.assertEqual(v, actual[k], f"mode regex failed on {k}") + + empty_edid = [""] + actual: Optional[Model] = _parse_model(empty_edid) + self.assertIsNone(actual) + if __name__ == '__main__': unittest.main() From 64676fda2edf4b3e1b8ff283251287987ddef70b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 20 Feb 2023 11:56:20 -0800 Subject: [PATCH 73/81] fix for older linux style output --- CHANGELOG | 1 + jc/parsers/ifconfig.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d2061e35..ba128697 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,7 @@ jc changelog - Add `zpool iostat` command parser - Add `zpool status` command parser - Fix `acpi` command parser for "will never fully discharge" battery state +- Fix `ifconfig` command parser for older-style linux output - Fix `xrandr` command parser for proper `is_current` output - Fix `xrandr` command parser for infinite loop with some device configurations - Add `reflection` key to `xrandr` parser schema diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index f8bc9503..6192d699 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -219,7 +219,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '2.2' + version = '2.3' description = '`ifconfig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -425,18 +425,18 @@ def parse( # Linux syntax re_linux_interface = re.compile(r''' (?P[a-zA-Z0-9:._-]+)\s+ - Link encap:(?P\S+\s?\S+) + Link\sencap:(?P\S+\s?\S+) (\s+HWaddr\s+\b(?P[0-9A-Fa-f:?]+))? ''', re.IGNORECASE | re.VERBOSE ) re_linux_ipv4 = re.compile(r''' - inet addr:(?P
(?:[0-9]{1,3}\.){3}[0-9]{1,3})(\s+ + inet\saddr:(?P
(?:[0-9]{1,3}\.){3}[0-9]{1,3})(\s+ Bcast:(?P(?:[0-9]{1,3}\.){3}[0-9]{1,3}))?\s+ Mask:(?P(?:[0-9]{1,3}\.){3}[0-9]{1,3}) ''', re.IGNORECASE | re.VERBOSE ) re_linux_ipv6 = re.compile(r''' - inet6 addr:\s+(?P
\S+)/ + inet6\saddr:\s+(?P
\S+)/ (?P[0-9]+)\s+ Scope:(?PLink|Host) ''', re.IGNORECASE | re.VERBOSE @@ -448,7 +448,7 @@ def parse( ''', re.IGNORECASE | re.VERBOSE ) re_linux_rx = re.compile(r''' - RX packets:(?P[0-9]+)\s+ + RX\spackets:(?P[0-9]+)\s+ errors:(?P[0-9]+)\s+ dropped:(?P[0-9]+)\s+ overruns:(?P[0-9]+)\s+ @@ -456,7 +456,7 @@ def parse( ''', re.IGNORECASE | re.VERBOSE ) re_linux_tx = re.compile(r''' - TX packets:(?P[0-9]+)\s+ + TX\spackets:(?P[0-9]+)\s+ errors:(?P[0-9]+)\s+ dropped:(?P[0-9]+)\s+ overruns:(?P[0-9]+)\s+ @@ -464,8 +464,8 @@ def parse( ''', re.IGNORECASE | re.VERBOSE ) re_linux_bytes = re.compile(r''' - \W+RX bytes:(?P\d+)\s+\(.*\)\s+ - TX bytes:(?P\d+)\s+\(.*\) + \W+RX\sbytes:(?P\d+)\s+\(.*\)\s+ + TX\sbytes:(?P\d+)\s+\(.*\) ''', re.IGNORECASE | re.VERBOSE ) re_linux_tx_stats = re.compile(r''' From ccef69ac37cd0f16322dc854255d3193ed5fbf8e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 20 Feb 2023 12:04:45 -0800 Subject: [PATCH 74/81] add ubuntu16 tests --- tests/fixtures/ubuntu-16.04/ifconfig.json | 1 + tests/fixtures/ubuntu-16.04/ifconfig.out | 17 +++++++++++++++++ tests/test_ifconfig.py | 12 ++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 tests/fixtures/ubuntu-16.04/ifconfig.json create mode 100644 tests/fixtures/ubuntu-16.04/ifconfig.out diff --git a/tests/fixtures/ubuntu-16.04/ifconfig.json b/tests/fixtures/ubuntu-16.04/ifconfig.json new file mode 100644 index 00000000..3715125a --- /dev/null +++ b/tests/fixtures/ubuntu-16.04/ifconfig.json @@ -0,0 +1 @@ +[{"name":"ens33","flags":null,"state":["UP BROADCAST RUNNING MULTICAST "],"mtu":1500,"type":"Ethernet","mac_addr":"00:0c:29:c2:c8:63","ipv4_addr":"192.168.248.129","ipv4_mask":"255.255.255.0","ipv4_bcast":"192.168.248.255","ipv6_addr":"fe80::c1ca:3dee:39f7:5937","ipv6_mask":64,"ipv6_scope":"Link","ipv6_type":null,"metric":1,"rx_packets":36,"rx_errors":0,"rx_dropped":0,"rx_overruns":0,"rx_frame":0,"tx_packets":152,"tx_errors":0,"tx_dropped":0,"tx_overruns":0,"tx_carrier":0,"tx_collisions":0,"rx_bytes":5602,"tx_bytes":13935,"ipv4":[{"address":"192.168.248.129","broadcast":"192.168.248.255","mask":"255.255.255.0"}],"ipv6":[{"address":"fe80::c1ca:3dee:39f7:5937","mask":64,"scope":"Link"}]},{"name":"lo","flags":null,"state":["UP LOOPBACK RUNNING "],"mtu":65536,"type":"Local Loopback","mac_addr":null,"ipv4_addr":"127.0.0.1","ipv4_mask":"255.0.0.0","ipv4_bcast":null,"ipv6_addr":"::1","ipv6_mask":128,"ipv6_scope":"Host","ipv6_type":null,"metric":1,"rx_packets":208,"rx_errors":0,"rx_dropped":0,"rx_overruns":0,"rx_frame":0,"tx_packets":208,"tx_errors":0,"tx_dropped":0,"tx_overruns":0,"tx_carrier":0,"tx_collisions":0,"rx_bytes":17363,"tx_bytes":17363,"ipv4":[{"address":"127.0.0.1","broadcast":null,"mask":"255.0.0.0"}],"ipv6":[{"address":"::1","mask":128,"scope":"Host"}]}] diff --git a/tests/fixtures/ubuntu-16.04/ifconfig.out b/tests/fixtures/ubuntu-16.04/ifconfig.out new file mode 100644 index 00000000..b9214223 --- /dev/null +++ b/tests/fixtures/ubuntu-16.04/ifconfig.out @@ -0,0 +1,17 @@ +ens33 Link encap:Ethernet HWaddr 00:0c:29:c2:c8:63 + inet addr:192.168.248.129 Bcast:192.168.248.255 Mask:255.255.255.0 + inet6 addr: fe80::c1ca:3dee:39f7:5937/64 Scope:Link + UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 + RX packets:36 errors:0 dropped:0 overruns:0 frame:0 + TX packets:152 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1000 + RX bytes:5602 (5.6 KB) TX bytes:13935 (13.9 KB) + +lo Link encap:Local Loopback + inet addr:127.0.0.1 Mask:255.0.0.0 + inet6 addr: ::1/128 Scope:Host + UP LOOPBACK RUNNING MTU:65536 Metric:1 + RX packets:208 errors:0 dropped:0 overruns:0 frame:0 + TX packets:208 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1000 + RX bytes:17363 (17.3 KB) TX bytes:17363 (17.3 KB) diff --git a/tests/test_ifconfig.py b/tests/test_ifconfig.py index 3f6db7e9..58ea8f3f 100644 --- a/tests/test_ifconfig.py +++ b/tests/test_ifconfig.py @@ -12,6 +12,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ifconfig.out'), 'r', encoding='utf-8') as f: centos_7_7_ifconfig = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-16.04/ifconfig.out'), 'r', encoding='utf-8') as f: + ubuntu_16_4_ifconfig = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ifconfig.out'), 'r', encoding='utf-8') as f: ubuntu_18_4_ifconfig = f.read() @@ -43,6 +46,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ifconfig.json'), 'r', encoding='utf-8') as f: centos_7_7_ifconfig_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-16.04/ifconfig.json'), 'r', encoding='utf-8') as f: + ubuntu_16_4_ifconfig_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ifconfig.json'), 'r', encoding='utf-8') as f: ubuntu_18_4_ifconfig_json = json.loads(f.read()) @@ -82,6 +88,12 @@ class MyTests(unittest.TestCase): """ self.assertEqual(jc.parsers.ifconfig.parse(self.centos_7_7_ifconfig, quiet=True), self.centos_7_7_ifconfig_json) + def test_ifconfig_ubuntu_16_4(self): + """ + Test 'ifconfig' on Ubuntu 16.4 + """ + self.assertEqual(jc.parsers.ifconfig.parse(self.ubuntu_16_4_ifconfig, quiet=True), self.ubuntu_16_4_ifconfig_json) + def test_ifconfig_ubuntu_18_4(self): """ Test 'ifconfig' on Ubuntu 18.4 From 0658668eb0ff6aeeb860d8ea0d8923f2ae8e22e2 Mon Sep 17 00:00:00 2001 From: Jake Ob Date: Tue, 21 Feb 2023 14:43:10 +0200 Subject: [PATCH 75/81] Fix broken TypedDict compatibility with older versions of python By missing to declare the Model as a generic dictionary type we cause the script to fail because the TypedDict is not supported by older versions of python (e.g. 3.6, 3.7). --- jc/parsers/xrandr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index 0f1e364f..9ac4c44e 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -297,6 +297,7 @@ except ImportError: Device = Dict[str, Union[str, int, bool]] Frequency = Dict[str, Union[float, bool]] Mode = Dict[str, Union[int, bool, List[Frequency]]] + Model = Dict[str, str] Response = Dict[str, Union[Device, Mode, Screen]] From 15ac5a9004885dcc276440c2a9ccd5460ddd8d31 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 21 Feb 2023 15:44:55 -0800 Subject: [PATCH 76/81] fix for crontab files with only shortcut entries --- jc/parsers/crontab.py | 5 ++++- jc/parsers/crontab_u.py | 5 ++++- .../fixtures/generic/crontab-no-normal-entries.json | 1 + tests/fixtures/generic/crontab-no-normal-entries.out | 2 ++ .../generic/crontab-u-no-normal-entries.json | 1 + .../fixtures/generic/crontab-u-no-normal-entries.out | 2 ++ tests/test_crontab.py | 12 ++++++++++++ tests/test_crontab_u.py | 11 +++++++++++ 8 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/generic/crontab-no-normal-entries.json create mode 100644 tests/fixtures/generic/crontab-no-normal-entries.out create mode 100644 tests/fixtures/generic/crontab-u-no-normal-entries.json create mode 100644 tests/fixtures/generic/crontab-u-no-normal-entries.out diff --git a/jc/parsers/crontab.py b/jc/parsers/crontab.py index 07f7da53..031bd8d5 100644 --- a/jc/parsers/crontab.py +++ b/jc/parsers/crontab.py @@ -174,7 +174,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`crontab` command and file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -273,6 +273,9 @@ def parse(data, raw=False, quiet=False): raw_output['schedule'] = cron_list # Add shortcut entries back in + if 'schedule' not in raw_output: + raw_output['schedule'] = [] + for item in shortcut_list: raw_output['schedule'].append(item) diff --git a/jc/parsers/crontab_u.py b/jc/parsers/crontab_u.py index 7fb5a938..b847c6ad 100644 --- a/jc/parsers/crontab_u.py +++ b/jc/parsers/crontab_u.py @@ -171,7 +171,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.7' + version = '1.8' description = '`crontab` file parser with user support' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -271,6 +271,9 @@ def parse(data, raw=False, quiet=False): raw_output['schedule'] = cron_list # Add shortcut entries back in + if 'schedule' not in raw_output: + raw_output['schedule'] = [] + for item in shortcut_list: raw_output['schedule'].append(item) diff --git a/tests/fixtures/generic/crontab-no-normal-entries.json b/tests/fixtures/generic/crontab-no-normal-entries.json new file mode 100644 index 00000000..ec0e89fb --- /dev/null +++ b/tests/fixtures/generic/crontab-no-normal-entries.json @@ -0,0 +1 @@ +{"variables":[],"schedule":[{"occurrence":"daily","command":"/bin/sh do_the_thing"}]} diff --git a/tests/fixtures/generic/crontab-no-normal-entries.out b/tests/fixtures/generic/crontab-no-normal-entries.out new file mode 100644 index 00000000..ac72d233 --- /dev/null +++ b/tests/fixtures/generic/crontab-no-normal-entries.out @@ -0,0 +1,2 @@ +#this is a test for the jc module +@daily /bin/sh do_the_thing diff --git a/tests/fixtures/generic/crontab-u-no-normal-entries.json b/tests/fixtures/generic/crontab-u-no-normal-entries.json new file mode 100644 index 00000000..250f49df --- /dev/null +++ b/tests/fixtures/generic/crontab-u-no-normal-entries.json @@ -0,0 +1 @@ +{"variables":[],"schedule":[{"occurrence":"daily","user":"root","command":"/bin/sh do_the_thing"}]} diff --git a/tests/fixtures/generic/crontab-u-no-normal-entries.out b/tests/fixtures/generic/crontab-u-no-normal-entries.out new file mode 100644 index 00000000..559adfe3 --- /dev/null +++ b/tests/fixtures/generic/crontab-u-no-normal-entries.out @@ -0,0 +1,2 @@ +#this is a test for the jc module +@daily root /bin/sh do_the_thing diff --git a/tests/test_crontab.py b/tests/test_crontab.py index ccb5e145..81ae4b49 100644 --- a/tests/test_crontab.py +++ b/tests/test_crontab.py @@ -12,10 +12,16 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/crontab.out'), 'r', encoding='utf-8') as f: centos_7_7_crontab = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/crontab-no-normal-entries.out'), 'r', encoding='utf-8') as f: + generic_crontab_no_normal_entries = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/crontab.json'), 'r', encoding='utf-8') as f: centos_7_7_crontab_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/crontab-no-normal-entries.json'), 'r', encoding='utf-8') as f: + generic_crontab_no_normal_entries_json = json.loads(f.read()) + def test_crontab_nodata(self): """ @@ -29,6 +35,12 @@ class MyTests(unittest.TestCase): """ self.assertEqual(jc.parsers.crontab.parse(self.centos_7_7_crontab, quiet=True), self.centos_7_7_crontab_json) + def test_crontab_no_normal_entries(self): + """ + Test 'crontab' with no normal entries - only shortcuts + """ + self.assertEqual(jc.parsers.crontab.parse(self.generic_crontab_no_normal_entries, quiet=True), self.generic_crontab_no_normal_entries_json) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_crontab_u.py b/tests/test_crontab_u.py index 9e4c07b2..ad5429cd 100644 --- a/tests/test_crontab_u.py +++ b/tests/test_crontab_u.py @@ -18,6 +18,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/debian10/crontab-u.out'), 'r', encoding='utf-8') as f: debian10_crontab_u = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/crontab-u-no-normal-entries.out'), 'r', encoding='utf-8') as f: + generic_crontab_u_no_normal_entries = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/crontab-u.json'), 'r', encoding='utf-8') as f: ubuntu_18_4_crontab_u_json = json.loads(f.read()) @@ -28,6 +31,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/debian10/crontab-u.json'), 'r', encoding='utf-8') as f: debian10_crontab_u_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/crontab-u-no-normal-entries.json'), 'r', encoding='utf-8') as f: + generic_crontab_u_no_normal_entries_json = json.loads(f.read()) + def test_crontab_u_nodata(self): """ @@ -53,6 +59,11 @@ class MyTests(unittest.TestCase): """ self.assertEqual(jc.parsers.crontab_u.parse(self.debian10_crontab_u, quiet=True), self.debian10_crontab_u_json) + def test_crontab_u_no_normal_entries(self): + """ + Test 'crontab' with no normal entries - only shortcut entries (has a user field) + """ + self.assertEqual(jc.parsers.crontab_u.parse(self.generic_crontab_u_no_normal_entries, quiet=True), self.generic_crontab_u_no_normal_entries_json) if __name__ == '__main__': unittest.main() From a213ad9a85c118de6cf070fb87ae975e5c0c3fb3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 21 Feb 2023 15:50:09 -0800 Subject: [PATCH 77/81] doc update --- CHANGELOG | 2 ++ docs/parsers/crontab.md | 2 +- docs/parsers/crontab_u.md | 2 +- docs/parsers/ifconfig.md | 2 +- man/jc.1 | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ba128697..ecff3df1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,8 @@ jc changelog - Add `zpool iostat` command parser - Add `zpool status` command parser - Fix `acpi` command parser for "will never fully discharge" battery state +- Fix `crontab` and `crontab-u` command and file parsers for cases where only + shortcut schedule items exist - Fix `ifconfig` command parser for older-style linux output - Fix `xrandr` command parser for proper `is_current` output - Fix `xrandr` command parser for infinite loop with some device configurations diff --git a/docs/parsers/crontab.md b/docs/parsers/crontab.md index 35e40d80..7b26cf96 100644 --- a/docs/parsers/crontab.md +++ b/docs/parsers/crontab.md @@ -196,4 +196,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, aix, freebsd -Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/crontab_u.md b/docs/parsers/crontab_u.md index a0ba9ca9..74845c70 100644 --- a/docs/parsers/crontab_u.md +++ b/docs/parsers/crontab_u.md @@ -193,4 +193,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, aix, freebsd -Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/ifconfig.md b/docs/parsers/ifconfig.md index 7e806e82..6bd77ff8 100644 --- a/docs/parsers/ifconfig.md +++ b/docs/parsers/ifconfig.md @@ -240,4 +240,4 @@ Returns: ### Parser Information Compatibility: linux, aix, freebsd, darwin -Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 2.3 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/man/jc.1 b/man/jc.1 index 85f33dcd..66190af2 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-02-05 1.23.0 "JSON Convert" +.TH jc 1 2023-02-21 1.23.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings From 0c8c4a9c537bb95c2dbad99a21610a8efebd2f86 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 21 Feb 2023 17:18:24 -0800 Subject: [PATCH 78/81] formatting --- jc/parsers/xrandr.py | 57 ++++---------------------------------------- 1 file changed, 4 insertions(+), 53 deletions(-) diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index 9ac4c44e..363741f9 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -410,22 +410,22 @@ _edid_line_pattern = r"\s*(?P[0-9a-fA-F]{32})\s*" def _parse_model(next_lines: List[str], quiet: bool = False) -> Optional[Model]: if not next_lines: return None - + next_line = next_lines.pop() if not re.match(_edid_head_pattern, next_line): next_lines.append(next_line) return None - + edid_hex_value = "" while next_lines: next_line = next_lines.pop() result = re.match(_edid_line_pattern, next_line) - + if not result: next_lines.append(next_line) break - + matches = result.groupdict() edid_hex_value += matches["edid_line"] @@ -516,52 +516,3 @@ def parse(data: str, raw: bool =False, quiet: bool =False) -> Dict: return {} return result - -if __name__ == '__main__': - data = '''\ -Screen 0: minimum 320 x 200, current 2806 x 900, maximum 8192 x 8192 -LVDS-1 connected primary 1366x768+0+0 (normal left inverted right x axis y axis) 344mm x 194mm - 1366x768 60.00*+ - 1280x720 60.00 59.99 59.86 59.74 - 1024x768 60.04 60.00 - 960x720 60.00 - 928x696 60.05 - 896x672 60.01 - 1024x576 59.95 59.96 59.90 59.82 - 960x600 59.93 60.00 - 960x540 59.96 59.99 59.63 59.82 - 800x600 60.00 60.32 56.25 - 840x525 60.01 59.88 - 864x486 59.92 59.57 - 700x525 59.98 - 800x450 59.95 59.82 - 640x512 60.02 - 700x450 59.96 59.88 - 640x480 60.00 59.94 - 720x405 59.51 58.99 - 684x384 59.88 59.85 - 640x400 59.88 59.98 - 640x360 59.86 59.83 59.84 59.32 - 512x384 60.00 - 512x288 60.00 59.92 - 480x270 59.63 59.82 - 400x300 60.32 56.34 - 432x243 59.92 59.57 - 320x240 60.05 - 360x202 59.51 59.13 - 320x180 59.84 59.32 -VGA-1 connected 1440x900+1366+0 normal Y axis (normal left inverted right x axis y axis) 408mm x 255mm - 1440x900 59.89*+ 74.98 - 1280x1024 75.02 60.02 - 1280x960 60.00 - 1280x800 74.93 59.81 - 1152x864 75.00 - 1024x768 75.03 70.07 60.00 - 832x624 74.55 - 800x600 72.19 75.00 60.32 56.25 - 640x480 75.00 72.81 66.67 59.94 - 720x400 70.08 -HDMI-1 disconnected (normal left inverted right x axis y axis) -DP-1 disconnected (normal left inverted right x axis y axis)''' - - parse(data) \ No newline at end of file From 6e10965aedf8e16b45f2473dcddac4e7f9d4abde Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 21 Feb 2023 17:19:17 -0800 Subject: [PATCH 79/81] doc update --- docs/parsers/xrandr.md | 73 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/docs/parsers/xrandr.md b/docs/parsers/xrandr.md index 6359fedc..b3c355fc 100644 --- a/docs/parsers/xrandr.md +++ b/docs/parsers/xrandr.md @@ -8,6 +8,7 @@ jc - JSON Convert `xrandr` command output parser Usage (cli): $ xrandr | jc --xrandr + $ xrandr --properties | jc --xrandr or @@ -49,13 +50,17 @@ Schema: "is_connected": boolean, "is_primary": boolean, "device_name": string, + "model_name": string, + "product_id" string, + "serial_number": string, "resolution_width": integer, "resolution_height": integer, "offset_width": integer, "offset_height": integer, "dimension_width": integer, "dimension_height": integer, - "rotation": string + "rotation": string, + "reflection": string } ], "unassociated_devices": [ @@ -132,7 +137,71 @@ Examples: "offset_height": 0, "dimension_width": 310, "dimension_height": 170, - "rotation": "normal" + "rotation": "normal", + "reflection": "normal" + } + } + ], + "unassociated_devices": [] + } + + $ xrandr --properties | jc --xrandr -p + { + "screens": [ + { + "screen_number": 0, + "minimum_width": 8, + "minimum_height": 8, + "current_width": 1920, + "current_height": 1080, + "maximum_width": 32767, + "maximum_height": 32767, + "associated_device": { + "associated_modes": [ + { + "resolution_width": 1920, + "resolution_height": 1080, + "is_high_resolution": false, + "frequencies": [ + { + "frequency": 60.03, + "is_current": true, + "is_preferred": true + }, + { + "frequency": 59.93, + "is_current": false, + "is_preferred": false + } + ] + }, + { + "resolution_width": 1680, + "resolution_height": 1050, + "is_high_resolution": false, + "frequencies": [ + { + "frequency": 59.88, + "is_current": false, + "is_preferred": false + } + ] + } + ], + "is_connected": true, + "is_primary": true, + "device_name": "eDP1", + "model_name": "ASUS VW193S", + "product_id": "54297", + "serial_number": "78L8021107", + "resolution_width": 1920, + "resolution_height": 1080, + "offset_width": 0, + "offset_height": 0, + "dimension_width": 310, + "dimension_height": 170, + "rotation": "normal", + "reflection": "normal" } } ], From dbcff80907289e59fc8933d2819878b00b501764 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 27 Feb 2023 13:44:16 -0800 Subject: [PATCH 80/81] add zpool tests --- man/jc.1 | 2 +- tests/fixtures/generic/zpool-iostat-v.json | 1 + tests/fixtures/generic/zpool-iostat-v.out | 12 ++++ tests/fixtures/generic/zpool-iostat.json | 1 + tests/fixtures/generic/zpool-iostat.out | 7 ++ tests/fixtures/generic/zpool-status-v.json | 1 + tests/fixtures/generic/zpool-status-v.out | 24 +++++++ tests/fixtures/generic/zpool-status-v2.json | 1 + tests/fixtures/generic/zpool-status-v2.out | 17 +++++ tests/fixtures/generic/zpool-status-v3.json | 1 + tests/fixtures/generic/zpool-status-v3.out | 18 ++++++ tests/test_zpool_iostat.py | 59 +++++++++++++++++ tests/test_zpool_status.py | 71 +++++++++++++++++++++ 13 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/generic/zpool-iostat-v.json create mode 100644 tests/fixtures/generic/zpool-iostat-v.out create mode 100644 tests/fixtures/generic/zpool-iostat.json create mode 100644 tests/fixtures/generic/zpool-iostat.out create mode 100644 tests/fixtures/generic/zpool-status-v.json create mode 100644 tests/fixtures/generic/zpool-status-v.out create mode 100644 tests/fixtures/generic/zpool-status-v2.json create mode 100644 tests/fixtures/generic/zpool-status-v2.out create mode 100644 tests/fixtures/generic/zpool-status-v3.json create mode 100644 tests/fixtures/generic/zpool-status-v3.out create mode 100644 tests/test_zpool_iostat.py create mode 100644 tests/test_zpool_status.py diff --git a/man/jc.1 b/man/jc.1 index 66190af2..f2b1bf87 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-02-21 1.23.0 "JSON Convert" +.TH jc 1 2023-02-27 1.23.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings diff --git a/tests/fixtures/generic/zpool-iostat-v.json b/tests/fixtures/generic/zpool-iostat-v.json new file mode 100644 index 00000000..eebbc114 --- /dev/null +++ b/tests/fixtures/generic/zpool-iostat-v.json @@ -0,0 +1 @@ +[{"pool":"zhgstera6","cap_alloc":2.89,"cap_free":2.2,"ops_read":0,"ops_write":2,"bw_read":349.0,"bw_write":448.0,"cap_alloc_unit":"T","cap_free_unit":"T","bw_read_unit":"K","bw_write_unit":"K"},{"pool":"726060ALE614-K8JAPRGN:10","parent":"zhgstera6","cap_alloc":2.89,"cap_free":2.2,"ops_read":0,"ops_write":2,"bw_read":349.0,"bw_write":448.0,"cap_alloc_unit":"T","cap_free_unit":"T","bw_read_unit":"K","bw_write_unit":"K"},{"pool":"zint500","cap_alloc":230.0,"cap_free":24.0,"ops_read":0,"ops_write":9,"bw_read":112.0,"bw_write":318.0,"cap_alloc_unit":"G","cap_free_unit":"G","bw_read_unit":"K","bw_write_unit":"K"},{"pool":"ST3500418AS-5VMSTSSX:5","parent":"zint500","cap_alloc":230.0,"cap_free":24.0,"ops_read":0,"ops_write":9,"bw_read":112.0,"bw_write":318.0,"cap_alloc_unit":"G","cap_free_unit":"G","bw_read_unit":"K","bw_write_unit":"K"},{"pool":"zsam53","cap_alloc":211.0,"cap_free":24.5,"ops_read":0,"ops_write":0,"bw_read":14.7,"bw_write":74.4,"cap_alloc_unit":"G","cap_free_unit":"G","bw_read_unit":"K","bw_write_unit":"K"},{"pool":"Portable_SSD_T5-S49WNP0N120517B:8","parent":"zsam53","cap_alloc":211.0,"cap_free":24.5,"ops_read":0,"ops_write":0,"bw_read":14.7,"bw_write":74.4,"cap_alloc_unit":"G","cap_free_unit":"G","bw_read_unit":"K","bw_write_unit":"K"}] diff --git a/tests/fixtures/generic/zpool-iostat-v.out b/tests/fixtures/generic/zpool-iostat-v.out new file mode 100644 index 00000000..bdf5f561 --- /dev/null +++ b/tests/fixtures/generic/zpool-iostat-v.out @@ -0,0 +1,12 @@ + capacity operations bandwidth +pool alloc free read write read write +----------------------------------- ----- ----- ----- ----- ----- ----- +zhgstera6 2.89T 2.20T 0 2 349K 448K + 726060ALE614-K8JAPRGN:10 2.89T 2.20T 0 2 349K 448K +----------------------------------- ----- ----- ----- ----- ----- ----- +zint500 230G 24.0G 0 9 112K 318K + ST3500418AS-5VMSTSSX:5 230G 24.0G 0 9 112K 318K +----------------------------------- ----- ----- ----- ----- ----- ----- +zsam53 211G 24.5G 0 0 14.7K 74.4K + Portable_SSD_T5-S49WNP0N120517B:8 211G 24.5G 0 0 14.7K 74.4K +----------------------------------- ----- ----- ----- ----- ----- ----- diff --git a/tests/fixtures/generic/zpool-iostat.json b/tests/fixtures/generic/zpool-iostat.json new file mode 100644 index 00000000..d9b92004 --- /dev/null +++ b/tests/fixtures/generic/zpool-iostat.json @@ -0,0 +1 @@ +[{"pool":"zhgstera6","cap_alloc":2.89,"cap_free":2.2,"ops_read":0,"ops_write":2,"bw_read":349.0,"bw_write":448.0,"cap_alloc_unit":"T","cap_free_unit":"T","bw_read_unit":"K","bw_write_unit":"K"},{"pool":"zint500","cap_alloc":230.0,"cap_free":24.0,"ops_read":0,"ops_write":9,"bw_read":112.0,"bw_write":318.0,"cap_alloc_unit":"G","cap_free_unit":"G","bw_read_unit":"K","bw_write_unit":"K"},{"pool":"zsam53","cap_alloc":211.0,"cap_free":24.5,"ops_read":0,"ops_write":0,"bw_read":14.7,"bw_write":74.4,"cap_alloc_unit":"G","cap_free_unit":"G","bw_read_unit":"K","bw_write_unit":"K"}] diff --git a/tests/fixtures/generic/zpool-iostat.out b/tests/fixtures/generic/zpool-iostat.out new file mode 100644 index 00000000..8192fab1 --- /dev/null +++ b/tests/fixtures/generic/zpool-iostat.out @@ -0,0 +1,7 @@ + capacity operations bandwidth +pool alloc free read write read write +---------- ----- ----- ----- ----- ----- ----- +zhgstera6 2.89T 2.20T 0 2 349K 448K +zint500 230G 24.0G 0 9 112K 318K +zsam53 211G 24.5G 0 0 14.7K 74.4K +---------- ----- ----- ----- ----- ----- ----- diff --git a/tests/fixtures/generic/zpool-status-v.json b/tests/fixtures/generic/zpool-status-v.json new file mode 100644 index 00000000..bb257c8c --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v.json @@ -0,0 +1 @@ +[{"pool":"zhgstera6","state":"ONLINE","scan":"scrub canceled on Fri Aug 19 15:33:19 2022","config":[{"name":"zhgstera6","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"726060ALE614-K8JAPRGN:10","state":"ONLINE","read":0,"write":0,"checksum":0}],"errors":"No known data errors"},{"pool":"zint500","state":"ONLINE","scan":"scrub repaired 0 in 0 days 00:19:47 with 0 errors on Tue Aug 16 00:20:50 2022","config":[{"name":"zint500","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"ST3500418AS-5VMSTSSX:5","state":"ONLINE","read":0,"write":0,"checksum":0}],"errors":"No known data errors"},{"pool":"zsam53","state":"ONLINE","scan":"scrub repaired 0 in 0 days 01:25:43 with 0 errors on Mon Aug 15 01:26:46 2022","config":[{"name":"zsam53","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"Portable_SSD_T5-S49WNP0N120517B:8","state":"ONLINE","read":0,"write":0,"checksum":0}],"errors":"No known data errors"}] diff --git a/tests/fixtures/generic/zpool-status-v.out b/tests/fixtures/generic/zpool-status-v.out new file mode 100644 index 00000000..8d92c7ce --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v.out @@ -0,0 +1,24 @@ + pool: zhgstera6 + state: ONLINE + scan: scrub canceled on Fri Aug 19 15:33:19 2022 +config: + NAME STATE READ WRITE CKSUM + zhgstera6 ONLINE 0 0 0 + 726060ALE614-K8JAPRGN:10 ONLINE 0 0 0 +errors: No known data errors + pool: zint500 + state: ONLINE + scan: scrub repaired 0 in 0 days 00:19:47 with 0 errors on Tue Aug 16 00:20:50 2022 +config: + NAME STATE READ WRITE CKSUM + zint500 ONLINE 0 0 0 + ST3500418AS-5VMSTSSX:5 ONLINE 0 0 0 +errors: No known data errors + pool: zsam53 + state: ONLINE + scan: scrub repaired 0 in 0 days 01:25:43 with 0 errors on Mon Aug 15 01:26:46 2022 +config: + NAME STATE READ WRITE CKSUM + zsam53 ONLINE 0 0 0 + Portable_SSD_T5-S49WNP0N120517B:8 ONLINE 0 0 0 +errors: No known data errors diff --git a/tests/fixtures/generic/zpool-status-v2.json b/tests/fixtures/generic/zpool-status-v2.json new file mode 100644 index 00000000..c3ab6d9e --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v2.json @@ -0,0 +1 @@ +[{"pool":"tank","state":"DEGRADED","status":"One or more devices could not be opened. Sufficient replicas exist for\nthe pool to continue functioning in a degraded state.","action":"Attach the missing device and online it using 'zpool online'.","see":"http://www.sun.com/msg/ZFS-8000-2Q","scrub":"none requested","config":[{"name":"tank","state":"DEGRADED","read":0,"write":0,"checksum":0},{"name":"mirror-0","state":"DEGRADED","read":0,"write":0,"checksum":0},{"name":"c1t0d0","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"c1t1d0","state":"UNAVAIL","read":0,"write":0,"checksum":0,"errors":"cannot open"}],"errors":"No known data errors"}] diff --git a/tests/fixtures/generic/zpool-status-v2.out b/tests/fixtures/generic/zpool-status-v2.out new file mode 100644 index 00000000..a7b3a23e --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v2.out @@ -0,0 +1,17 @@ + pool: tank + state: DEGRADED +status: One or more devices could not be opened. Sufficient replicas exist for + the pool to continue functioning in a degraded state. +action: Attach the missing device and online it using 'zpool online'. + see: http://www.sun.com/msg/ZFS-8000-2Q + scrub: none requested +config: + + NAME STATE READ WRITE CKSUM + tank DEGRADED 0 0 0 + mirror-0 DEGRADED 0 0 0 + c1t0d0 ONLINE 0 0 0 + c1t1d0 UNAVAIL 0 0 0 cannot open + +errors: No known data errors + diff --git a/tests/fixtures/generic/zpool-status-v3.json b/tests/fixtures/generic/zpool-status-v3.json new file mode 100644 index 00000000..6b54de88 --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v3.json @@ -0,0 +1 @@ +[{"pool":"tank","state":"UNAVAIL","status":"One or more devices are faulted in response to IO failures.","action":"Make sure the affected devices are connected, then run 'zpool clear'.","see":"http://www.sun.com/msg/ZFS-8000-HC","scrub":"scrub completed after 0h0m with 0 errors on Tue Feb 2 13:08:42 2010","config":[{"name":"tank","state":"UNAVAIL","read":0,"write":0,"checksum":0,"errors":"insufficient replicas"},{"name":"c1t0d0","state":"ONLINE","read":0,"write":0,"checksum":0},{"name":"c1t1d0","state":"UNAVAIL","read":4,"write":1,"checksum":0,"errors":"cannot open"}],"errors":"Permanent errors have been detected in the following files:\n/tank/data/aaa\n/tank/data/bbb\n/tank/data/ccc"}] diff --git a/tests/fixtures/generic/zpool-status-v3.out b/tests/fixtures/generic/zpool-status-v3.out new file mode 100644 index 00000000..4314f1ee --- /dev/null +++ b/tests/fixtures/generic/zpool-status-v3.out @@ -0,0 +1,18 @@ + pool: tank + state: UNAVAIL +status: One or more devices are faulted in response to IO failures. +action: Make sure the affected devices are connected, then run 'zpool clear'. + see: http://www.sun.com/msg/ZFS-8000-HC + scrub: scrub completed after 0h0m with 0 errors on Tue Feb 2 13:08:42 2010 +config: + + NAME STATE READ WRITE CKSUM + tank UNAVAIL 0 0 0 insufficient replicas + c1t0d0 ONLINE 0 0 0 + c1t1d0 UNAVAIL 4 1 0 cannot open + +errors: Permanent errors have been detected in the following files: + +/tank/data/aaa +/tank/data/bbb +/tank/data/ccc diff --git a/tests/test_zpool_iostat.py b/tests/test_zpool_iostat.py new file mode 100644 index 00000000..64e961fd --- /dev/null +++ b/tests/test_zpool_iostat.py @@ -0,0 +1,59 @@ +import os +import unittest +import json +from typing import Dict +from jc.parsers.zpool_iostat import parse + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + f_in: Dict = {} + f_json: Dict = {} + + @classmethod + def setUpClass(cls): + fixtures = { + 'zpool_iostat': ( + 'fixtures/generic/zpool-iostat.out', + 'fixtures/generic/zpool-iostat.json'), + 'zpool_iostat_v': ( + 'fixtures/generic/zpool-iostat-v.out', + 'fixtures/generic/zpool-iostat-v.json') + } + + for file, filepaths in fixtures.items(): + with open(os.path.join(THIS_DIR, filepaths[0]), 'r', encoding='utf-8') as a, \ + open(os.path.join(THIS_DIR, filepaths[1]), 'r', encoding='utf-8') as b: + cls.f_in[file] = a.read() + cls.f_json[file] = json.loads(b.read()) + + + def test_zpool_iostat_nodata(self): + """ + Test 'zpool iostat' with no data + """ + self.assertEqual(parse('', quiet=True), []) + + + def test_zpool_iostat(self): + """ + Test 'zpool iostat' + """ + self.assertEqual( + parse(self.f_in['zpool_iostat'], quiet=True), + self.f_json['zpool_iostat'] + ) + + def test_zpool_iostat_v(self): + """ + Test 'zpool iostat -v' + """ + self.assertEqual( + parse(self.f_in['zpool_iostat_v'], quiet=True), + self.f_json['zpool_iostat_v'] + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_zpool_status.py b/tests/test_zpool_status.py new file mode 100644 index 00000000..e31c4c4c --- /dev/null +++ b/tests/test_zpool_status.py @@ -0,0 +1,71 @@ +import os +import unittest +import json +from typing import Dict +from jc.parsers.zpool_status import parse + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + f_in: Dict = {} + f_json: Dict = {} + + @classmethod + def setUpClass(cls): + fixtures = { + 'zpool_status': ( + 'fixtures/generic/zpool-status-v.out', + 'fixtures/generic/zpool-status-v.json'), + 'zpool_status2': ( + 'fixtures/generic/zpool-status-v2.out', + 'fixtures/generic/zpool-status-v2.json'), + 'zpool_status3': ( + 'fixtures/generic/zpool-status-v3.out', + 'fixtures/generic/zpool-status-v3.json') + } + + for file, filepaths in fixtures.items(): + with open(os.path.join(THIS_DIR, filepaths[0]), 'r', encoding='utf-8') as a, \ + open(os.path.join(THIS_DIR, filepaths[1]), 'r', encoding='utf-8') as b: + cls.f_in[file] = a.read() + cls.f_json[file] = json.loads(b.read()) + + + def test_zpool_status_nodata(self): + """ + Test 'zpool_status' with no data + """ + self.assertEqual(parse('', quiet=True), []) + + + def test_zpool_status_v(self): + """ + Test 'zpool status -v' + """ + self.assertEqual( + parse(self.f_in['zpool_status'], quiet=True), + self.f_json['zpool_status'] + ) + + def test_zpool_status_v_2(self): + """ + Test 'zpool status -v' #2 + """ + self.assertEqual( + parse(self.f_in['zpool_status2'], quiet=True), + self.f_json['zpool_status2'] + ) + + def test_zpool_status_v_3(self): + """ + Test 'zpool status -v' #3 + """ + self.assertEqual( + parse(self.f_in['zpool_status3'], quiet=True), + self.f_json['zpool_status3'] + ) + + +if __name__ == '__main__': + unittest.main() From 743e1ae90f5f5b5d38b4ad4778f9ac21ba348271 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 27 Feb 2023 15:01:00 -0800 Subject: [PATCH 81/81] doc update --- CHANGELOG | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index ecff3df1..201df851 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,6 @@ jc changelog -20230125 v1.23.0 +20230227 v1.23.0 - Add input slicing as a `jc` command-line option - Add `ssh` configuration file parser - Add `ver` Version string parser @@ -13,6 +13,7 @@ jc changelog - Fix `xrandr` command parser for proper `is_current` output - Fix `xrandr` command parser for infinite loop with some device configurations - Add `reflection` key to `xrandr` parser schema +- Add display model info from EDID to `xrandr` parser - Add `MPX-specific VMA` support for VM Flags in `/proc//smaps` parser 20230111 v1.22.5