diff --git a/README.md b/README.md index 1c5c3bef..f2646061 100644 --- a/README.md +++ b/README.md @@ -9,62 +9,56 @@ JSON CLI output utility `jc` JSONifies the output of many CLI tools and file-types for easier parsing in scripts. See the [**Parsers**](#parsers) section for supported commands and file-types. - -This allows further command-line processing of output with tools like `jq` by piping commands: ```bash -ls -l /usr/bin | jc --ls | jq '.[] | select(.size > 50000000)' +dig example.com | jc --dig ``` ```json -{ - "filename": "docker", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "root", - "size": 68677120, - "date": "Aug 14 19:41" -} +[{"id":38052,"opcode":"QUERY","status":"NOERROR","flags":["qr","rd","ra"],"query_num":1,"answer_num":1, +"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":[],"udp":4096}},"question": +{"name":"example.com.","class":"IN","type":"A"},"answer":[{"name":"example.com.","class":"IN","type":"A","ttl": +39049,"data":"93.184.216.34"}],"query_time":49,"server":"2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)","when": +"Fri Apr 16 16:09:00 PDT 2021","rcvd":56,"when_epoch":1618614540,"when_epoch_utc":null}] +``` +This allows further command-line processing of output with tools like `jq` by piping commands: +```bash +$ dig example.com | jc --dig | jq -r '.[].answer[].data' +93.184.216.34 ``` or using the alternative "magic" syntax: ```bash -jc ls -l /usr/bin | jq '.[] | select(.size > 50000000)' -``` -```json -{ - "filename": "docker", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "root", - "size": 68677120, - "date": "Aug 14 19:41" -} +$ jc dig example.com | jq -r '.[].answer[].data' +93.184.216.34 ``` The `jc` parsers can also be used as python modules. In this case the output will be a python dictionary, or list of dictionaries, instead of JSON: ```python ->>> import jc.parsers.ls +>>> import jc.parsers.dig >>> ->>> data='''-rwxr-xr-x 1 root wheel 23648 May 3 22:26 cat -... -rwxr-xr-x 1 root wheel 30016 May 3 22:26 chmod -... -rwxr-xr-x 1 root wheel 29024 May 3 22:26 cp -... -rwxr-xr-x 1 root wheel 375824 May 3 22:26 csh -... -rwxr-xr-x 1 root wheel 28608 May 3 22:26 date -... -rwxr-xr-x 1 root wheel 32000 May 3 22:26 dd -... -rwxr-xr-x 1 root wheel 23392 May 3 22:26 df -... -rwxr-xr-x 1 root wheel 18128 May 3 22:26 echo''' +>>> data = '''; <<>> DiG 9.10.6 <<>> example.com +... ;; global options: +cmd +... ;; Got answer: +... ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64612 +... ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 +... +... ;; OPT PSEUDOSECTION: +... ; EDNS: version: 0, flags:; udp: 4096 +... ;; QUESTION SECTION: +... ;example.com. IN A +... +... ;; ANSWER SECTION: +... example.com. 29658 IN A 93.184.216.34 +... +... ;; Query time: 52 msec +... ;; SERVER: 2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1) +... ;; WHEN: Fri Apr 16 16:13:00 PDT 2021 +... ;; MSG SIZE rcvd: 56''' >>> ->>> jc.parsers.ls.parse(data) -[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 23648, -'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', -'group': 'wheel', 'size': 30016, 'date': 'May 3 22:26'}, {'filename': 'cp', 'flags': '-rwxr-xr-x', -'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 29024, 'date': 'May 3 22:26'}, {'filename': 'csh', -'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 375824, 'date': 'May 3 -22:26'}, {'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', -'size': 28608, 'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': -'root', 'group': 'wheel', 'size': 32000, 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': -'-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 23392, 'date': 'May 3 22:26'}, -{'filename': 'echo', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 18128, -'date': 'May 3 22:26'}] +>>> jc.parsers.dig.parse(data) +[{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': +1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp': +4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.', +'class': 'IN', 'type': 'A', 'ttl': 29658, 'data': '93.184.216.34'}], 'query_time': 52, 'server': +'2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)', 'when': 'Fri Apr 16 16:13:00 PDT 2021', 'rcvd': 56, +'when_epoch': 1618614780, 'when_epoch_utc': None}] ``` Two representations of the data are possible. The default representation uses a strict schema per parser and converts known numbers to int/float JSON values. Certain known values of `None` are converted to JSON `null`, known boolean values are converted, and, in some cases, additional semantic context fields are added. diff --git a/docs/parsers/dig.md b/docs/parsers/dig.md index bf6ad803..92821c2d 100644 --- a/docs/parsers/dig.md +++ b/docs/parsers/dig.md @@ -24,64 +24,83 @@ Schema: [ { - "id": integer, - "opcode": string, - "status": string, + "id": integer, + "opcode": string, + "status": string, "flags": [ - string + string ], - "query_num": integer, - "answer_num": integer, - "authority_num": integer, - "additional_num": integer, + "query_num": integer, + "answer_num": integer, + "authority_num": integer, + "additional_num": integer, "axfr": [ { - "name": string, - "class": string, - "type": string, - "ttl": integer, - "data": string + "name": string, + "class": string, + "type": string, + "ttl": integer, + "data": string } ], + "opt_pseudosection": { + "edns": { + "version": integer, + "flags": [ + string + ], + "udp": integer + }, + "cookie": string + }, "question": { - "name": string, - "class": string, - "type": string + "name": string, + "class": string, + "type": string }, "answer": [ { - "name": string, - "class": string, - "type": string, - "ttl": integer, - "data": string + "name": string, + "class": string, + "type": string, + "ttl": integer, + "data": string + } + ], + "additional": [ + { + "name": string, + "class": string, + "type": string, + "ttl": integer, + "data": string } ], "authority": [ { - "name": string, - "class": string, - "type": string, - "ttl": integer, - "data": string + "name": string, + "class": string, + "type": string, + "ttl": integer, + "data": string } ], - "query_time": integer, # in msec - "server": string, - "when": string, - "when_epoch": integer, # naive timestamp if when field is parsable, else null - "when_epoch_utc": integer, # timezone aware timestamp availabe for UTC, else null - "rcvd": integer - "size": string + "query_time": integer, # in msec + "server": string, + "when": string, + "when_epoch": integer, # naive timestamp if when field is parsable, else null + "when_epoch_utc": integer, # timezone aware timestamp availabe for UTC, else null + "rcvd": integer + "size": string } ] Examples: - $ dig cnn.com www.cnn.com @205.251.194.64 | jc --dig -p + $ dig example.com | jc --dig -p [ { - "id": 52172, + "id": 2951, "opcode": "QUERY", "status": "NOERROR", "flags": [ @@ -90,113 +109,35 @@ Examples: "ra" ], "query_num": 1, - "answer_num": 4, + "answer_num": 1, "authority_num": 0, "additional_num": 1, + "opt_pseudosection": { + "edns": { + "version": 0, + "flags": [], + "udp": 4096 + } + }, "question": { - "name": "cnn.com.", + "name": "example.com.", "class": "IN", "type": "A" }, "answer": [ { - "name": "cnn.com.", + "name": "example.com.", "class": "IN", "type": "A", - "ttl": 27, - "data": "151.101.65.67" - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "A", - "ttl": 27, - "data": "151.101.129.67" - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "A", - "ttl": 27, - "data": "151.101.1.67" - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "A", - "ttl": 27, - "data": "151.101.193.67" + "ttl": 39302, + "data": "93.184.216.34" } ], - "query_time": 38, - "server": "2600", - "when": "Tue Mar 30 20:07:59 PDT 2021", - "rcvd": 100, - "when_epoch": 1617160079, - "when_epoch_utc": null - }, - { - "id": 36292, - "opcode": "QUERY", - "status": "NOERROR", - "flags": [ - "qr", - "aa", - "rd" - ], - "query_num": 1, - "answer_num": 1, - "authority_num": 4, - "additional_num": 1, - "question": { - "name": "www.cnn.com.", - "class": "IN", - "type": "A" - }, - "answer": [ - { - "name": "www.cnn.com.", - "class": "IN", - "type": "CNAME", - "ttl": 300, - "data": "turner-tls.map.fastly.net." - } - ], - "authority": [ - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": 3600, - "data": "ns-1086.awsdns-07.org." - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": 3600, - "data": "ns-1630.awsdns-11.co.uk." - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": 3600, - "data": "ns-47.awsdns-05.com." - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": 3600, - "data": "ns-576.awsdns-08.net." - } - ], - "query_time": 27, - "server": "205.251.194.64#53(205.251.194.64)", - "when": "Tue Mar 30 20:07:59 PDT 2021", - "rcvd": 212, - "when_epoch": 1617160079, + "query_time": 49, + "server": "2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)", + "when": "Fri Apr 16 16:05:10 PDT 2021", + "rcvd": 56, + "when_epoch": 1618614310, "when_epoch_utc": null } ] @@ -204,7 +145,7 @@ Examples: $ dig cnn.com www.cnn.com @205.251.194.64 | jc --dig -p -r [ { - "id": "23843", + "id": "46052", "opcode": "QUERY", "status": "NOERROR", "flags": [ @@ -213,110 +154,34 @@ Examples: "ra" ], "query_num": "1", - "answer_num": "4", + "answer_num": "1", "authority_num": "0", "additional_num": "1", + "opt_pseudosection": { + "edns": { + "version": "0", + "flags": [], + "udp": "4096" + } + }, "question": { - "name": "cnn.com.", + "name": "example.com.", "class": "IN", "type": "A" }, "answer": [ { - "name": "cnn.com.", + "name": "example.com.", "class": "IN", "type": "A", - "ttl": "30", - "data": "151.101.193.67" - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "A", - "ttl": "30", - "data": "151.101.1.67" - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "A", - "ttl": "30", - "data": "151.101.65.67" - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "A", - "ttl": "30", - "data": "151.101.129.67" + "ttl": "40426", + "data": "93.184.216.34" } ], - "query_time": "24 msec", - "server": "192.168.1.254#53(192.168.1.254)", - "when": "Tue Nov 12 07:16:19 PST 2019", - "rcvd": "100" - }, - { - "id": "8266", - "opcode": "QUERY", - "status": "NOERROR", - "flags": [ - "qr", - "aa", - "rd" - ], - "query_num": "1", - "answer_num": "1", - "authority_num": "4", - "additional_num": "1", - "question": { - "name": "www.cnn.com.", - "class": "IN", - "type": "A" - }, - "answer": [ - { - "name": "www.cnn.com.", - "class": "IN", - "type": "CNAME", - "ttl": "300", - "data": "turner-tls.map.fastly.net." - } - ], - "authority": [ - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": "3600", - "data": "ns-1086.awsdns-07.org." - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": "3600", - "data": "ns-1630.awsdns-11.co.uk." - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": "3600", - "data": "ns-47.awsdns-05.com." - }, - { - "name": "cnn.com.", - "class": "IN", - "type": "NS", - "ttl": "3600", - "data": "ns-576.awsdns-08.net." - } - ], - "query_time": "26 msec", - "server": "205.251.194.64#53(205.251.194.64)", - "when": "Tue Nov 12 07:16:19 PST 2019", - "rcvd": "212" + "query_time": "48 msec", + "server": "2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)", + "when": "Fri Apr 16 16:06:12 PDT 2021", + "rcvd": "56" } ] @@ -421,4 +286,4 @@ Returns: ## Parser Information Compatibility: linux, aix, freebsd, darwin -Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 2.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/man/jc.1.gz b/jc/man/jc.1.gz index 5767a72b..9fd9c3c2 100644 Binary files a/jc/man/jc.1.gz and b/jc/man/jc.1.gz differ diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py index 5cfd47a1..cea973b0 100644 --- a/jc/parsers/dig.py +++ b/jc/parsers/dig.py @@ -94,11 +94,93 @@ Schema: Examples: - $ dig cnn.com www.cnn.com @205.251.194.64 | jc --dig -p - + $ dig example.com | jc --dig -p + [ + { + "id": 2951, + "opcode": "QUERY", + "status": "NOERROR", + "flags": [ + "qr", + "rd", + "ra" + ], + "query_num": 1, + "answer_num": 1, + "authority_num": 0, + "additional_num": 1, + "opt_pseudosection": { + "edns": { + "version": 0, + "flags": [], + "udp": 4096 + } + }, + "question": { + "name": "example.com.", + "class": "IN", + "type": "A" + }, + "answer": [ + { + "name": "example.com.", + "class": "IN", + "type": "A", + "ttl": 39302, + "data": "93.184.216.34" + } + ], + "query_time": 49, + "server": "2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)", + "when": "Fri Apr 16 16:05:10 PDT 2021", + "rcvd": 56, + "when_epoch": 1618614310, + "when_epoch_utc": null + } + ] $ dig cnn.com www.cnn.com @205.251.194.64 | jc --dig -p -r - + [ + { + "id": "46052", + "opcode": "QUERY", + "status": "NOERROR", + "flags": [ + "qr", + "rd", + "ra" + ], + "query_num": "1", + "answer_num": "1", + "authority_num": "0", + "additional_num": "1", + "opt_pseudosection": { + "edns": { + "version": "0", + "flags": [], + "udp": "4096" + } + }, + "question": { + "name": "example.com.", + "class": "IN", + "type": "A" + }, + "answer": [ + { + "name": "example.com.", + "class": "IN", + "type": "A", + "ttl": "40426", + "data": "93.184.216.34" + } + ], + "query_time": "48 msec", + "server": "2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)", + "when": "Fri Apr 16 16:06:12 PDT 2021", + "rcvd": "56" + } + ] $ dig -x 1.1.1.1 | jc --dig -p [ diff --git a/man/jc.1.gz b/man/jc.1.gz index 5767a72b..9fd9c3c2 100644 Binary files a/man/jc.1.gz and b/man/jc.1.gz differ diff --git a/templates/readme_template b/templates/readme_template index 5642888d..a1bfa03a 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -9,62 +9,56 @@ JSON CLI output utility `jc` JSONifies the output of many CLI tools and file-types for easier parsing in scripts. See the [**Parsers**](#parsers) section for supported commands and file-types. - -This allows further command-line processing of output with tools like `jq` by piping commands: ```bash -ls -l /usr/bin | jc --ls | jq '.[] | select(.size > 50000000)' +dig example.com | jc --dig ``` ```json -{ - "filename": "docker", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "root", - "size": 68677120, - "date": "Aug 14 19:41" -} +[{"id":38052,"opcode":"QUERY","status":"NOERROR","flags":["qr","rd","ra"],"query_num":1,"answer_num":1, +"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":[],"udp":4096}},"question": +{"name":"example.com.","class":"IN","type":"A"},"answer":[{"name":"example.com.","class":"IN","type":"A","ttl": +39049,"data":"93.184.216.34"}],"query_time":49,"server":"2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)","when": +"Fri Apr 16 16:09:00 PDT 2021","rcvd":56,"when_epoch":1618614540,"when_epoch_utc":null}] +``` +This allows further command-line processing of output with tools like `jq` by piping commands: +```bash +$ dig example.com | jc --dig | jq -r '.[].answer[].data' +93.184.216.34 ``` or using the alternative "magic" syntax: ```bash -jc ls -l /usr/bin | jq '.[] | select(.size > 50000000)' -``` -```json -{ - "filename": "docker", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "root", - "size": 68677120, - "date": "Aug 14 19:41" -} +$ jc dig example.com | jq -r '.[].answer[].data' +93.184.216.34 ``` The `jc` parsers can also be used as python modules. In this case the output will be a python dictionary, or list of dictionaries, instead of JSON: ```python ->>> import jc.parsers.ls +>>> import jc.parsers.dig >>> ->>> data='''-rwxr-xr-x 1 root wheel 23648 May 3 22:26 cat -... -rwxr-xr-x 1 root wheel 30016 May 3 22:26 chmod -... -rwxr-xr-x 1 root wheel 29024 May 3 22:26 cp -... -rwxr-xr-x 1 root wheel 375824 May 3 22:26 csh -... -rwxr-xr-x 1 root wheel 28608 May 3 22:26 date -... -rwxr-xr-x 1 root wheel 32000 May 3 22:26 dd -... -rwxr-xr-x 1 root wheel 23392 May 3 22:26 df -... -rwxr-xr-x 1 root wheel 18128 May 3 22:26 echo''' +>>> data = '''; <<>> DiG 9.10.6 <<>> example.com +... ;; global options: +cmd +... ;; Got answer: +... ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64612 +... ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 +... +... ;; OPT PSEUDOSECTION: +... ; EDNS: version: 0, flags:; udp: 4096 +... ;; QUESTION SECTION: +... ;example.com. IN A +... +... ;; ANSWER SECTION: +... example.com. 29658 IN A 93.184.216.34 +... +... ;; Query time: 52 msec +... ;; SERVER: 2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1) +... ;; WHEN: Fri Apr 16 16:13:00 PDT 2021 +... ;; MSG SIZE rcvd: 56''' >>> ->>> jc.parsers.ls.parse(data) -[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 23648, -'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', -'group': 'wheel', 'size': 30016, 'date': 'May 3 22:26'}, {'filename': 'cp', 'flags': '-rwxr-xr-x', -'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 29024, 'date': 'May 3 22:26'}, {'filename': 'csh', -'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 375824, 'date': 'May 3 -22:26'}, {'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', -'size': 28608, 'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': -'root', 'group': 'wheel', 'size': 32000, 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': -'-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 23392, 'date': 'May 3 22:26'}, -{'filename': 'echo', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': 18128, -'date': 'May 3 22:26'}] +>>> jc.parsers.dig.parse(data) +[{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': +1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp': +4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.', +'class': 'IN', 'type': 'A', 'ttl': 29658, 'data': '93.184.216.34'}], 'query_time': 52, 'server': +'2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)', 'when': 'Fri Apr 16 16:13:00 PDT 2021', 'rcvd': 56, +'when_epoch': 1618614780, 'when_epoch_utc': None}] ``` Two representations of the data are possible. The default representation uses a strict schema per parser and converts known numbers to int/float JSON values. Certain known values of `None` are converted to JSON `null`, known boolean values are converted, and, in some cases, additional semantic context fields are added.