python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-23 00:29:59 +02:00
Code Issues Releases Activity

doc update

Browse Source
This commit is contained in:
Kelly Brazil
2022-08-20 12:30:52 -07:00
parent 944ae9f8d9
commit b6cd123719
6 changed files with 49 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
docs/parsers/cef.md
View File

@ -16,6 +16,9 @@ Extended fields, as defined in the CEF specification, are relabeled
and the values are converted to their respective types. Extra naive and
UTC epoch timestamps are added where appropriate per the CEF specification.
A warning message to `STDERR` will be printed if an unparsable line is found
unless `--quiet` or `quiet=True` is used.
To preserve escaping and original keynames and to prevent type conversions
use the `--raw` CLI option or `raw=True` param in the `parse()` function.
@ -40,15 +43,17 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-
"deviceProduct": string,
"deviceVersion": string,
"deviceEventClassId": string,
"deviceEventClassIdNum": integer/null,
"name": string,
"agentSeverity": string/integer,
"agentSeverityString": string,
"agentSeverityNum": integer,
"agentSeverityNum": integer/null,
"CEFVersion": integer,
<extended fields> string/integer/float, # [0]
<extended fields>"_epoch": integer, # [1]
<extended fields>"_epoch_utc": integer, # [2]
<custom fields> string
<extended fields>"_epoch": integer/null, # [1]
<extended fields>"_epoch_utc": integer/null, # [2]
<custom fields> string,
"unparsable": string # [3]
}
]
@ -59,6 +64,8 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-
[2] Timezone-aware calculated epoch timestamp. (UTC only) This value
will be null if a UTC timezone cannot be extracted from the original
timestamp string value.
[3] this field exists if the CEF line is not parsable. The value
is the original syslog line.
Examples:

2
docs/parsers/syslog_bsd_s.md
View File

@ -49,10 +49,12 @@ Examples:
$ cat syslog.txt | jc --syslog-bsd-s -p
{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","t...}
{"priority":34,"date":"Oct 11 22:14:16","hostname":"mymachine","t...}
...
$ cat syslog.txt | jc --syslog-bsd-s -p -r
{"priority":"34","date":"Oct 11 22:14:15","hostname":"mymachine","...}
{"priority":"34","date":"Oct 11 22:14:16","hostname":"mymachine","...}
...
<a id="jc.parsers.syslog_bsd_s.parse"></a>

2
docs/parsers/syslog_s.md
View File

@ -72,10 +72,12 @@ Examples:
$ cat syslog.txt | jc --syslog-s -p
{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-...}
{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:16.000003-...}
...
$ cat syslog.txt | jc --syslog-s -p -r
{"priority":"165","version":"1","timestamp":"2003-08-24T05:14:15.000...}
{"priority":"165","version":"1","timestamp":"2003-08-24T05:15:15.000...}
...
<a id="jc.parsers.syslog_s.parse"></a>

40
jc/parsers/cef.py
View File

@ -11,6 +11,9 @@ Extended fields, as defined in the CEF specification, are relabeled
and the values are converted to their respective types. Extra naive and
UTC epoch timestamps are added where appropriate per the CEF specification.
A warning message to `STDERR` will be printed if an unparsable line is found
unless `--quiet` or `quiet=True` is used.
To preserve escaping and original keynames and to prevent type conversions
use the `--raw` CLI option or `raw=True` param in the `parse()` function.
@ -35,15 +38,17 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-
"deviceProduct": string,
"deviceVersion": string,
"deviceEventClassId": string,
"deviceEventClassIdNum": integer/null,
"name": string,
"agentSeverity": string/integer,
"agentSeverityString": string,
"agentSeverityNum": integer,
"agentSeverityNum": integer/null,
"CEFVersion": integer,
<extended fields> string/integer/float, # [0]
<extended fields>"_epoch": integer, # [1]
<extended fields>"_epoch_utc": integer, # [2]
<custom fields> string
<extended fields>"_epoch": integer/null, # [1]
<extended fields>"_epoch_utc": integer/null, # [2]
<custom fields> string,
"unparsable": string # [3]
}
]
@ -54,6 +59,8 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-
[2] Timezone-aware calculated epoch timestamp. (UTC only) This value
will be null if a UTC timezone cannot be extracted from the original
timestamp string value.
[3] this field exists if the CEF line is not parsable. The value
is the original syslog line.
Examples:
@ -357,15 +364,21 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
if key in int_list:
item[key] = jc.utils.convert_to_int(item[key])
# set SeverityString and SeverityNum:
# set agentSeverityString and agentSeverityNum:
if 'agentSeverity' in item:
if isinstance(item['agentSeverity'], str) and item['agentSeverity'].lower() in severity_set:
if item['agentSeverity'].lower() in severity_set:
item['agentSeverityString'] = item['agentSeverity']
item['agentSeverityNum'] = None
else:
item['agentSeverity'] = int(item['agentSeverity'])
item['agentSeverityString'] = severity_map[item['agentSeverity']]
item['agentSeverityNum'] = item['agentSeverity']
try:
item['agentSeverityString'] = severity_map[int(item['agentSeverity'])]
item['agentSeverityNum'] = int(item['agentSeverity'])
except Exception:
pass
# set deviceEventClassIdNum:
if 'deviceEventClassId' in item:
item['deviceEventClassIdNum'] = jc.utils.convert_to_int(item['deviceEventClassId'])
return proc_data
@ -395,6 +408,13 @@ def parse(
if jc.utils.has_data(data):
for line in filter(None, data.splitlines()):
raw_output.append(_pycef_parse(line))
try:
raw_output.append(_pycef_parse(line))
except Exception:
if not quiet:
jc.utils.warning_message(
[f'Unparsable CEF line found: {line}']
)
raw_output.append({"unparsable": line})
return raw_output if raw else _process(raw_output)

2
jc/parsers/syslog_bsd_s.py
View File

@ -44,10 +44,12 @@ Examples:
$ cat syslog.txt | jc --syslog-bsd-s -p
{"priority":34,"date":"Oct 11 22:14:15","hostname":"mymachine","t...}
{"priority":34,"date":"Oct 11 22:14:16","hostname":"mymachine","t...}
...
$ cat syslog.txt | jc --syslog-bsd-s -p -r
{"priority":"34","date":"Oct 11 22:14:15","hostname":"mymachine","...}
{"priority":"34","date":"Oct 11 22:14:16","hostname":"mymachine","...}
...
"""
from typing import Dict, Iterable, Union

2
jc/parsers/syslog_s.py
View File

@ -67,10 +67,12 @@ Examples:
$ cat syslog.txt | jc --syslog-s -p
{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:15.000003-...}
{"priority":165,"version":1,"timestamp":"2003-08-24T05:14:16.000003-...}
...
$ cat syslog.txt | jc --syslog-s -p -r
{"priority":"165","version":"1","timestamp":"2003-08-24T05:14:15.000...}
{"priority":"165","version":"1","timestamp":"2003-08-24T05:15:15.000...}
...
"""
from typing import List, Dict, Iterable, Union, Optional