python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-08-06 22:32:54 +02:00
Code Issues Releases Activity

remove underscore in CEF_Version, doc update, timestamp optimization

Browse Source
This commit is contained in:
Kelly Brazil
2022-08-19 08:44:23 -07:00
parent 7bf6f1b870
commit 89574faef7
3 changed files with 21 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
docs/parsers/cef.md
View File

@ -5,10 +5,7 @@
jc - JSON Convert CEF string parser jc - JSON Convert CEF string parser
This parser conforms to the Microfocus Arcsight CEF specification. If you This parser conforms to the Microfocus Arcsight CEF specification.
require special handling for your CEF input, you can copy this parser code
to the `jc` pluggin directory for your system and modify it to suit your
needs.
This parser will accept a single CEF string or multiple CEF string lines. This parser will accept a single CEF string or multiple CEF string lines.
Any text before "CEF" will be ignored. Syslog and CEF escaped characters Any text before "CEF" will be ignored. Syslog and CEF escaped characters
@ -19,8 +16,8 @@ Extended fields, as defined in the CEF specification, are relabeled
and the values are converted to their respective types. Extra naive and and the values are converted to their respective types. Extra naive and
UTC epoch timestamps are added where appropriate per the CEF specification. UTC epoch timestamps are added where appropriate per the CEF specification.
To preserve escaping, original keynames, and value types use the `--raw` or To preserve escaping and original keynames and to prevent type conversions
`raw=True` option in the `parse()` function. use the `--raw` or `raw=True` option in the `parse()` function.
Usage (cli): Usage (cli):
@ -47,7 +44,7 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-
"agentSeverity": string/integer, "agentSeverity": string/integer,
"agentSeverityString": string, "agentSeverityString": string,
"agentSeverityNum": integer, "agentSeverityNum": integer,
"CEF_Version": integer, "CEFVersion": integer,
<extended fields> string/integer/float, # [0] <extended fields> string/integer/float, # [0]
<extended fields>"_epoch": integer, # [1] <extended fields>"_epoch": integer, # [1]
<extended fields>"_epoch_utc": integer, # [2] <extended fields>"_epoch_utc": integer, # [2]
@ -74,7 +71,7 @@ Examples:
"deviceEventClassId": "4000000", "deviceEventClassId": "4000000",
"name": "Eicar_test_file", "name": "Eicar_test_file",
"agentSeverity": 6, "agentSeverity": 6,
"CEF_Version": 0, "CEFVersion": 0,
"dvchost": "hostname", "dvchost": "hostname",
"string": "hello \"world\"!", "string": "hello \"world\"!",
"start": "Nov 08 2020 12:30:00.111 UTC", "start": "Nov 08 2020 12:30:00.111 UTC",
@ -100,7 +97,7 @@ Examples:
"deviceEventClassId": "4000000", "deviceEventClassId": "4000000",
"name": "Eicar_test_file", "name": "Eicar_test_file",
"agentSeverity": "6", "agentSeverity": "6",
"CEF_Version": "0", "CEFVersion": "0",
"cn1": "1", "cn1": "1",
"cn1Label": "Host ID", "cn1Label": "Host ID",
"dvchost": "hostname", "dvchost": "hostname",

29
jc/parsers/cef.py
View File

@ -1,9 +1,6 @@
"""jc - JSON Convert CEF string parser """jc - JSON Convert CEF string parser
This parser conforms to the Microfocus Arcsight CEF specification. If you This parser conforms to the Microfocus Arcsight CEF specification.
require special handling for your CEF input, you can copy this parser code
to the `jc` pluggin directory for your system and modify it to suit your
needs.
This parser will accept a single CEF string or multiple CEF string lines. This parser will accept a single CEF string or multiple CEF string lines.
Any text before "CEF" will be ignored. Syslog and CEF escaped characters Any text before "CEF" will be ignored. Syslog and CEF escaped characters
@ -14,8 +11,8 @@ Extended fields, as defined in the CEF specification, are relabeled
and the values are converted to their respective types. Extra naive and and the values are converted to their respective types. Extra naive and
UTC epoch timestamps are added where appropriate per the CEF specification. UTC epoch timestamps are added where appropriate per the CEF specification.
To preserve escaping, original keynames, and value types use the `--raw` or To preserve escaping and original keynames and to prevent type conversions
`raw=True` option in the `parse()` function. use the `--raw` or `raw=True` option in the `parse()` function.
Usage (cli): Usage (cli):
@ -42,7 +39,7 @@ See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-
"agentSeverity": string/integer, "agentSeverity": string/integer,
"agentSeverityString": string, "agentSeverityString": string,
"agentSeverityNum": integer, "agentSeverityNum": integer,
"CEF_Version": integer, "CEFVersion": integer,
<extended fields> string/integer/float, # [0] <extended fields> string/integer/float, # [0]
<extended fields>"_epoch": integer, # [1] <extended fields>"_epoch": integer, # [1]
<extended fields>"_epoch_utc": integer, # [2] <extended fields>"_epoch_utc": integer, # [2]
@ -69,7 +66,7 @@ Examples:
"deviceEventClassId": "4000000", "deviceEventClassId": "4000000",
"name": "Eicar_test_file", "name": "Eicar_test_file",
"agentSeverity": 6, "agentSeverity": 6,
"CEF_Version": 0, "CEFVersion": 0,
"dvchost": "hostname", "dvchost": "hostname",
"string": "hello \"world\"!", "string": "hello \"world\"!",
"start": "Nov 08 2020 12:30:00.111 UTC", "start": "Nov 08 2020 12:30:00.111 UTC",
@ -95,7 +92,7 @@ Examples:
"deviceEventClassId": "4000000", "deviceEventClassId": "4000000",
"name": "Eicar_test_file", "name": "Eicar_test_file",
"agentSeverity": "6", "agentSeverity": "6",
"CEF_Version": "0", "CEFVersion": "0",
"cn1": "1", "cn1": "1",
"cn1Label": "Host ID", "cn1Label": "Host ID",
"dvchost": "hostname", "dvchost": "hostname",
@ -203,7 +200,7 @@ def _pycef_parse(str_input):
if cef_start == -1: if cef_start == -1:
raise ParseError('Invalid CEF string.') raise ParseError('Invalid CEF string.')
(cef, version) = spl[0][cef_start:].split(':') (cef, version) = spl[0][cef_start:].split(':')
values["CEF_Version"] = version values["CEFVersion"] = version
# The ugly, gnarly regex here finds a single key=value pair, # The ugly, gnarly regex here finds a single key=value pair,
# taking into account multiple whitespaces, escaped '=' and '|' # taking into account multiple whitespaces, escaped '=' and '|'
@ -249,7 +246,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
r'\r': '\r' r'\r': '\r'
} }
int_list = {'CEF_Version'} int_list = {'CEFVersion'}
severity_map = { severity_map = {
None: 'Unknown', None: 'Unknown',
@ -304,7 +301,8 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
pass pass
if key in extended_dt: if key in extended_dt:
dt = jc.utils.timestamp(item[key]) formats = (1400, 1410, 1420, 1430)
dt = jc.utils.timestamp(item[key], formats)
item[key + '_epoch'] = dt.naive item[key + '_epoch'] = dt.naive
item[key + '_epoch_utc'] = dt.utc item[key + '_epoch_utc'] = dt.utc
@ -315,19 +313,20 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
if key.endswith('Label'): if key.endswith('Label'):
customlabel = key[:-5] customlabel = key[:-5]
for customfield in custom_fields: for customfield in custom_fields:
new_name = item[key]
# check for normal custom fields # check for normal custom fields
if customfield == customlabel: if customfield == customlabel:
item[item[key]] = item[customfield] item[new_name] = item[customfield]
cleanup_list.append(customfield) cleanup_list.append(customfield)
cleanup_list.append(key) cleanup_list.append(key)
# check for datetime objects # check for datetime objects
if customfield == customlabel + '_epoch': if customfield == customlabel + '_epoch':
item[item[key] + '_epoch'] = item[customfield] item[new_name + '_epoch'] = item[customfield]
cleanup_list.append(customfield) cleanup_list.append(customfield)
if customfield == customlabel + '_epoch_utc': if customfield == customlabel + '_epoch_utc':
item[item[key] + '_epoch_utc'] = item[customfield] item[new_name + '_epoch_utc'] = item[customfield]
cleanup_list.append(customfield) cleanup_list.append(customfield)
# cleanup extra custom fields # cleanup extra custom fields

2
man/jc.1
View File

@ -1,4 +1,4 @@
.TH jc 1 2022-08-18 1.21.0 "JSON Convert" .TH jc 1 2022-08-19 1.21.0 "JSON Convert"
.SH NAME .SH NAME
\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types
.SH SYNOPSIS .SH SYNOPSIS