python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-08-06 22:32:54 +02:00
Code Issues Releases Activity

move conversions into process(). initial timestamp fields working

Browse Source
This commit is contained in:
Kelly Brazil
2022-08-17 17:46:24 -07:00
parent 42b24d71e9
commit 5b597b6583
1 changed files with 54 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
jc/parsers/cef.py
View File

@ -23,6 +23,8 @@ Schema:
See: https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/Content/CEF/Chapter%201%20What%20is%20CEF.htm
> Note: Special characters in key names will be converted to underscores.
[
{
"deviceVendor": string,
@ -155,46 +157,6 @@ def _pycef_parse(str_input):
# Split the tuples and put them into the dictionary
values[i[0]] = i[1]
# set defined types for extended fields
# see https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/#CEF/Chapter%202%20ArcSight%20Extension.htm
extended_ints = {
'spid', 'customerKey', 'deviceTranslatedZoneKey', 'oldFileSize',
'destination TranslatedPort', 'cn3', 'source TranslatedPort', 'in', 'fsize', 'slat',
'dpid', 'cnt', 'agentZoneKey', 'out', 'type', 'eventId', 'dlong', 'cn2',
'deviceDirection', 'spt', 'agentTranslatedZoneKey', 'sTranslatedZoneKey', 'cn1',
'slong', 'dZoneKey', 'deviceZoneKey', 'dvcpid', 'dpt', 'dTranslatedZoneKey', 'dlat',
'sZoneKey'
}
extended_floats = {
'cfp1', 'cfp2', 'cfp3', 'cfp4'
}
for k, v in values.items():
if k in extended_ints:
try:
values[k] = int(v)
except Exception:
pass
if k in extended_floats:
try:
values[k] = float(v)
except Exception:
pass
# Process custom field labels
for key in list(values.keys()):
# If the key string ends with Label, replace it in the appropriate
# custom field
if key[-5:] == "Label":
customlabel = key[:-5]
# Find the corresponding customfield and replace with the label
for customfield in list(values.keys()):
if customfield == customlabel:
values[values[key]] = values[customfield]
del values[customfield]
del values[key]
else:
raise ParseError('Could not parse record. Is it valid CEF format?')
@ -249,7 +211,59 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
severity_set = {'unknown', 'low', 'medium', 'high', 'very-high'}
# set defined types for extended fields
# see https://www.microfocus.com/documentation/arcsight/arcsight-smartconnectors-8.3/cef-implementation-standard/#CEF/Chapter%202%20ArcSight%20Extension.htm
extended_ints = {
'spid', 'customerKey', 'deviceTranslatedZoneKey', 'oldFileSize',
'destinationTranslatedPort', 'cn3', 'sourceTranslatedPort', 'in', 'fsize', 'slat',
'dpid', 'cnt', 'agentZoneKey', 'out', 'type', 'eventId', 'dlong', 'cn2',
'deviceDirection', 'spt', 'agentTranslatedZoneKey', 'sTranslatedZoneKey', 'cn1',
'slong', 'dZoneKey', 'deviceZoneKey', 'dvcpid', 'dpt', 'dTranslatedZoneKey', 'dlat',
'sZoneKey'
}
extended_floats = {
'cfp1', 'cfp2', 'cfp3', 'cfp4'
}
extended_dt = {
'deviceCustomDate1', 'deviceCustomDate2', 'end', 'fileCreateTime',
'fileModificationTime', 'flexDate1', 'oldFileCreateTime', 'oldFileModificationTime',
'rt', 'start', 'art'
}
for item in proc_data:
for key, value in item.copy().items():
if key in extended_ints:
try:
item[key] = int(value)
except Exception:
pass
if key in extended_floats:
try:
item[key] = float(value)
except Exception:
pass
if key in extended_dt:
dt = jc.utils.timestamp(item[key])
item[key + '_epoch'] = dt.naive
item[key + '_epoch_utc'] = dt.utc
# Process custom field labels (from pycef library)
for key in list(item.keys()):
# If the key string ends with Label, replace it in the appropriate
# custom field
if key[-5:] == "Label":
customlabel = key[:-5]
# Find the corresponding customfield and replace with the label
for customfield in list(item.keys()):
if customfield == customlabel:
item[item[key]] = item[customfield]
del item[customfield]
del item[key]
for key, value in item.copy().items():
if isinstance(item[key], str):
# remove any spaces around values