python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-25 00:37:31 +02:00
Code Issues Releases Activity

cleanup custom field processing and add datetime support

Browse Source
This commit is contained in:
Kelly Brazil
2022-08-18 10:55:37 -07:00
parent 5b597b6583
commit d71a7fbbed
1 changed files with 23 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
jc/parsers/cef.py
View File

@ -251,19 +251,33 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
item[key + '_epoch'] = dt.naive item[key + '_epoch'] = dt.naive
item[key + '_epoch_utc'] = dt.utc item[key + '_epoch_utc'] = dt.utc
# Process custom field labels (from pycef library) # Process custom field labels (adapted from pycef library)
for key in list(item.keys()): cleanup_list = []
# If the key string ends with Label, replace it in the appropriate custom_fields = list(item.keys())
# custom field for key in custom_fields:
if key[-5:] == "Label": if key.endswith('Label'):
customlabel = key[:-5] customlabel = key[:-5]
# Find the corresponding customfield and replace with the label for customfield in custom_fields:
for customfield in list(item.keys()): # check for normal custom fields
if customfield == customlabel: if customfield == customlabel:
item[item[key]] = item[customfield] item[item[key]] = item[customfield]
del item[customfield] cleanup_list.append(customfield)
cleanup_list.append(key)
# check for datetime objects
if customfield == customlabel + '_epoch':
item[item[key] + '_epoch'] = item[customfield]
cleanup_list.append(customfield)
if customfield == customlabel + '_epoch_utc':
item[item[key] + '_epoch_utc'] = item[customfield]
cleanup_list.append(customfield)
# cleanup extra custom fields
for key in cleanup_list:
del item[key] del item[key]
# more normalization
for key, value in item.copy().items(): for key, value in item.copy().items():
if isinstance(item[key], str): if isinstance(item[key], str):
# remove any spaces around values # remove any spaces around values