diff --git a/tests/fixtures/generic/cef-streaming.json b/tests/fixtures/generic/cef-streaming.json new file mode 100644 index 00000000..3343b452 --- /dev/null +++ b/tests/fixtures/generic/cef-streaming.json @@ -0,0 +1 @@ +[{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"SAMBA Login with password: lovely\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"this is a description\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"Medium","CEFVersion":0,"dvchost":"hostname","filePath":"C:\\Users\\trend\\Desktop\\eicar.exe","act":"Delete","result":"Delete","msg":"Realtime","TrendMicroDsMalwareTarget":"N/A","N_TrendMicroDsFileMD5":"44D88612FEA8A8F36DE82E1278ABB02F","TrendMicroDsFileSHA1":"3395856CE81F2B7382DEE72602F798B642F14140","TrendMicroDsFileSHA256":"275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F","TrendMicroDsDetectionConfidence":"95","TrendMicroDsRelevantDetectionNames":"Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM","Host_ID":1,"Quarantine_File_Size":205,"Container":"ContainerImageName | ContainerName | ContainerID","agentSeverityString":"Medium","agentSeverityNum":null,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ] this is equal =, this is pipe |, this is newline \n and another newline \n the end!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","start":"Nov 08 2020 12:30:00.111 UTC","start_epoch":1604867400,"start_epoch_utc":1604838600,"Host_ID":1,"Quarantine":205,"myDate":"Nov 08 2022 12:30:00.111","myDate_epoch":1667939400,"myDate_epoch_utc":null,"myFloat":3.14,"myTimestampDate":"1660966164045","myTimestampDate_epoch":1660966164,"myTimestampDate_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Illegal Resource Access","agentSeverity":"3","CEFVersion":0,"fileid":"3412341160002518171","sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cn1":200,"in":54,"xff":"44.44.44.44","dproc":"Browser","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/","requestmethod":"GET","qstr":"p=%2fetc%2fpasswd","app":"HTTP","act":"REQ_CHALLENGE_CAPTCHA","deviceExternalID":"33411452762204224","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","filetype":"30037,1001,","filepermission":"2,1,","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"Javascript_Support":"true","CO_Support":"true","Cap_Support":"NOT_SUPPORTED","VID":"c2e72124-0e8a-4dd8-b13b-3da246af3ab2","clappsig":"de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4","clapp":"Firefox","latitude":"31.8969","longitude":"34.8186","Rule_name":"Block Malicious User,High Risk Resources,","Rule_Additional_Info":",,[{\"api_specification_violation_type\":\"INVALID_PARAM_NAME\",\"parameter_name\":\"somename\"}]","agentSeverityString":"Low","agentSeverityNum":3,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"my device id","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"deviceVendor":"Kaspersky Lab","deviceProduct":"Kaspersky ICAP Server","deviceVersion":"%VERSION%","deviceEventClassId":"%EVENT_CLASS_ID%","name":"%EVENT_NAME%","agentSeverity":"%SEVERITY%","CEFVersion":0,"msg":"%EVENT_MSG%","src":"%CLIENT_IP%","dvcpid":"%ICAP_SERVER_PID%","start":"%EVENT_TIME%","fileHash":"%SCANNED_FILE_HASH%","request":"%SCANNED_URL%","start_epoch":null,"start_epoch_utc":null,"X_Client_Username":"%HTTP_USER_NAME%","X_Client_IP":"%HTTP_USER_IP%","Scan_result":"%SCAN_RESULT%","Virus_name":"%VIRUS_NAME%","SHA256":"%SCANNED_FILE_SHA256_HASH%","deviceEventClassIdNum":null},{"deviceVendor":"Elastic","deviceProduct":"Vaporware","deviceVersion":"1.0.0-alpha","deviceEventClassId":"18","name":"Web request","agentSeverity":"low","CEFVersion":0,"eventId":3457,"msg":"hello","agentSeverityString":"low","agentSeverityNum":null,"deviceEventClassIdNum":18},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.69058","deviceEventClassId":"0-1-0","name":"Insight Logs","agentSeverity":"0","CEFVersion":0,"Auth_Username":"host/Asif-Test-PC2","Auth_Authorization_Sources":"null","Auth_Login_Status":"216","Auth_Request_Timestamp":"2017-12-03 16:28:20+05:30","Auth_Protocol":"RADIUS","Auth_Source":"null","Auth_Enforcement_Profiles":"[Allow Access Profile]","Auth_NAS_Port":"null","Auth_SSID":"cppm-dot1x-test","TimestampFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","Auth_NAS_Port_Type":"19","Auth_Error_Code":"216","Auth_Roles":"null","Auth_Service":"Test Wireless","Auth_Host_MAC_Address":"6817294b0636","Auth_Unhealthy":"null","Auth_NAS_IP_Address":"10.17.4.7","src":"10.17.4.208","Auth_CalledStationId":"000B8661CD70","Auth_NAS_Identifier":"ClearPassLab3600","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"unparsable":"unparsable line"},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.68754","deviceEventClassId":"13-1-0","name":"Audit Records","agentSeverity":"5","CEFVersion":0,"cat":"Role","timeFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","rt":"Nov 19, 2014 18:21:13 IST","src":"Test Role 10","act":"ADD","usrName":"admin","rt_epoch":null,"rt_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":5,"deviceEventClassIdNum":null}] diff --git a/tests/fixtures/generic/cef.json b/tests/fixtures/generic/cef.json new file mode 100644 index 00000000..3343b452 --- /dev/null +++ b/tests/fixtures/generic/cef.json @@ -0,0 +1 @@ +[{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"SAMBA Login with password: lovely\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Fortinet","deviceProduct":"FortiDeceptor","deviceVersion":"3.2.0","deviceEventClassId":"1","name":"SYSTEM","agentSeverity":"1","CEFVersion":0,"date":"2020-12-08","time":"16:59:33","logid":"0136000001","type":"event","subtype":"attack","level":"alert","user":"system","ui":"GUI","action":"Incident_Detection","status":"success","reason":"none","EventID":"1845921387423247329","IncidentID":"1845921507147395878","Tagkey":"192.168.100.1:59840:192.168.100.21:1836840592250413230","AttackerIP":"192.168.100.1","AttackerPort":"59840","VictimIP":"192.168.100.21","VictimPort":"445","Operation":"Logon_via_net_share","Service":"SAMBA","Username":"glen","Password":"lovely","Description":"\"this is a description\"\"","agentSeverityString":"Low","agentSeverityNum":1,"deviceEventClassIdNum":1},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"Medium","CEFVersion":0,"dvchost":"hostname","filePath":"C:\\Users\\trend\\Desktop\\eicar.exe","act":"Delete","result":"Delete","msg":"Realtime","TrendMicroDsMalwareTarget":"N/A","N_TrendMicroDsFileMD5":"44D88612FEA8A8F36DE82E1278ABB02F","TrendMicroDsFileSHA1":"3395856CE81F2B7382DEE72602F798B642F14140","TrendMicroDsFileSHA256":"275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F","TrendMicroDsDetectionConfidence":"95","TrendMicroDsRelevantDetectionNames":"Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM","Host_ID":1,"Quarantine_File_Size":205,"Container":"ContainerImageName | ContainerName | ContainerID","agentSeverityString":"Medium","agentSeverityNum":null,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ] this is equal =, this is pipe |, this is newline \n and another newline \n the end!","another":"field","Host_ID":1,"Quarantine":205,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Trend Micro","deviceProduct":"Deep Security Agent","deviceVersion":"","deviceEventClassId":"4000000","name":"Eicar_test_file","agentSeverity":"6","CEFVersion":0,"dvchost":"hostname","string":"hello \"world\" this is a backslash: \\ and this is a bracket ]!","another":"field","start":"Nov 08 2020 12:30:00.111 UTC","start_epoch":1604867400,"start_epoch_utc":1604838600,"Host_ID":1,"Quarantine":205,"myDate":"Nov 08 2022 12:30:00.111","myDate_epoch":1667939400,"myDate_epoch_utc":null,"myFloat":3.14,"myTimestampDate":"1660966164045","myTimestampDate_epoch":1660966164,"myTimestampDate_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":6,"deviceEventClassIdNum":4000000},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Illegal Resource Access","agentSeverity":"3","CEFVersion":0,"fileid":"3412341160002518171","sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cn1":200,"in":54,"xff":"44.44.44.44","dproc":"Browser","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/","requestmethod":"GET","qstr":"p=%2fetc%2fpasswd","app":"HTTP","act":"REQ_CHALLENGE_CAPTCHA","deviceExternalID":"33411452762204224","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","filetype":"30037,1001,","filepermission":"2,1,","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"Javascript_Support":"true","CO_Support":"true","Cap_Support":"NOT_SUPPORTED","VID":"c2e72124-0e8a-4dd8-b13b-3da246af3ab2","clappsig":"de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4","clapp":"Firefox","latitude":"31.8969","longitude":"34.8186","Rule_name":"Block Malicious User,High Risk Resources,","Rule_Additional_Info":",,[{\"api_specification_violation_type\":\"INVALID_PARAM_NAME\",\"parameter_name\":\"somename\"}]","agentSeverityString":"Low","agentSeverityNum":3,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"1","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":1},{"deviceVendor":"Incapsula","deviceProduct":"SIEMintegration","deviceVersion":"1","deviceEventClassId":"my device id","name":"Normal","agentSeverity":"0","CEFVersion":0,"sourceServiceName":"site123.abcd.info","siteid":"1509732","suid":"50005477","requestClientApplication":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0","deviceFacility":"mia","ccode":"IL","tag":"www.elvis.com","cicode":"Rehovot","Customer":"CEFcustomer123","siteTag":"my-site-tag","start":"1453290121336","request":"site123.abcd.info/main.css","ref":"www.incapsula.com/lama","requestmethod":"GET","cn1":200,"app":"HTTP","deviceExternalID":"33411452762204224","in":54,"xff":"44.44.44.44","cpt":"443","src":"12.12.12.12","ver":"TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256","end":"1566300670892","additionalReqHeaders":"[{\"Accept\":\"*/*\"},{\"x-v\":\"1\"},{\"x-fapi-interaction-id\":\"10.10.10.10\"}]","additionalResHeaders":"[{\"Content-Type\":\"text/html; charset=UTF-8\"}]","start_epoch":1453290121,"start_epoch_utc":null,"end_epoch":1566300670,"end_epoch_utc":null,"latitude":"31.8969","longitude":"34.8186","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"deviceVendor":"Kaspersky Lab","deviceProduct":"Kaspersky ICAP Server","deviceVersion":"%VERSION%","deviceEventClassId":"%EVENT_CLASS_ID%","name":"%EVENT_NAME%","agentSeverity":"%SEVERITY%","CEFVersion":0,"msg":"%EVENT_MSG%","src":"%CLIENT_IP%","dvcpid":"%ICAP_SERVER_PID%","start":"%EVENT_TIME%","fileHash":"%SCANNED_FILE_HASH%","request":"%SCANNED_URL%","start_epoch":null,"start_epoch_utc":null,"X_Client_Username":"%HTTP_USER_NAME%","X_Client_IP":"%HTTP_USER_IP%","Scan_result":"%SCAN_RESULT%","Virus_name":"%VIRUS_NAME%","SHA256":"%SCANNED_FILE_SHA256_HASH%","deviceEventClassIdNum":null},{"deviceVendor":"Elastic","deviceProduct":"Vaporware","deviceVersion":"1.0.0-alpha","deviceEventClassId":"18","name":"Web request","agentSeverity":"low","CEFVersion":0,"eventId":3457,"msg":"hello","agentSeverityString":"low","agentSeverityNum":null,"deviceEventClassIdNum":18},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.69058","deviceEventClassId":"0-1-0","name":"Insight Logs","agentSeverity":"0","CEFVersion":0,"Auth_Username":"host/Asif-Test-PC2","Auth_Authorization_Sources":"null","Auth_Login_Status":"216","Auth_Request_Timestamp":"2017-12-03 16:28:20+05:30","Auth_Protocol":"RADIUS","Auth_Source":"null","Auth_Enforcement_Profiles":"[Allow Access Profile]","Auth_NAS_Port":"null","Auth_SSID":"cppm-dot1x-test","TimestampFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","Auth_NAS_Port_Type":"19","Auth_Error_Code":"216","Auth_Roles":"null","Auth_Service":"Test Wireless","Auth_Host_MAC_Address":"6817294b0636","Auth_Unhealthy":"null","Auth_NAS_IP_Address":"10.17.4.7","src":"10.17.4.208","Auth_CalledStationId":"000B8661CD70","Auth_NAS_Identifier":"ClearPassLab3600","agentSeverityString":"Low","agentSeverityNum":0,"deviceEventClassIdNum":null},{"unparsable":"unparsable line"},{"deviceVendor":"Aruba Networks","deviceProduct":"ClearPass","deviceVersion":"6.5.0.68754","deviceEventClassId":"13-1-0","name":"Audit Records","agentSeverity":"5","CEFVersion":0,"cat":"Role","timeFormat":"MMM dd yyyy HH:mm:ss.SSS zzz","rt":"Nov 19, 2014 18:21:13 IST","src":"Test Role 10","act":"ADD","usrName":"admin","rt_epoch":null,"rt_epoch_utc":null,"agentSeverityString":"Medium","agentSeverityNum":5,"deviceEventClassIdNum":null}] diff --git a/tests/fixtures/generic/cef.out b/tests/fixtures/generic/cef.out new file mode 100644 index 00000000..4e986b0c --- /dev/null +++ b/tests/fixtures/generic/cef.out @@ -0,0 +1,23 @@ +CEF:0|Fortinet|FortiDeceptor|3.2.0|1|SYSTEM|1|date=2020-12-08 time=16:59:33 logid=0136000001 type=event subtype=attack level=alert user=system ui=GUI action=Incident_Detection status=success reason=none msg="EventID=1845921387423247329 IncidentID=1845921507147395878 Tagkey=192.168.100.1:59840:192.168.100.21:1836840592250413230 AttackerIP=192.168.100.1 AttackerPort=59840 VictimIP=192.168.100.21 VictimPort=445 Operation=Logon_via_net_share Service=SAMBA Username=glen Password=lovely Description=\"SAMBA Login with password: lovely\"" +CEF:0|Fortinet|FortiDeceptor|3.2.0|1|SYSTEM|1|date=2020-12-08 time=16:59:33 logid=0136000001 type=event subtype=attack level=alert user=system ui=GUI action=Incident_Detection status=success reason=none msg="EventID=1845921387423247329 IncidentID=1845921507147395878 Tagkey=192.168.100.1:59840:192.168.100.21:1836840592250413230 AttackerIP=192.168.100.1 AttackerPort=59840 VictimIP=192.168.100.21 VictimPort=445 Operation=Logon_via_net_share Service=SAMBA Username=glen Password=lovely Description=\"this is a description\"" +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \]! another=field +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|Medium|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine File Size cs6=ContainerImageName | ContainerName | ContainerID cs6Label=Container filePath=C:\Users\trend\Desktop\eicar.exe act=Delete result=Delete msg=Realtime TrendMicroDsMalwareTarget=N/A TrendMicroDsMalwareTargetType=N/TrendMicroDsFileMD5=44D88612FEA8A8F36DE82E1278ABB02F TrendMicroDsFileSHA1=3395856CE81F2B7382DEE72602F798B642F14140 TrendMicroDsFileSHA256=275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F TrendMicroDsDetectionConfidence=95 TrendMicroDsRelevantDetectionNames=Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \] this is equal \=, this is pipe \|, this is newline \n and another newline \n the end! another=field + +CEF:0|Trend Micro|Deep Security Agent||4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \]! another=field start=Nov 08 2020 12:30:00.111 UTC deviceCustomDate1=Nov 08 2022 12:30:00.111 deviceCustomDate1Label=myDate cfp1=3.14 cfp1Label=myFloat deviceCustomDate2=1660966164045 deviceCustomDate2Label=myTimestampDate + +CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsig dproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}] filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name cs11=,,[{"api_specification_violation_type":"INVALID_PARAM_NAME","parameter_name":"somename"}] cs11Label=Rule Additional Info + +CEF:0|Incapsula|SIEMintegration|1|1|Normal|0| sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia ccode=IL tag=www.elvis.com cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/main.css ref=www.incapsula.com/lama requestmethod=GET cn1=200 app=HTTP deviceExternalID=33411452762204224 in=54 xff=44.44.44.44 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}] + +CEF:0|Incapsula|SIEMintegration|1|my device id|Normal|0| sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia ccode=IL tag=www.elvis.com cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/main.css ref=www.incapsula.com/lama requestmethod=GET cn1=200 app=HTTP deviceExternalID=33411452762204224 in=54 xff=44.44.44.44 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}] + +CEF:0|Kaspersky Lab|Kaspersky ICAP Server|%VERSION%|%EVENT_CLASS_ID%|%EVENT_NAME%|%SEVERITY%| msg=%EVENT_MSG% src=%CLIENT_IP% dvcpid=%ICAP_SERVER_PID% cs2=%HTTP_USER_NAME% cs2Label=X-Client-Username cs3=%HTTP_USER_IP% cs3Label=X-Client-IP start=%EVENT_TIME% fileHash=%SCANNED_FILE_HASH% request=%SCANNED_URL% cs1=%SCAN_RESULT% cs1Label=Scan result cs4=%VIRUS_NAME% cs4Label=Virus name cs5=%SCANNED_FILE_SHA256_HASH% cs5Label=SHA256 + +<189>1 2021-06-18T10:55:50.000003Z host app - - - CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 msg=hello + +Dec 03 2017 16:31:28.861 IST 10.17.4.208 CEF:0|Aruba Networks|ClearPass|6.5.0.69058|0-1-0|Insight Logs|0|Auth.Username=host/Asif-Test-PC2 Auth.Authorization-Sources=null Auth.Login-Status=216 Auth.Request-Timestamp=2017-12-03 16:28:20+05:30 Auth.Protocol=RADIUS Auth.Source=null Auth.Enforcement-Profiles=[Allow Access Profile] Auth.NAS-Port=null Auth.SSID=cppm-dot1x-test TimestampFormat=MMM dd yyyy HH:mm:ss.SSS zzz Auth.NAS-Port-Type=19 Auth.Error-Code=216 Auth.Roles=null Auth.Service=Test Wireless Auth.Host-MAC-Address=6817294b0636 Auth.Unhealthy=null Auth.NAS-IP-Address=10.17.4.7 src=10.17.4.208 Auth.CalledStationId=000B8661CD70 Auth.NAS-Identifier=ClearPassLab3600 + +unparsable line + +Nov 19 2017 18:22:40.700 IST 10.17.4.221 CEF:0|Aruba Networks|ClearPass|6.5.0.68754|13-1-0|Audit Records|5|cat=Role timeFormat=MMM dd yyyy HH:mm:ss.SSS zzz rt=Nov 19, 2014 18:21:13 IST src=Test Role 10 act=ADD usrName=admin \ No newline at end of file diff --git a/tests/test_cef.py b/tests/test_cef.py new file mode 100644 index 00000000..5154bfff --- /dev/null +++ b/tests/test_cef.py @@ -0,0 +1,35 @@ +import os +import unittest +import json +import jc.parsers.cef + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.out'), 'r', encoding='utf-8') as f: + self.cef = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.json'), 'r', encoding='utf-8') as f: + self.cef_json = json.loads(f.read()) + + + def test_cef_nodata(self): + """ + Test 'cef' with no data + """ + self.assertEqual(jc.parsers.cef.parse('', quiet=True), []) + + def test_cef_sample(self): + """ + Test with sample cef log + """ + self.assertEqual(jc.parsers.cef.parse(self.cef, quiet=True), self.cef_json) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_cef_s.py b/tests/test_cef_s.py new file mode 100644 index 00000000..426d2523 --- /dev/null +++ b/tests/test_cef_s.py @@ -0,0 +1,37 @@ +import os +import json +import unittest +import jc.parsers.cef_s + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# To create streaming output use: +# $ cat cef.out | jc --cef-s | jello -c > cef-streaming.json + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.out'), 'r', encoding='utf-8') as f: + self.cef = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef-streaming.json'), 'r', encoding='utf-8') as f: + self.cef_streaming_json = json.loads(f.read()) + + def test_cef_s_nodata(self): + """ + Test 'cef' with no data + """ + self.assertEqual(list(jc.parsers.cef_s.parse([], quiet=True)), []) + + def test_cef_s_sample(self): + """ + Test with sample cef log + """ + self.assertEqual(list(jc.parsers.cef_s.parse(self.cef.splitlines(), quiet=True)), self.cef_streaming_json) + + +if __name__ == '__main__': + unittest.main()