add cef tests

tests/fixtures/generic/cef.out

@@ -0,0 +1,23 @@
+CEF:0|Fortinet|FortiDeceptor|3.2.0|1|SYSTEM|1|date=2020-12-08 time=16:59:33 logid=0136000001 type=event subtype=attack  level=alert user=system ui=GUI action=Incident_Detection status=success reason=none msg="EventID=1845921387423247329  IncidentID=1845921507147395878 Tagkey=192.168.100.1:59840:192.168.100.21:1836840592250413230  AttackerIP=192.168.100.1 AttackerPort=59840 VictimIP=192.168.100.21 VictimPort=445 Operation=Logon_via_net_share Service=SAMBA Username=glen Password=lovely Description=\"SAMBA Login with password: lovely\""
+CEF:0|Fortinet|FortiDeceptor|3.2.0|1|SYSTEM|1|date=2020-12-08 time=16:59:33 logid=0136000001 type=event subtype=attack  level=alert user=system ui=GUI action=Incident_Detection status=success reason=none msg="EventID=1845921387423247329  IncidentID=1845921507147395878 Tagkey=192.168.100.1:59840:192.168.100.21:1836840592250413230  AttackerIP=192.168.100.1 AttackerPort=59840 VictimIP=192.168.100.21 VictimPort=445 Operation=Logon_via_net_share Service=SAMBA Username=glen Password=lovely Description=\"this is a description\""
+CEF:0|Trend Micro|Deep Security Agent|<DSA version>|4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \]! another=field
+CEF:0|Trend Micro|Deep Security Agent|<DSA version>|4000000|Eicar_test_file|Medium|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine File Size cs6=ContainerImageName | ContainerName | ContainerID cs6Label=Container filePath=C:\Users\trend\Desktop\eicar.exe act=Delete result=Delete msg=Realtime TrendMicroDsMalwareTarget=N/A TrendMicroDsMalwareTargetType=N/TrendMicroDsFileMD5=44D88612FEA8A8F36DE82E1278ABB02F TrendMicroDsFileSHA1=3395856CE81F2B7382DEE72602F798B642F14140 TrendMicroDsFileSHA256=275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F TrendMicroDsDetectionConfidence=95 TrendMicroDsRelevantDetectionNames=Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM
+CEF:0|Trend Micro|Deep Security Agent|<DSA version>|4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \] this is equal \=, this is pipe \|, this is newline \n and another newline \n the end! another=field
+
+CEF:0|Trend Micro|Deep Security Agent|<DSA version>|4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine string=hello \"world\" this is a backslash: \\ and this is a bracket \]! another=field start=Nov 08 2020 12:30:00.111 UTC deviceCustomDate1=Nov 08 2022 12:30:00.111 deviceCustomDate1Label=myDate cfp1=3.14 cfp1Label=myFloat deviceCustomDate2=1660966164045 deviceCustomDate2Label=myTimestampDate
+
+CEF:0|Incapsula|SIEMintegration|1|1|Illegal Resource Access|3| fileid=3412341160002518171 sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support ccode=IL tag=www.elvis.com cn1=200 in=54 xff=44.44.44.44 cs1=NOT_SUPPORTED cs1Label=Cap Support cs4=c2e72124-0e8a-4dd8-b13b-3da246af3ab2 cs4Label=VID cs5=de3c633ac428e0678f3aac20cf7f239431e54cbb8a17e8302f53653923305e1835a9cd871db32aa4fc7b8a9463366cc4 cs5Label=clappsig dproc=Browser cs6=Firefox cs6Label=clapp ccode=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/ requestmethod=GET qstr=p\=%2fetc%2fpasswd app=HTTP act=REQ_CHALLENGE_CAPTCHA deviceExternalID=33411452762204224 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}] filetype=30037,1001, filepermission=2,1, cs9=Block Malicious User,High Risk Resources, cs9Label=Rule name cs11=,,[{"api_specification_violation_type":"INVALID_PARAM_NAME","parameter_name":"somename"}] cs11Label=Rule Additional Info
+
+CEF:0|Incapsula|SIEMintegration|1|1|Normal|0| sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia ccode=IL tag=www.elvis.com cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/main.css ref=www.incapsula.com/lama requestmethod=GET cn1=200 app=HTTP deviceExternalID=33411452762204224 in=54 xff=44.44.44.44 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}]
+
+CEF:0|Incapsula|SIEMintegration|1|my device id|Normal|0| sourceServiceName=site123.abcd.info siteid=1509732 suid=50005477 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 deviceFacility=mia ccode=IL tag=www.elvis.com cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=CEFcustomer123 siteTag=my-site-tag start=1453290121336 request=site123.abcd.info/main.css ref=www.incapsula.com/lama requestmethod=GET cn1=200 app=HTTP deviceExternalID=33411452762204224 in=54 xff=44.44.44.44 cpt=443 src=12.12.12.12 ver=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 end=1566300670892 additionalReqHeaders=[{"Accept":"*/*"},{"x-v":"1"},{"x-fapi-interaction-id":"10.10.10.10"}] additionalResHeaders=[{"Content-Type":"text/html; charset\=UTF-8"}]
+
+CEF:0|Kaspersky Lab|Kaspersky ICAP Server|%VERSION%|%EVENT_CLASS_ID%|%EVENT_NAME%|%SEVERITY%| msg=%EVENT_MSG% src=%CLIENT_IP% dvcpid=%ICAP_SERVER_PID% cs2=%HTTP_USER_NAME% cs2Label=X-Client-Username cs3=%HTTP_USER_IP% cs3Label=X-Client-IP start=%EVENT_TIME% fileHash=%SCANNED_FILE_HASH% request=%SCANNED_URL% cs1=%SCAN_RESULT% cs1Label=Scan result cs4=%VIRUS_NAME% cs4Label=Virus name cs5=%SCANNED_FILE_SHA256_HASH% cs5Label=SHA256
+
+<189>1 2021-06-18T10:55:50.000003Z host app - - - CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 msg=hello
+
+Dec 03 2017 16:31:28.861 IST 10.17.4.208 CEF:0|Aruba Networks|ClearPass|6.5.0.69058|0-1-0|Insight Logs|0|Auth.Username=host/Asif-Test-PC2 Auth.Authorization-Sources=null Auth.Login-Status=216 Auth.Request-Timestamp=2017-12-03 16:28:20+05:30 Auth.Protocol=RADIUS Auth.Source=null Auth.Enforcement-Profiles=[Allow Access Profile] Auth.NAS-Port=null Auth.SSID=cppm-dot1x-test TimestampFormat=MMM dd yyyy HH:mm:ss.SSS zzz Auth.NAS-Port-Type=19 Auth.Error-Code=216 Auth.Roles=null Auth.Service=Test Wireless Auth.Host-MAC-Address=6817294b0636 Auth.Unhealthy=null Auth.NAS-IP-Address=10.17.4.7 src=10.17.4.208 Auth.CalledStationId=000B8661CD70 Auth.NAS-Identifier=ClearPassLab3600
+
+unparsable line
+
+Nov 19 2017 18:22:40.700 IST 10.17.4.221 CEF:0|Aruba Networks|ClearPass|6.5.0.68754|13-1-0|Audit Records|5|cat=Role timeFormat=MMM dd yyyy HH:mm:ss.SSS zzz rt=Nov 19, 2014 18:21:13 IST src=Test Role 10 act=ADD usrName=admin

tests/test_cef.py

@@ -0,0 +1,35 @@
+import os
+import unittest
+import json
+import jc.parsers.cef
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class MyTests(unittest.TestCase):
+
+    def setUp(self):
+        # input
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.out'), 'r', encoding='utf-8') as f:
+            self.cef = f.read()
+
+        # output
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.json'), 'r', encoding='utf-8') as f:
+            self.cef_json = json.loads(f.read())
+
+
+    def test_cef_nodata(self):
+        """
+        Test 'cef' with no data
+        """
+        self.assertEqual(jc.parsers.cef.parse('', quiet=True), [])
+
+    def test_cef_sample(self):
+        """
+        Test with sample cef log
+        """
+        self.assertEqual(jc.parsers.cef.parse(self.cef, quiet=True), self.cef_json)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_cef_s.py

@@ -0,0 +1,37 @@
+import os
+import json
+import unittest
+import jc.parsers.cef_s
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# To create streaming output use:
+# $ cat cef.out | jc --cef-s | jello -c > cef-streaming.json
+
+
+class MyTests(unittest.TestCase):
+
+    def setUp(self):
+        # input
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef.out'), 'r', encoding='utf-8') as f:
+            self.cef = f.read()
+
+        # output
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/cef-streaming.json'), 'r', encoding='utf-8') as f:
+            self.cef_streaming_json = json.loads(f.read())
+
+    def test_cef_s_nodata(self):
+        """
+        Test 'cef' with no data
+        """
+        self.assertEqual(list(jc.parsers.cef_s.parse([], quiet=True)), [])
+
+    def test_cef_s_sample(self):
+        """
+        Test with sample cef log
+        """
+        self.assertEqual(list(jc.parsers.cef_s.parse(self.cef.splitlines(), quiet=True)), self.cef_streaming_json)
+
+
+if __name__ == '__main__':
+    unittest.main()