fix file_stats in git-log parsers

CHANGELOG

@@ -1,7 +1,8 @@
 jc changelog
 
 20240510 v1.25.3
-- Add `battery_percentage` field to `bluetoothctl` parser output
+- Enhance `bluetoothctl` parser with added `battery_percentage` field
+- Enhance `git-log` standard and streaming parsers with added `lines_changed` field under `file_stats`
 - Fix `pci-ids` parser to correctly handle multiple subdevices
 - Fix `pip-show` parser to handle multi-line fields with a beginning blank line
 - Fix `top` parsers to quiet uptime info parsing

docs/parsers/git_log.md

@@ -55,6 +55,12 @@ Schema:
           "deletions":          integer,
           "files": [
                                 string
+          ],
+          "file_stats": [
+            {
+              "name":           string,
+              "lines_changed":  integer
+            }
           ]
         }
       }
@@ -174,4 +180,4 @@ Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
 Source: [`jc/parsers/git_log.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/git_log.py)
 
-Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/git_log_s.md

@@ -56,6 +56,12 @@ Schema:
         "deletions":          integer,
         "files": [
                               string
+        ],
+        "file_stats": [
+          {
+            "name":           string,
+            "lines_changed":  integer
+          }
         ]
       }
 
@@ -109,4 +115,4 @@ Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
 Source: [`jc/parsers/git_log_s.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/git_log_s.py)
 
-Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/parsers/git_log.py

@@ -50,6 +50,12 @@ Schema:
           "deletions":          integer,
           "files": [
                                 string
+          ],
+          "file_stats": [
+            {
+              "name":           string,
+              "lines_changed":  integer
+            }
           ]
         }
       }
@@ -145,7 +151,7 @@ Examples:
     ]
 """
 import re
-from typing import List, Dict
+from typing import List, Dict, Any
 import jc.utils
 
 hash_pattern = re.compile(r'(?:[0-9]|[a-f]){40}')
@@ -153,7 +159,7 @@ changes_pattern = re.compile(r'\s(?P<files>\d+)\s+(files? changed)(?:,\s+(?P<ins
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.4'
+    version = '1.5'
     description = '`git log` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -177,7 +183,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
 
         List of Dictionaries. Structured to conform to the schema.
     """
-    int_list = {'files_changed', 'insertions', 'deletions'}
+    int_list = {'files_changed', 'insertions', 'deletions', 'lines_changed'}
 
     for entry in proc_data:
         if 'date' in entry:
@@ -190,6 +196,13 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
                 if key in int_list:
                     entry['stats'][key] = jc.utils.convert_to_int(entry['stats'][key])
 
+            if 'file_stats' in entry['stats']:
+                file_stats = entry['stats']['file_stats']
+                for file_entry in file_stats:
+                    for key in file_entry:
+                        if key in int_list:
+                            file_entry[key] = jc.utils.convert_to_int(file_entry[key])
+
     return proc_data
 
 
@@ -251,6 +264,7 @@ def parse(
     output_line: Dict = {}
     message_lines: List[str] = []
     file_list: List[str] = []
+    file_stats_list: List[Dict[str, Any]] = []
 
     if jc.utils.has_data(data):
 
@@ -263,10 +277,14 @@ def parse(
                     if file_list:
                         output_line['stats']['files'] = file_list
 
+                    if file_stats_list:
+                        output_line['stats']['file_stats'] = file_stats_list
+
                     raw_output.append(output_line)
                     output_line = {}
                     message_lines = []
                     file_list = []
+                    file_stats_list = []
                 output_line = {
                     'commit': line_list[0],
                     'message': line_list[1]
@@ -282,10 +300,14 @@ def parse(
                     if file_list:
                         output_line['stats']['files'] = file_list
 
+                    if file_stats_list:
+                        output_line['stats']['file_stats'] = file_stats_list
+
                     raw_output.append(output_line)
                     output_line = {}
                     message_lines = []
                     file_list = []
+                    file_stats_list = []
                 output_line['commit'] = line_list[1]
                 continue
 
@@ -319,21 +341,19 @@ def parse(
 
             if line.startswith(' ') and 'changed, ' not in line:
                 # this is a file name
-                file_name = line.split('|')[0].strip()
-                file_stats = line.split('|')[1].strip()
+                file_line_split = line.split('|')
+                file_name = file_line_split[0].strip()
+                file_list.append(file_name)
+
+                if len(file_line_split) > 1:
+                    file_stats = file_line_split[1].strip()
                     lines_changed_str = file_stats.split(' ')
                     lines_changed_count_str = lines_changed_str[0].strip()
-                lines_changed = 0
-                try:
-                    lines_changed = int(lines_changed_count_str)
-                except:
-                    #nothing to do
-                    pass
 
-                file = {}
-                file["name"] = file_name
-                file["lines_changed"] = lines_changed
-                file_list.append(file)
+                file_stat = {}
+                file_stat["name"] = file_name
+                file_stat["lines_changed"] = lines_changed_count_str
+                file_stats_list.append(file_stat)
                 continue
 
             if line.startswith(' ') and 'changed, ' in line:
@@ -357,6 +377,9 @@ def parse(
         if file_list:
             output_line['stats']['files'] = file_list
 
+        if file_stats_list:
+            output_line['stats']['file_stats'] = file_stats_list
+
         raw_output.append(output_line)
 
     return raw_output if raw else _process(raw_output)

jc/parsers/git_log_s.py

@@ -51,6 +51,12 @@ Schema:
         "deletions":          integer,
         "files": [
                               string
+        ],
+        "file_stats": [
+          {
+            "name":           string,
+            "lines_changed":  integer
+          }
         ]
       }
 
@@ -73,7 +79,7 @@ Examples:
     ...
 """
 import re
-from typing import List, Dict, Iterable, Union
+from typing import List, Dict, Any, Iterable, Union
 import jc.utils
 from jc.parsers.git_log import _parse_name_email
 from jc.streaming import (
@@ -88,7 +94,7 @@ changes_pattern = re.compile(r'\s(?P<files>\d+)\s+(files? changed)(?:,\s+(?P<ins
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.4'
+    version = '1.5'
     description = '`git log` command streaming parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -112,7 +118,7 @@ def _process(proc_data: Dict) -> Dict:
 
         Dictionary. Structured data to conform to the schema.
     """
-    int_list = {'files_changed', 'insertions', 'deletions'}
+    int_list = {'files_changed', 'insertions', 'deletions', 'lines_changed'}
 
     if 'date' in proc_data:
         ts = jc.utils.timestamp(proc_data['date'], format_hint=(1100,))
@@ -124,6 +130,13 @@ def _process(proc_data: Dict) -> Dict:
             if key in int_list:
                 proc_data['stats'][key] = jc.utils.convert_to_int(proc_data['stats'][key])
 
+        if 'file_stats' in proc_data['stats']:
+                file_stats = proc_data['stats']['file_stats']
+                for file_entry in file_stats:
+                    for key in file_entry:
+                        if key in int_list:
+                            file_entry[key] = jc.utils.convert_to_int(file_entry[key])
+
     return proc_data
 
 
@@ -168,6 +181,7 @@ def parse(
     output_line: Dict = {}
     message_lines: List[str] = []
     file_list: List[str] = []
+    file_stats_list: List[Dict[str, Any]] = []
 
     for line in data:
         try:
@@ -184,11 +198,15 @@ def parse(
                     if file_list:
                         output_line['stats']['files'] = file_list
 
+                    if file_stats_list:
+                        output_line['stats']['file_stats'] = file_stats_list
+
                     yield output_line if raw else _process(output_line)
 
                     output_line = {}
                     message_lines = []
                     file_list = []
+                    file_stats_list = []
                 output_line = {
                     'commit': line_list[0],
                     'message': line_list[1]
@@ -204,11 +222,15 @@ def parse(
                     if file_list:
                         output_line['stats']['files'] = file_list
 
+                    if file_stats_list:
+                        output_line['stats']['file_stats'] = file_stats_list
+
                     yield output_line if raw else _process(output_line)
 
                     output_line = {}
                     message_lines = []
                     file_list = []
+                    file_stats_list = []
                 output_line['commit'] = line_list[1]
                 continue
 
@@ -242,8 +264,19 @@ def parse(
 
             if line.startswith(' ') and 'changed, ' not in line:
                 # this is a file name
-                file_name = line.split('|')[0].strip()
+                file_line_split = line.split('|')
+                file_name = file_line_split[0].strip()
                 file_list.append(file_name)
+
+                if len(file_line_split) > 1:
+                    file_stats = file_line_split[1].strip()
+                    lines_changed_str = file_stats.split(' ')
+                    lines_changed_count_str = lines_changed_str[0].strip()
+
+                file_stat = {}
+                file_stat["name"] = file_name
+                file_stat["lines_changed"] = lines_changed_count_str
+                file_stats_list.append(file_stat)
                 continue
 
             if line.startswith(' ') and 'changed, ' in line:
@@ -274,6 +307,9 @@ def parse(
             if file_list:
                 output_line['stats']['files'] = file_list
 
+            if file_stats_list:
+                output_line['stats']['file_stats'] = file_stats_list
+
             yield output_line if raw else _process(output_line)
 
     except Exception as e:

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2024-05-10 1.25.3 "JSON Convert"
+.TH jc 1 2024-05-14 1.25.3 "JSON Convert"
 .SH NAME
 \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
 and strings

tests/fixtures/generic/git-log-hash-in-message-fix-streaming.json

@@ -1 +1 @@
-[{"commit":"e05824a36ca62aa9f3a21854ec8b40a3e0f7a68d","author":"Benedikt Heine","author_email":"bebe@bebehei.de","date":"Mon Oct 28 12:42:22 2019 +0100","commit_by":"Benedikt Heine","commit_by_email":"bebe@bebehei.de","commit_by_date":"Sun Apr 12 17:27:16 2020 +0200","stats":{"files_changed":1,"insertions":13,"deletions":3,"files":["salt/modules/monit.py"]},"message":"Split monit status fields on monit version\n\nWith the commit [0] on monit, the field size changed. So splitting hard\nafter 35 chars, new versions of monit break when using monit.status.\n\n[0] https://bitbucket.org/tildeslash/monit/commits/\n471c4bbc388c1c536f07ce1dd26b811bd39a9467","epoch":1572291742,"epoch_utc":null},{"commit":"910a2ac4809bb05b886adfe75f4857eb53fdfbb1","merge":"6c3964ce30 f0a1e923e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:37 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:37 2020 -0700","message":"Merge pull request #53911 from terminalmage/squelch-log\n\nalternatives: Don't log error when running \"alternatives --display\" on nonexistant target","epoch":1586675377,"epoch_utc":null},{"commit":"6c3964ce30929e749c0965bc0d60527e9fe8dbb1","merge":"3026c25faf 2ac4da54e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:16 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:16 2020 -0700","message":"Merge pull request #54199 from driskell/patch-2\n\nFix broken sdb.get_or_set_hash for Hashicorp Vault","epoch":1586675356,"epoch_utc":null}]
+[{"commit":"e05824a36ca62aa9f3a21854ec8b40a3e0f7a68d","author":"Benedikt Heine","author_email":"bebe@bebehei.de","date":"Mon Oct 28 12:42:22 2019 +0100","commit_by":"Benedikt Heine","commit_by_email":"bebe@bebehei.de","commit_by_date":"Sun Apr 12 17:27:16 2020 +0200","stats":{"files_changed":1,"insertions":13,"deletions":3,"files":["salt/modules/monit.py"],"file_stats":[{"name":"salt/modules/monit.py","lines_changed":16}]},"message":"Split monit status fields on monit version\n\nWith the commit [0] on monit, the field size changed. So splitting hard\nafter 35 chars, new versions of monit break when using monit.status.\n\n[0] https://bitbucket.org/tildeslash/monit/commits/\n471c4bbc388c1c536f07ce1dd26b811bd39a9467","epoch":1572291742,"epoch_utc":null},{"commit":"910a2ac4809bb05b886adfe75f4857eb53fdfbb1","merge":"6c3964ce30 f0a1e923e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:37 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:37 2020 -0700","message":"Merge pull request #53911 from terminalmage/squelch-log\n\nalternatives: Don't log error when running \"alternatives --display\" on nonexistant target","epoch":1586675377,"epoch_utc":null},{"commit":"6c3964ce30929e749c0965bc0d60527e9fe8dbb1","merge":"3026c25faf 2ac4da54e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:16 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:16 2020 -0700","message":"Merge pull request #54199 from driskell/patch-2\n\nFix broken sdb.get_or_set_hash for Hashicorp Vault","epoch":1586675356,"epoch_utc":null}]

tests/fixtures/generic/git-log-hash-in-message-fix.json

@@ -1 +1 @@
-[{"commit":"e05824a36ca62aa9f3a21854ec8b40a3e0f7a68d","author":"Benedikt Heine","author_email":"bebe@bebehei.de","date":"Mon Oct 28 12:42:22 2019 +0100","commit_by":"Benedikt Heine","commit_by_email":"bebe@bebehei.de","commit_by_date":"Sun Apr 12 17:27:16 2020 +0200","stats":{"files_changed":1,"insertions":13,"deletions":3,"files":["salt/modules/monit.py"]},"message":"Split monit status fields on monit version\n\nWith the commit [0] on monit, the field size changed. So splitting hard\nafter 35 chars, new versions of monit break when using monit.status.\n\n[0] https://bitbucket.org/tildeslash/monit/commits/\n471c4bbc388c1c536f07ce1dd26b811bd39a9467","epoch":1572291742,"epoch_utc":null},{"commit":"910a2ac4809bb05b886adfe75f4857eb53fdfbb1","merge":"6c3964ce30 f0a1e923e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:37 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:37 2020 -0700","message":"Merge pull request #53911 from terminalmage/squelch-log\n\nalternatives: Don't log error when running \"alternatives --display\" on nonexistant target","epoch":1586675377,"epoch_utc":null},{"commit":"6c3964ce30929e749c0965bc0d60527e9fe8dbb1","merge":"3026c25faf 2ac4da54e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:16 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:16 2020 -0700","message":"Merge pull request #54199 from driskell/patch-2\n\nFix broken sdb.get_or_set_hash for Hashicorp Vault","epoch":1586675356,"epoch_utc":null}]
+[{"commit":"e05824a36ca62aa9f3a21854ec8b40a3e0f7a68d","author":"Benedikt Heine","author_email":"bebe@bebehei.de","date":"Mon Oct 28 12:42:22 2019 +0100","commit_by":"Benedikt Heine","commit_by_email":"bebe@bebehei.de","commit_by_date":"Sun Apr 12 17:27:16 2020 +0200","stats":{"files_changed":1,"insertions":13,"deletions":3,"files":["salt/modules/monit.py"],"file_stats":[{"name":"salt/modules/monit.py","lines_changed":16}]},"message":"Split monit status fields on monit version\n\nWith the commit [0] on monit, the field size changed. So splitting hard\nafter 35 chars, new versions of monit break when using monit.status.\n\n[0] https://bitbucket.org/tildeslash/monit/commits/\n471c4bbc388c1c536f07ce1dd26b811bd39a9467","epoch":1572291742,"epoch_utc":null},{"commit":"910a2ac4809bb05b886adfe75f4857eb53fdfbb1","merge":"6c3964ce30 f0a1e923e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:37 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:37 2020 -0700","message":"Merge pull request #53911 from terminalmage/squelch-log\n\nalternatives: Don't log error when running \"alternatives --display\" on nonexistant target","epoch":1586675377,"epoch_utc":null},{"commit":"6c3964ce30929e749c0965bc0d60527e9fe8dbb1","merge":"3026c25faf 2ac4da54e3","author":"Daniel Wozniak","author_email":"dwozniak@saltstack.com","date":"Sun Apr 12 00:09:16 2020 -0700","commit_by":"GitHub","commit_by_email":"noreply@github.com","commit_by_date":"Sun Apr 12 00:09:16 2020 -0700","message":"Merge pull request #54199 from driskell/patch-2\n\nFix broken sdb.get_or_set_hash for Hashicorp Vault","epoch":1586675356,"epoch_utc":null}]

tests/test_git_log_s.py

@@ -7,7 +7,7 @@ from jc.exceptions import ParseError
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 
 # To create streaming output use:
-# $ cat git-log.out | jc --git-log-s | jello -c > git-log-streaming.json
+# $ cat git-log.out | jc --git-log-s | jello -c > git-log-streaming.json    # or jq -sc
 
 
 class MyTests(unittest.TestCase):