Merge pull request #673 from native-api/haslib_mode

Correctly parse mode indicators in hashsum output

jc/parsers/hashsum.py

@@ -28,6 +28,7 @@ Schema:
     [
       {
         "filename":     string,
+        "mode":         string,
         "hash":         string,
       }
     ]
@@ -38,31 +39,39 @@ Examples:
     [
       {
         "filename": "devtoolset-3-gcc-4.9.2-6.el7.x86_64.rpm",
+        "mode": "text",
         "hash": "65fc958c1add637ec23c4b137aecf3d3"
       },
       {
         "filename": "digout",
+        "mode": "text",
         "hash": "5b9312ee5aff080927753c63a347707d"
       },
       {
         "filename": "dmidecode.out",
+        "mode": "text",
         "hash": "716fd11c2ac00db109281f7110b8fb9d"
       },
       {
         "filename": "file with spaces in the name",
+        "mode": "text",
         "hash": "d41d8cd98f00b204e9800998ecf8427e"
       },
       {
         "filename": "id-centos.out",
+        "mode": "text",
         "hash": "4295be239a14ad77ef3253103de976d2"
       },
       {
         "filename": "ifcfg.json",
+        "mode": "text",
         "hash": "01fda0d9ba9a75618b072e64ff512b43"
       },
       ...
     ]
 """
+import re
+
 import jc.utils
 
 
@@ -81,6 +90,15 @@ class info():
 
 __version__ = info.version
 
+_mode_friendly_names = {
+    " ": "text",
+    "*": "binary",
+    # Perl shasum -- specific
+    "U": "universal",
+    "^": "bits",
+    # BSD-style format only supports binary mode
+    None: "binary"
+}
 
 def _process(proc_data):
     """
@@ -95,7 +113,9 @@ def _process(proc_data):
         List of Dictionaries. Structured data to conform to the schema.
     """
 
-    # no further processing for this parser
+    for entry in proc_data:
+        entry['mode'] = _mode_friendly_names.get(entry['mode'],entry['mode'])
+
     return proc_data
 
 
@@ -127,13 +147,18 @@ def parse(data, raw=False, quiet=False):
                 file_name = line.split('=', maxsplit=1)[0].strip()
                 file_name = file_name[5:]
                 file_name = file_name[:-1]
+                # filler, legacy md5 always uses binary mode
+                file_mode = None
             # standard md5sum and shasum command output
             else:
-                file_hash = line.split(maxsplit=1)[0]
-                file_name = line.split(maxsplit=1)[1]
+                m = re.match('([0-9a-f]+) (.)(.*)$', line)
+                if not m:
+                    raise ValueError(f'Invalid line format: "{line}"')
+                file_hash, file_mode, file_name = m.groups()
 
             item = {
                 'filename': file_name,
+                'mode': file_mode,
                 'hash': file_hash
             }
             raw_output.append(item)

tests/fixtures/ubuntu-18.04/shasum-portable-raw.json

@@ -0,0 +1 @@
+[{"filename": "out.txt", "mode": "?", "hash": "5a3c9b9e4594dd4a8a5e963a6e917deb844458e6"}]

tests/fixtures/ubuntu-18.04/shasum-portable.json

@@ -0,0 +1 @@
+[{"filename": "out.txt", "mode": "?", "hash": "5a3c9b9e4594dd4a8a5e963a6e917deb844458e6"}]

tests/fixtures/ubuntu-18.04/shasum-portable.out

@@ -0,0 +1 @@
+5a3c9b9e4594dd4a8a5e963a6e917deb844458e6 ?out.txt

tests/fixtures/ubuntu-24.04/sha256sum-binary-raw.json

@@ -0,0 +1 @@
+[{"filename": "openssl-3.6.0.tar.gz", "mode": "*", "hash": "b6a5f44b7eb69e3fa35dbf15524405b44837a481d43d81daddde3ff21fcbb8e9"}]

tests/fixtures/ubuntu-24.04/sha256sum-binary.json

@@ -0,0 +1 @@
+[{"filename": "openssl-3.6.0.tar.gz", "mode": "binary", "hash": "b6a5f44b7eb69e3fa35dbf15524405b44837a481d43d81daddde3ff21fcbb8e9"}]

tests/fixtures/ubuntu-24.04/sha256sum-binary.out

@@ -0,0 +1 @@
+b6a5f44b7eb69e3fa35dbf15524405b44837a481d43d81daddde3ff21fcbb8e9 *openssl-3.6.0.tar.gz

tests/fixtures/ubuntu-24.04/shasum-universal-bits-raw.json

@@ -0,0 +1 @@
+[{"filename": "out.txt", "mode": "U", "hash": "6fe4d572948d4c132d1b1b0ab91e89de4be01efd"}, {"filename": "out.txt", "mode": "^", "hash": "68382a729a930a2219f0bd10c5c4d61eec856a96"}]

tests/fixtures/ubuntu-24.04/shasum-universal-bits.json

@@ -0,0 +1 @@
+[{"filename": "out.txt", "mode": "universal", "hash": "6fe4d572948d4c132d1b1b0ab91e89de4be01efd"}, {"filename": "out.txt", "mode": "bits", "hash": "68382a729a930a2219f0bd10c5c4d61eec856a96"}]

tests/fixtures/ubuntu-24.04/shasum-universal-bits.out

@@ -0,0 +1,2 @@
+6fe4d572948d4c132d1b1b0ab91e89de4be01efd Uout.txt
+68382a729a930a2219f0bd10c5c4d61eec856a96 ^out.txt

tests/test_hashsum.py

@@ -18,6 +18,15 @@ class MyTests(unittest.TestCase):
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/sha384sum.out'), 'r', encoding='utf-8') as f:
         centos_7_7_sha384sum = f.read()
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/shasum-portable.out'), 'r', encoding='utf-8') as f:
+        ubuntu_18_04_shasum_portable = f.read()
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-24.04/sha256sum-binary.out'), 'r', encoding='utf-8') as f:
+        ubuntu_24_04_sha256sum_binary = f.read()
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-24.04/shasum-universal-bits.out'), 'r', encoding='utf-8') as f:
+        ubuntu_24_04_shasum_universal_bits = f.read()
+
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/md5.out'), 'r', encoding='utf-8') as f:
         osx_10_14_6_md5 = f.read()
 
@@ -28,18 +37,51 @@ class MyTests(unittest.TestCase):
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/md5sum.json'), 'r', encoding='utf-8') as f:
         centos_7_7_md5sum_json = json.loads(f.read())
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/md5sum-raw.json'), 'r', encoding='utf-8') as f:
+        centos_7_7_md5sum_raw_json = json.loads(f.read())
+
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/sha256sum.json'), 'r', encoding='utf-8') as f:
         centos_7_7_sha256sum_json = json.loads(f.read())
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/sha256sum-raw.json'), 'r', encoding='utf-8') as f:
+        centos_7_7_sha256sum_raw_json = json.loads(f.read())
+
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/sha384sum.json'), 'r', encoding='utf-8') as f:
         centos_7_7_sha384sum_json = json.loads(f.read())
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/sha384sum-raw.json'), 'r', encoding='utf-8') as f:
+        centos_7_7_sha384sum_raw_json = json.loads(f.read())
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/shasum-portable.json'), 'r', encoding='utf-8') as f:
+        ubuntu_18_04_shasum_portable_json = json.loads(f.read())
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/shasum-portable-raw.json'), 'r', encoding='utf-8') as f:
+        ubuntu_18_04_shasum_portable_raw_json = json.loads(f.read())
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-24.04/sha256sum-binary.json'), 'r', encoding='utf-8') as f:
+        ubuntu_24_04_sha256sum_json = json.loads(f.read())
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-24.04/sha256sum-binary-raw.json'), 'r', encoding='utf-8') as f:
+        ubuntu_24_04_sha256sum_raw_json = json.loads(f.read())
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-24.04/shasum-universal-bits.json'), 'r', encoding='utf-8') as f:
+        ubuntu_24_04_shasum_json = json.loads(f.read())
+
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-24.04/shasum-universal-bits-raw.json'), 'r', encoding='utf-8') as f:
+        ubuntu_24_04_shasum_raw_json = json.loads(f.read())
+
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/md5.json'), 'r', encoding='utf-8') as f:
         osx_10_14_6_md5_json = json.loads(f.read())
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/md5-raw.json'), 'r', encoding='utf-8') as f:
+        osx_10_14_6_md5_raw_json = json.loads(f.read())
+
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/shasum.json'), 'r', encoding='utf-8') as f:
         osx_10_14_6_shasum_json = json.loads(f.read())
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/shasum-raw.json'), 'r', encoding='utf-8') as f:
+        osx_10_14_6_shasum_raw_json = json.loads(f.read())
+
 
     def test_hashsum_nodata(self):
         """
@@ -53,30 +95,113 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.hashsum.parse(self.centos_7_7_md5sum, quiet=True), self.centos_7_7_md5sum_json)
 
+    def test_md5sum_centos_7_7_raw(self):
+        """
+        Test 'md5sum' on Centos 7.7, raw output
+        """
+        self.assertEqual(
+            jc.parsers.hashsum.parse(self.centos_7_7_md5sum, quiet=True, raw=True),
+            self.centos_7_7_md5sum_raw_json)
+
     def test_sha256sum_centos_7_7(self):
         """
         Test 'sha256sum' on Centos 7.7
         """
         self.assertEqual(jc.parsers.hashsum.parse(self.centos_7_7_sha256sum, quiet=True), self.centos_7_7_sha256sum_json)
 
+    def test_sha256sum_centos_7_7_raw(self):
+        """
+        Test 'sha256sum' on Centos 7.7, raw output
+        """
+        self.assertEqual(
+            jc.parsers.hashsum.parse(self.centos_7_7_sha256sum, quiet=True, raw=True),
+            self.centos_7_7_sha256sum_raw_json)
+
     def test_sha384sum_centos_7_7(self):
         """
         Test 'sha384sum' on Centos 7.7
         """
         self.assertEqual(jc.parsers.hashsum.parse(self.centos_7_7_sha384sum, quiet=True), self.centos_7_7_sha384sum_json)
 
+    def test_sha384sum_centos_7_7_raw(self):
+        """
+        Test 'sha384sum' on Centos 7.7, raw output
+        """
+        self.assertEqual(jc.parsers.hashsum.parse(
+            self.centos_7_7_sha384sum, quiet=True, raw=True),
+            self.centos_7_7_sha384sum_raw_json)
+
+    def test_sha256sum_ubuntu_18_04_unsupported_mode(self):
+        """
+        Test 'sha256sum' on Ubuntu 18.04, portable mode (no firendly name)
+        """
+        self.assertEqual(jc.parsers.hashsum.parse(
+            self.ubuntu_18_04_shasum_portable, quiet=True),
+            self.ubuntu_18_04_shasum_portable_json)
+
+    def test_sha256sum_ubuntu_18_04_unsupported_mode_raw(self):
+        """
+        Test 'sha256sum' on Ubuntu 18.04, portable mode (no firendly name), raw output
+        """
+        self.assertEqual(jc.parsers.hashsum.parse(
+            self.ubuntu_18_04_shasum_portable, quiet=True, raw=True),
+            self.ubuntu_18_04_shasum_portable_raw_json)
+
+    def test_sha256sum_ubuntu_24_04_binary(self):
+        """
+        Test 'sha256sum' on Ubuntu 24.04, binary mode
+        """
+        self.assertEqual(jc.parsers.hashsum.parse(self.ubuntu_24_04_sha256sum_binary, quiet=True), self.ubuntu_24_04_sha256sum_json)
+
+    def test_sha256sum_ubuntu_24_04_binary_raw(self):
+        """
+        Test 'sha256sum' on Ubuntu 24.04, binary mode, raw output
+        """
+        self.assertEqual(jc.parsers.hashsum.parse(
+            self.ubuntu_24_04_sha256sum_binary, quiet=True, raw=True),
+            self.ubuntu_24_04_sha256sum_raw_json)
+
+    def test_shasum_ubuntu_24_04_universal_bits(self):
+        """
+        Test 'shasum' on Ubuntu 24.04, universal and bits modes
+        """
+        self.assertEqual(jc.parsers.hashsum.parse(self.ubuntu_24_04_shasum_universal_bits, quiet=True), self.ubuntu_24_04_shasum_json)
+
+    def test_shasum_ubuntu_24_04_raw(self):
+        """
+        Test 'shasum' on Ubuntu 24.04, universal and bits modes, raw output
+        """
+        self.assertEqual(
+            jc.parsers.hashsum.parse(self.ubuntu_24_04_shasum_universal_bits, quiet=True, raw=True),
+            self.ubuntu_24_04_shasum_raw_json)
+
     def test_md5_osx_10_14_6(self):
         """
         Test 'md5' on OSX 10.14.6
         """
         self.assertEqual(jc.parsers.hashsum.parse(self.osx_10_14_6_md5, quiet=True), self.osx_10_14_6_md5_json)
 
+    def test_md5_osx_10_14_6_raw(self):
+        """
+        Test 'md5' on OSX 10.14.6, raw output
+        """
+        self.assertEqual(
+            jc.parsers.hashsum.parse(self.osx_10_14_6_md5, quiet=True, raw=True),
+            self.osx_10_14_6_md5_raw_json)
+
     def test_shasum_osx_10_14_6(self):
         """
         Test 'shasum' on OSX 10.14.6
         """
         self.assertEqual(jc.parsers.hashsum.parse(self.osx_10_14_6_shasum, quiet=True), self.osx_10_14_6_shasum_json)
 
+    def test_shasum_osx_10_14_6_raw(self):
+        """
+        Test 'shasum' on OSX 10.14.6, raw output
+        """
+        self.assertEqual(
+            jc.parsers.hashsum.parse(self.osx_10_14_6_shasum, quiet=True, raw=True),
+            self.osx_10_14_6_shasum_raw_json)
 
 if __name__ == '__main__':
     unittest.main()