Repository: LLNL/Surfactant

Issue:
Recognize perl scripts
Add support for recognizing perl scripts -- this is most likely just adding to the list of recognized file extensions for file type id, but any other ideas for verifying that the file is actually a perl script based on contents would be nice to have.

Related file: https://github.com/LLNL/Surfactant/blob/main/surfactant/filetypeid/id_extension.py

Discussion:
@Eriken79: Hey @nightlark just put in a PR for this, its PR #316. Not much of a change, but hopefully it looks good! Also wrote out some detailed comments on problems with recognizing Perl scripts!

Solution patch:
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 547b042..66832d8 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -15,7 +15,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.10", "3.13"]
 
     steps:
     - uses: actions/checkout@v4
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6f33bc8..414a8b7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 exclude: ^(.gitignore|generate_sbom.py|extract_file_info.py|pe_info.py)
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.4
+    rev: v0.8.6
     hooks:
       # Run the linter
       - id: ruff
diff --git a/surfactant/filetypeid/id_extension.py b/surfactant/filetypeid/id_extension.py
index 2ca839f..bf02e76 100644
--- a/surfactant/filetypeid/id_extension.py
+++ b/surfactant/filetypeid/id_extension.py
@@ -26,6 +26,8 @@ def identify_file_type(filepath: str) -> Optional[str]:
         ".htm": "HTML",
         ".php": "PHP",
         ".bat": "BATCH",
+        ".pl": "PERL_OR_PROLOG",
+        ".pm": "PERL_MODULE",
     }
     _interpreters = {
         b"sh": "SHELL",
@@ -34,6 +36,7 @@ def identify_file_type(filepath: str) -> Optional[str]:
         b"php": "PHP",
         b"python": "PYTHON",
         b"python3": "PYTHON",
+        b"perl": "PERL",
     }
     try:
         with open(filepath, "rb") as f:
diff --git a/surfactant/filetypeid/id_magic.py b/surfactant/filetypeid/id_magic.py
index 7567ede..c218e2a 100755
--- a/surfactant/filetypeid/id_magic.py
+++ b/surfactant/filetypeid/id_magic.py
@@ -192,6 +192,14 @@ def identify_file_type(filepath: str) -> Optional[str]:
                 int.from_bytes(magic_bytes[0:4], byteorder="big", signed=False) & 0xFF0F80FF
             ) == 0xF00D0000:
                 return "OMF_LIB"
+            # zlib:
+            # https://www.rfc-editor.org/rfc/rfc1950
+            cmf = magic_bytes[0]
+            flg = magic_bytes[1]
+            cm = cmf & 0x0F
+            if cm == 8:
+                if (cmf * 256 + flg) % 31 == 0:
+                    return "ZLIB"
             return None
     except FileNotFoundError:
         return None
diff --git a/surfactant/infoextractors/pe_file.py b/surfactant/infoextractors/pe_file.py
index 7e7dabc..bbe5ed9 100755
--- a/surfactant/infoextractors/pe_file.py
+++ b/surfactant/infoextractors/pe_file.py
@@ -35,11 +35,15 @@ def extract_file_info(sbom: SBOM, software: Software, filename: str, filetype: s
 # Values for CPU types that can appear in a PE file
 pe_machine_types = {
     0x0: "UNKNOWN",
+    0x184: "ALPHA",  # Alpha_AXP
+    0x284: "ALPHA64",  # Also AXP64
     0x1D3: "AM33",
-    0x8664: "AMD",
+    0x8664: "AMD64",
     0x1C0: "ARM",
     0xAA64: "ARM64",
     0x1C4: "ARMNT",
+    0xC0EE: "CEE",
+    0xCEF: "CEF",
     0xEBC: "EBC",
     0x14C: "I386",
     0x200: "IA64",
@@ -51,12 +55,21 @@ pe_machine_types = {
     0x466: "MIPSFPU16",
     0x1F0: "POWERPC",
     0x1F1: "POWERPCFP",
+    0x160: "R3000",  # Big-Endian
+    0x162: "R3000",  # Little-Endian
     0x166: "R4000",
+    0x168: "R10000",
     0x5032: "RISCV32",
     0x5064: "RISCV64",
     0x5128: "RISCV128",
     0x1A2: "SH3",
     0x1A3: "SH3DSP",
+    0x1A4: "SH3E",
+    0x1A6: "SH4",
+    0x1A8: "SH5",
+    0x1C2: "THUMB",
+    0x520: "TRICORE",
+    0x169: "WCEMIPSV2",
 }
 
 # https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#windows-subsystem
@@ -76,6 +89,7 @@ pe_subsystem_types = {
     13: "EFI_ROM",
     14: "XBOX",
     16: "WINDOWS_BOOT_APPLICATION",
+    17: "XBOX_CODE_CATALOG",
 }
 
 


Test patch:
diff --git a/tests/file_types/test_file_magic.py b/tests/file_types/test_file_magic.py
index 0452059..0296321 100644
--- a/tests/file_types/test_file_magic.py
+++ b/tests/file_types/test_file_magic.py
@@ -4,6 +4,10 @@
 # SPDX-License-Identifier: MIT
 import os
 import pathlib
+import sys
+import zlib
+
+import pytest
 
 from surfactant.filetypeid.id_magic import identify_file_type
 
@@ -33,3 +37,21 @@ def test_magic_id():
     data_dir = os.path.join(base_path, "..", "data")
     for file_name, file_type in _file_to_file_type.items():
         assert identify_file_type(os.path.join(data_dir, file_name)) == file_type
+
+
+def test_zlib_basic(tmp_path):
+    for compress_level in range(10):
+        write_to = tmp_path / f"basic_{compress_level}.zlib"
+        write_to.write_bytes(zlib.compress(b"hello", level=compress_level))
+        assert identify_file_type(write_to) == "ZLIB"
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 11), reason="zlib.compress wbits only available from Python 3.11+"
+)
+def test_zlib_window(tmp_path):
+    for compress_level in range(10):
+        for window_size in range(9, 16):
+            write_to = tmp_path / f"window_{compress_level}_{window_size}.zlib"
+            write_to.write_bytes(zlib.compress(b"hello", level=compress_level, wbits=window_size))
+            assert identify_file_type(write_to) == "ZLIB"