Repository: CybercentreCanada/Maco

Issue:
CLI issue - 'module' key not found
The CLI tries to access the list of extractor instances found by the maco collector, and access the 'module' property, which was previously an instance of a maco extractor.

Due to the new importing behaviour, this property no longer exists and the CLI crashes.

The CLI is attempting to print metadata for the maco extractor such as family, author, etc. This is also useful when interacting with maco programmatically.

I've attempted to import via importlib manually as a fix but due to the modifications that are made by uv, these imports are no longer valid to import directly.

I've added a test in my own branch that shows this issue - https://github.com/seb-acsc/Maco/blob/feat/library_check/tests/test_cli.py.

```python

path_extractors = '/home/seb/work/Maco/demo_extractors', path_samples = '/home/seb/work/Maco/tests/data', include = [], exclude = []

    def process_filesystem(
        path_extractors: str,
        path_samples: str,
        include: List[str],
        exclude: List[str],
        *,
        pretty: bool,
        force: bool,
        include_base64: bool,
        create_venv: bool = False,
    ):
        if force:
            logger.warning("force execute will cause errors if an extractor requires a yara rule hit during execution")
        collected = collector.Collector(path_extractors, include=include, exclude=exclude, create_venv=create_venv)
    
        logger.info(f"extractors loaded: {[x for x in collected.extractors.keys()]}\n")
        for _, extractor in collected.extractors.items():
>           extractor = extractor["module"]
E           KeyError: 'module'

maco/cli.py:101: KeyError
==================================================================================================================================== short test summary info =====================================================================================================================================
FAILED tests/test_cli.py::TestCLI::test_process_filesystem - KeyError: 'module'
1 failed in 0.69s
```

Discussion:
@cccs-rs: Should be resolved in this release: https://github.com/CybercentreCanada/Maco/releases/tag/v1.2.2

Solution patch:
diff --git a/maco/cli.py b/maco/cli.py
index 6dd92a7..e408690 100644
--- a/maco/cli.py
+++ b/maco/cli.py
@@ -8,7 +8,7 @@
 import logging
 import os
 import sys
-from typing import BinaryIO, List
+from typing import BinaryIO, List, Tuple
 
 import cart
 
@@ -57,7 +57,7 @@ def process_file(
         # run and store results for extractor
         logger.info(f"run {extractor_name} extractor from rules {[x.rule for x in hits]}")
         try:
-            resp = collected.extract(stream, hits, extractor_name)
+            resp = collected.extract(stream, extractor_name)
         except Exception as e:
             logger.exception(f"extractor error with {path_file} ({e})")
             resp = None
@@ -91,18 +91,22 @@ def process_filesystem(
     force: bool,
     include_base64: bool,
     create_venv: bool = False,
-):
+) -> Tuple[int, int, int]:
+    """Process filesystem with extractors and print results of extraction.
+
+    Returns total number of analysed files, yara hits and successful maco extractions.
+    """
     if force:
         logger.warning("force execute will cause errors if an extractor requires a yara rule hit during execution")
     collected = collector.Collector(path_extractors, include=include, exclude=exclude, create_venv=create_venv)
 
     logger.info(f"extractors loaded: {[x for x in collected.extractors.keys()]}\n")
     for _, extractor in collected.extractors.items():
-        extractor = extractor["module"]
+        extractor_meta = extractor["metadata"]
         logger.info(
-            f"{extractor.family} by {extractor.author}"
-            f" {extractor.last_modified} {extractor.sharing}"
-            f"\n{extractor.__doc__}\n"
+            f"{extractor_meta['family']} by {extractor_meta['author']}"
+            f" {extractor_meta['last_modified']} {extractor_meta['sharing']}"
+            f"\n{extractor_meta['description']}\n"
         )
 
     num_analysed = 0
@@ -144,7 +148,7 @@ def process_filesystem(
     finally:
         logger.info("")
         logger.info(f"{num_analysed} analysed, {num_hits} hits, {num_extracted} extracted")
-
+    return num_analysed, num_hits, num_extracted
 
 def main():
     parser = argparse.ArgumentParser(description="Run extractors over samples.")
diff --git a/maco/collector.py b/maco/collector.py
index c73182d..8c35e9b 100644
--- a/maco/collector.py
+++ b/maco/collector.py
@@ -40,7 +40,11 @@ def _verify_response(resp: Union[BaseModel, dict]) -> Dict:
 
 class Collector:
     def __init__(
-        self, path_extractors: str, include: List[str] = None, exclude: List[str] = None, create_venv: bool = False
+        self,
+        path_extractors: str,
+        include: List[str] = None,
+        exclude: List[str] = None,
+        create_venv: bool = False,
     ):
         """Discover and load extractors from file system."""
         path_extractors = os.path.realpath(path_extractors)
@@ -72,6 +76,13 @@ def extractor_module_callback(module: ModuleType, venv: str):
                         module_path=module.__file__,
                         module_name=member.__module__,
                         extractor_class=member.__name__,
+                        metadata={
+                            "family": member.family,
+                            "author": member.author,
+                            "last_modified": member.last_modified,
+                            "sharing": member.sharing,
+                            "description": member.__doc__,
+                        },
                     )
                     namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
 
@@ -116,7 +127,6 @@ def match(self, stream: BinaryIO) -> Dict[str, List[yara.Match]]:
     def extract(
         self,
         stream: BinaryIO,
-        matches: List[yara.Match],
         extractor_name: str,
     ) -> Dict[str, Any]:
         """Run extractor with stream and verify output matches the model."""
@@ -127,7 +137,15 @@ def extract(
                 sample_path.write(stream.read())
                 sample_path.flush()
                 # enforce types and verify properties, and remove defaults
-                return _verify_response(utils.run_extractor(sample_path.name, **extractor))
+                return _verify_response(
+                    utils.run_extractor(
+                        sample_path.name,
+                        module_name=extractor["module_name"],
+                        extractor_class=extractor["extractor_class"],
+                        module_path=extractor["module_path"],
+                        venv=extractor["venv"],
+                    )
+                )
         except Exception:
             # caller can deal with the exception
             raise


Test patch:
diff --git a/maco/base_test.py b/maco/base_test.py
index 3c73739..6e72bf0 100644
--- a/maco/base_test.py
+++ b/maco/base_test.py
@@ -46,8 +46,7 @@ def extract(self, stream):
         runs = self.c.match(stream)
         if not runs:
             raise NoHitException("no yara rule hit")
-        hits = runs[self.name]
-        resp = self.c.extract(stream, hits, self.name)
+        resp = self.c.extract(stream, self.name)
         return resp
 
     def _get_location(self) -> str:
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 0000000..6ac74f0
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,20 @@
+import os
+
+import unittest
+from maco import cli
+
+
+class TestCLI(unittest.TestCase):
+    def test_process_filesystem(self):
+        maco_path = os.path.abspath(os.path.join(__file__, "../../demo_extractors"))
+        test_path = os.path.abspath(os.path.join(__file__, "../data"))
+        results = cli.process_filesystem(
+            maco_path,
+            test_path,
+            include=[],
+            exclude=[],
+            pretty=True,
+            force=False,
+            include_base64=False,
+        )
+        self.assertEqual(results, (2, 2, 2))
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index a5d52ac..60be1c4 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -43,11 +43,11 @@ def setUp(self):
 
     def test_analyse_stream(self):
         data = b""
-        resp = self.m.extract(io.BytesIO(data), [], "Complex")
+        resp = self.m.extract(io.BytesIO(data), "Complex")
         self.assertEqual(resp, None)
 
         data = b"data"
-        resp = self.m.extract(io.BytesIO(data), [], "Complex")
+        resp = self.m.extract(io.BytesIO(data), "Complex")
         self.assertEqual(
             resp,
             {