Repository: ansible/ansible-builder

Issue:
ansible-builder introspect --sanitize removes yamllint and ansible-lint from requirements.txt
I want yamllint and ansible-lint to be available inside of my Execution Environment, so I added it to my requirements.txt file, but when the builder step runs `ansible-builder introspect --sanitize...` it strips this out from the resulting requirements.txt and is not available in the resulting EE.

I can work around this by adding a step to the append: section of additional_build_steps: to install it manually
```
    - RUN pip3 install yamllint ansible-lint
```

but this seems to go against the idea of listing all required python modules in the requirements.txt when they are removed unexpectedly by the builder stage

Discussion:
Shrews: Likely need an option to `build` to pass this through.

Patch:
diff --git a/ansible_builder/cli.py b/ansible_builder/cli.py
index 5e67cea..349fe7b 100644
--- a/ansible_builder/cli.py
+++ b/ansible_builder/cli.py
@@ -163,6 +163,7 @@ def add_container_options(parser):
                                    help=('Sanitize and de-duplicate requirements. '
                                          'This is normally done separately from the introspect script, but this '
                                          'option is given to more accurately test collection content.'))
+
     introspect_parser.add_argument(
         'folder', default=base_collections_path, nargs='?',
         help=(
diff --git a/ansible_builder/requirements.py b/ansible_builder/requirements.py
index f615ee6..2c85317 100644
--- a/ansible_builder/requirements.py
+++ b/ansible_builder/requirements.py
@@ -18,9 +18,22 @@ EXCLUDE_REQUIREMENTS = frozenset((
 
 
 def sanitize_requirements(collection_py_reqs):
+    """
+    Cleanup Python requirements by removing duplicates and excluded packages.
+
+    The user requirements file will go through the deduplication process, but
+    skips the special package exclusion process.
+
+    :param dict collection_py_reqs: A dict of lists of Python requirements, keyed
+        by fully qualified collection name. The special key `user` holds requirements
+        from the user specified requirements file from the ``--user-pip`` CLI option.
+
+    :returns: A finalized list of sanitized Python requirements.
+    """
     # de-duplication
     consolidated = []
     seen_pkgs = set()
+
     for collection, lines in collection_py_reqs.items():
         try:
             for req in requirements.parse('\n'.join(lines)):
@@ -38,12 +51,13 @@ def sanitize_requirements(collection_py_reqs):
                 consolidated.append(req)
                 seen_pkgs.add(req.name)
         except Exception as e:
-            logger.warning('Warning: failed to parse requirments from {}, error: {}'.format(collection, e))
+            logger.warning('Warning: failed to parse requirements from {}, error: {}'.format(collection, e))
 
     # removal of unwanted packages
     sanitized = []
     for req in consolidated:
-        if req.name and req.name.lower() in EXCLUDE_REQUIREMENTS:
+        # Exclude packages, unless it was present in the user supplied requirements.
+        if req.name and req.name.lower() in EXCLUDE_REQUIREMENTS and 'user' not in req.collections:
             logger.debug(f'# Excluding requirement {req.name} from {req.collections}')
             continue
         if req.name is None and req.vcs:
diff --git a/docs/collection_metadata.rst b/docs/collection_metadata.rst
index 276c90d..ff3c6ee 100644
--- a/docs/collection_metadata.rst
+++ b/docs/collection_metadata.rst
@@ -26,6 +26,16 @@ The short-hand form of the command is also supported::
 
     ansible-builder introspect --sanitize ~/.ansible/collections/
 
+The ``--sanitize`` option will go through all of the collection requirements and
+remove duplicates, as well as remove some Python requirements that should normally
+be excluded (see :ref:`python_deps` below).
+
+.. note::
+    Use the ``-v3`` option to ``introspect`` to see logging messages about requirements
+    that are being excluded.
+
+.. _python_deps:
+
 Python Dependencies
 ^^^^^^^^^^^^^^^^^^^
 
@@ -42,6 +52,10 @@ not be included in the combined file. These include test packages and
 packages that provide Ansible itself. The full list can be found in
 ``EXCLUDE_REQUIREMENTS`` in the ``ansible_builder.requirements`` module.
 
+Any requirements supplied in the user requirements file, via the ``--user-pip``
+option to the ``introspect`` command, will not be processed against the list of
+excluded Python packages.
+
 System-level Dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 


Test patch:
diff --git a/test/integration/test_introspect_cli.py b/test/integration/test_introspect_cli.py
index 703d24a..7852acd 100644
--- a/test/integration/test_introspect_cli.py
+++ b/test/integration/test_introspect_cli.py
@@ -3,7 +3,6 @@ import yaml
 
 def test_introspect_write(cli, data_dir):
     r = cli(f'ansible-builder introspect {data_dir}')
-    print(r.stdout)
     data = yaml.safe_load(r.stdout)  # assure that output is valid YAML
     assert 'python' in data
     assert 'system' in data
@@ -12,7 +11,6 @@ def test_introspect_write(cli, data_dir):
 
 def test_introspect_with_sanitize(cli, data_dir):
     r = cli(f'ansible-builder introspect --sanitize {data_dir}')
-    print(r.stdout)
     data = yaml.safe_load(r.stdout)  # assure that output is valid YAML
     assert 'python' in data
     assert 'system' in data
@@ -26,7 +24,7 @@ def test_introspect_write_bindep(cli, data_dir, tmp_path):
     assert dest_file.read_text() == '\n'.join([
         'subversion [platform:rpm]  # from collection test.bindep',
         'subversion [platform:dpkg]  # from collection test.bindep',
-        ''
+        '',
     ])
 
 
@@ -39,7 +37,7 @@ def test_introspect_write_python(cli, data_dir, tmp_path):
         'pytz  # from collection test.reqfile',
         'tacacs_plus  # from collection test.reqfile',
         'pyvcloud>=18.0.10  # from collection test.reqfile',
-        ''
+        '',
     ])
 
 
@@ -51,5 +49,21 @@ def test_introspect_write_python_and_sanitize(cli, data_dir, tmp_path):
         'pyvcloud>=14,>=18.0.10  # from collection test.metadata,test.reqfile',
         'pytz  # from collection test.reqfile',
         'tacacs_plus  # from collection test.reqfile',
-        ''
+        '',
     ])
+
+
+def test_introspect_with_sanitize_user_reqs(cli, data_dir, tmp_path):
+    user_file = tmp_path / 'requirements.txt'
+    user_file.write_text("ansible\npytest\n")
+
+    r = cli(f'ansible-builder introspect --sanitize --user-pip={user_file} {data_dir}')
+    data = yaml.safe_load(r.stdout)  # assure that output is valid YAML
+    assert 'python' in data
+    assert 'system' in data
+    assert 'pytz  # from collection test.reqfile' in data['python']
+
+    # 'ansible' allowed in user requirements
+    assert 'ansible  # from collection user' in data['python']
+    # 'pytest' allowed in user requirements
+    assert 'pytest  # from collection user' in data['python']
diff --git a/test/unit/test_requirements.py b/test/unit/test_requirements.py
index 81f6b24..b252de8 100644
--- a/test/unit/test_requirements.py
+++ b/test/unit/test_requirements.py
@@ -34,3 +34,25 @@ def test_skip_bad_formats():
         'bar'
     ], 'foo.bad': ['zizzer zazzer zuzz']  # not okay
     }) == ['foo  # from collection foo.bar', 'bar  # from collection foo.bar']
+
+
+def test_sanitize_requirements_do_not_exclude():
+    py_reqs = {
+        'foo.bar': [
+            'foo',
+            'ansible',   # should not appear
+            'bar',
+        ],
+        'user': [
+            'pytest',    # should appear
+            'bar',
+            'zoo'
+        ]
+    }
+
+    assert sanitize_requirements(py_reqs) == [
+        'foo  # from collection foo.bar',
+        'bar  # from collection foo.bar,user',
+        'pytest  # from collection user',
+        'zoo  # from collection user',
+    ]