Repository: flyingcircusio/batou

Issue:
buildout 3 test fails due to buildout / pip 21.1 incompatibility
```
Run bin/tox
py36 create: /home/runner/work/batou/batou/.tox/py36
SKIPPED: InterpreterNotFound: python3.6
py37 create: /home/runner/work/batou/batou/.tox/py37
SKIPPED: InterpreterNotFound: python3.7
py38 create: /home/runner/work/batou/batou/.tox/py38
py38 develop-inst: /home/runner/work/batou/batou
py38 installed: apipkg==1.5,attrs==21.2.0,-e git+https://github.com/flyingcircusio/batou@c811fad72083579e945d0c0eddfbdc85393f304a#egg=batou,certifi==2020.12.5,chardet==4.0.0,ConfigUpdater==2.0,coverage==5.5,execnet==1.8.0,idna==2.10,iniconfig==1.1.1,Jinja2==3.0.1,MarkupSafe==2.0.1,mock==4.0.3,packaging==20.9,pluggy==0.13.1,py==1.10.0,pyparsing==2.4.7,pytest==6.2.4,pytest-cov==2.12.0,pytest-cover==3.0.0,pytest-coverage==0.0,pytest-timeout==1.4.2,PyYAML==5.4.1,requests==2.25.1,toml==0.10.2,urllib3==1.26.4
py38 run-test-pre: PYTHONHASHSEED='3322565937'
py38 run-test: commands[0] | pytest src/batou
============================= test session starts ==============================
platform linux -- Python 3.8.10, pytest-6.2.4, py-1.10.0, pluggy-0.13.1
cachedir: .tox/py38/.pytest_cache
rootdir: /home/runner/work/batou/batou, configfile: pytest.ini
plugins: timeout-1.4.2, cov-2.12.0
timeout: 45.0s
timeout method: signal
timeout func_only: False
collected 364 items

src/batou/lib/tests/test_appenv.py ..                                    [  0%]
src/batou/lib/tests/test_archive.py .......s.                            [  3%]
src/batou/lib/tests/test_buildout.py ...F                                [  4%]
src/batou/lib/tests/test_cmmi.py ......                                  [  5%]
src/batou/lib/tests/test_cron.py .....                                   [  7%]
src/batou/lib/tests/test_debian.py ...                                   [  7%]
src/batou/lib/tests/test_download.py .....                               [  9%]
src/batou/lib/tests/test_file.py ....................................... [ 20%]
.........................s..................                             [ 32%]
src/batou/lib/tests/test_git.py ..............                           [ 35%]
src/batou/lib/tests/test_mercurial.py ............                       [ 39%]
src/batou/lib/tests/test_nagios.py .                                     [ 39%]
src/batou/lib/tests/test_python.py ..                                    [ 40%]
src/batou/lib/tests/test_service.py ...                                  [ 40%]
src/batou/lib/tests/test_supervisor.py .......                           [ 42%]
src/batou/lib/tests/test_svn.py .                                        [ 43%]
src/batou/secrets/tests/test_editor.py ..                                [ 43%]
src/batou/secrets/tests/test_secrets.py ............                     [ 46%]
src/batou/tests/test_component.py ...................................... [ 57%]
.......................                                                  [ 63%]
src/batou/tests/test_config.py ........                                  [ 65%]
src/batou/tests/test_dependencies.py ...........                         [ 68%]
src/batou/tests/test_deploy.py .                                         [ 69%]
src/batou/tests/test_endtoend.py ........                                [ 71%]
src/batou/tests/test_environment.py ...........................          [ 78%]
src/batou/tests/test_exceptions.py .                                     [ 79%]
src/batou/tests/test_host.py ...                                         [ 79%]
src/batou/tests/test_remote.py ...                                       [ 80%]
src/batou/tests/test_remote_core.py .................                    [ 85%]
src/batou/tests/test_resources.py ..                                     [ 85%]
src/batou/tests/test_template.py .........                               [ 88%]
src/batou/tests/test_utils.py ........................................   [ 99%]
src/batou/tests/test_vfs.py ..                                           [100%]

=================================== FAILURES ===================================
_______________________ test_runs_buildout3_successfully _______________________
Traceback (most recent call last):
  File "/home/runner/work/batou/batou/src/batou/component.py", line 332, in deploy
    call_with_optional_args(
  File "/home/runner/work/batou/batou/src/batou/utils.py", line 424, in call_with_optional_args
    return func(**call_kw)
  File "/home/runner/work/batou/batou/src/batou/lib/buildout.py", line 72, in verify
    installed.assert_component_is_current([Presence("bin/buildout")] +
  File "/home/runner/work/batou/batou/src/batou/component.py", line 671, in assert_component_is_current
    raise batou.UpdateNeeded()
batou.UpdateNeeded

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/runner/work/batou/batou/src/batou/lib/tests/test_buildout.py", line 71, in test_runs_buildout3_successfully
    root.component.deploy()
  File "/home/runner/work/batou/batou/src/batou/component.py", line 320, in deploy
    sub_component.deploy(predict_only)
  File "/home/runner/work/batou/batou/src/batou/component.py", line 339, in deploy
    self.update()
  File "/home/runner/work/batou/batou/src/batou/lib/buildout.py", line 80, in update
    self.cmd('bin/buildout -t {} -c "{}"'.format(
  File "/home/runner/work/batou/batou/src/batou/component.py", line 776, in cmd
    return batou.utils.cmd(cmd, silent, ignore_returncode, communicate,
  File "/home/runner/work/batou/batou/src/batou/utils.py", line 384, in cmd
    raise CmdExecutionError(cmd, process.returncode, stdout, stderr)
batou.utils.CmdExecutionError: The deployment encountered an error.
----------------------------- Captured stdout call -----------------------------
host > MyComponent > Buildout > File('work/mycomponent/buildout3.cfg') > Presence('buildout3.cfg')
host > MyComponent > Buildout > File('work/mycomponent/buildout3.cfg') > Content('buildout3.cfg')
  buildout3.cfg ---
  buildout3.cfg +++
  buildout3.cfg @@ -0,0 +1,14 @@
  buildout3.cfg +[buildout]
  buildout3.cfg +parts = app
  buildout3.cfg +versions = versions
  buildout3.cfg +allow-picked-versions = false
  buildout3.cfg +
  buildout3.cfg +[app]
  buildout3.cfg +recipe = zc.recipe.egg
  buildout3.cfg +eggs = zc.buildout
  buildout3.cfg +interpreter = py
  buildout3.cfg +
  buildout3.cfg +[versions]
  buildout3.cfg +setuptools = 54.1.1
  buildout3.cfg +zc.buildout = 3.0.0b1
  buildout3.cfg +zc.recipe.egg = 2.0.7
host > MyComponent > Buildout > VirtualEnv('3') > VirtualEnvPy
host > MyComponent > Buildout > VirtualEnv('3') > Package('setuptools==54.1.1')
host > MyComponent > Buildout > VirtualEnv('3') > Package('wheel==0.36.2')
host > MyComponent > Buildout > VirtualEnv('3') > Package('zc.buildout==3.0.0b1')
host > MyComponent > Buildout
ERROR: bin/buildout -t 3 -c "buildout3.cfg"
Return code: 1
STDOUT
Setting socket time out to 3 seconds.
Creating directory '/tmp/pytest-of-runner/pytest-0/test_runs_buildout3_successful0/work/mycomponent/eggs'.
Creating directory '/tmp/pytest-of-runner/pytest-0/test_runs_buildout3_successful0/work/mycomponent/parts'.
Creating directory '/tmp/pytest-of-runner/pytest-0/test_runs_buildout3_successful0/work/mycomponent/develop-eggs'.
Getting distribution for 'zc.recipe.egg==2.0.7'.
An error occurred when trying to install /tmp/tmp6fj07r2tget_dist/zc.recipe.egg-2.0.7.tar.gz. Look above this message for any errors that were output by pip install.

STDERR
Traceback (most recent call last):
  File "<string>", line 1, in <module>
ImportError: cannot import name '_main' from 'pip.__main__' (/tmp/pytest-of-runner/pytest-0/test_runs_buildout3_successful0/work/mycomponent/lib/python3.8/site-packages/pip/__main__.py)


--------- generated xml file: /home/runner/work/batou/batou/report.xml ---------

---------- coverage: platform linux, python 3.8.10-final-0 -----------
Coverage HTML written to dir htmlcov

=========================== short test summary info ============================
FAILED src/batou/lib/tests/test_buildout.py::test_runs_buildout3_successfully
============= 1 failed, 361 passed, 2 skipped in 218.95s (0:03:38) =============
ERROR: InvocationError for command /home/runner/work/batou/batou/.tox/py38/bin/pytest src/batou (exited with code 1)
py39 create: /home/runner/work/batou/batou/.tox/py39
SKIPPED: InterpreterNotFound: python3.9
pre-commit create: /home/runner/work/batou/batou/.tox/pre-commit
SKIPPED: InterpreterNotFound: python3.9
___________________________________ summary ____________________________________
SKIPPED:  py36: InterpreterNotFound: python3.6
SKIPPED:  py37: InterpreterNotFound: python3.7
ERROR:   py38: commands failed
SKIPPED:  py39: InterpreterNotFound: python3.9
SKIPPED:  pre-commit: InterpreterNotFound: python3.9
Error: Process completed with exit code 1.
```

Sounds like `pip`s broken: https://stackoverflow.com/questions/49836676

Comments:

Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b3ab2188..baad4427 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 exclude: '(components/component5/component.py|haproxy.cfg|encrypted.cfg|\.appenv|\.batou|lib/python|examples/.*/environments/.*/secret.*)'
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.1.0
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
@@ -12,7 +12,7 @@ repos:
     -   id: isort
         name: isort (python)
 -   repo: https://github.com/pre-commit/mirrors-yapf
-    rev: 'v0.31.0'  # Use the sha / tag you want to point at
+    rev: 'v0.32.0'  # Use the sha / tag you want to point at
     hooks:
     -   id: yapf
         args: [-i, -p]
diff --git a/CHANGES.md b/CHANGES.md
index 6da4aab9..f07c0ec1 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -10,6 +10,9 @@
   `batou migrate`.
   ([#185](https://github.com/flyingcircusio/batou/issues/185))
 
+- Allow to pin the `pip` version used in the `Buildout` component.
+  ([#263](https://github.com/flyingcircusio/batou/issues/263))
+
 - Automatically migrate environments and secrets to the new structure using
   `./batou migrate`.
   ([#185](https://github.com/flyingcircusio/batou/issues/185))
diff --git a/doc/source/components/python.txt b/doc/source/components/python.txt
index 6c2f988e..917ef6cf 100644
--- a/doc/source/components/python.txt
+++ b/doc/source/components/python.txt
@@ -103,7 +103,8 @@ files makes it necessary. A typical usage example:
 
 .. code-block:: python
 
-    self += Buildout(python='2.7', version='2.2', setuptools='1.0',
+    self += Buildout(python='3.7', version='2.2', setuptools='1.0',
+                     pip='21.1',
                      additional_config=[Directory('profiles', source='profiles')])
 
 .. py:class:: batou.lib.buildout.Buildout()
@@ -129,6 +130,11 @@ files makes it necessary. A typical usage example:
     to the buildout version, e.g. since 2.2 buildout requires setuptools, but
     some versions before that required distribute) (**required**)
 
+.. py:attribute:: pip
+
+    Version of pip to install into the virtualenv (must be appropriate
+    to the buildout version).
+
 .. py:attribute:: distribute
 
     Version of distribute to install into the virtualenv. Mutually exclusive
diff --git a/src/batou/lib/buildout.py b/src/batou/lib/buildout.py
index 274df718..a217f702 100644
--- a/src/batou/lib/buildout.py
+++ b/src/batou/lib/buildout.py
@@ -19,6 +19,7 @@ class Buildout(Component):
     distribute = None
     setuptools = None
     wheel = None
+    pip = None
     version = None
 
     build_env = {}  # XXX not frozen. :/
@@ -58,6 +59,9 @@ class Buildout(Component):
         if self.wheel:
             venv += Package("wheel", version=self.wheel)
 
+        if self.pip:
+            venv += Package("pip", version=self.pip)
+
         # Install without dependencies (that's just setuptools anyway), since
         # that could cause pip to pull in the latest version of setuptools,
         # regardless of the version we wanted to be installed above.


Test patch:
diff --git a/src/batou/lib/tests/test_buildout.py b/src/batou/lib/tests/test_buildout.py
index 2ae6a58d..fd88c8b4 100644
--- a/src/batou/lib/tests/test_buildout.py
+++ b/src/batou/lib/tests/test_buildout.py
@@ -55,9 +55,7 @@ def test_runs_buildout_successfully(root):
         os.path.join(root.environment.workdir_base, "mycomponent/bin/py"))
 
 
-# XFAIL due to https://github.com/flyingcircusio/batou/issues/169
 @pytest.mark.slow
-@pytest.mark.xfail
 @pytest.mark.timeout(60)
 def test_runs_buildout3_successfully(root, output):
     b = Buildout(
@@ -65,6 +63,7 @@ def test_runs_buildout3_successfully(root, output):
         version="3.0.0b1",
         setuptools="54.1.1",
         wheel="0.36.2",
+        pip="21.0.1",
         config=File(
             "buildout3.cfg",
             source=pkg_resources.resource_filename(__name__,