Repository: zulip/zulip

Issue:
Improve virtualenv-clone hashing logic
Our `virtualenv` management logic is designed to have a few properties:
* After running `provision` or the equivalent production operation, you always have a correct virtualenv as though you'd built it fresh
* We avoid the performance pain of redownloading and/or rebuilding all of our packages fresh for the common operations of "adding a new package" or "upgrading a minor version of a package", both in the development environment and in production.
* https://zulip.readthedocs.io/en/latest/subsystems/dependencies.html is a useful reference.

There are a few issues we need to fix with the current implementation:
* We don't currently compare the `Python` version when deciding whether we need to build a new virtualenv from scratch.  
* Our index of packages included in the virtualenv doesn't have package versions; we should include those and only allow package upgrades to use the `virtualenv-clone` plus cache approach (downgrades are often not tested upstream).

We should be able to test whether the implementation is working manually through inspection of `/srv/zulip-venv-cache/` before and after adding a new python dependency (see the docs for how to do this using `update-locked-requirements`), downgrading in `dev.txt`, and upgrading/downgrading the Python version itself.  We don't currently have unit tests of any this logic, though it wouldn't be a bad idea to write some simple tests of the logic functions from `scripts/lib/setup_venv.py` in `tools/tests/` somewhere (`tools/test-tools` is the runner for those).

@andersk FYI; we might want to fix this before we merge https://github.com/zulip/zulip/pull/12837.


Discussion:
Hello @zulip/server-production, @zulip/server-tooling members, this issue was labeled with the "area: production", "area: tooling" labels, so you may want to check it out!

<!-- areaLabelAddition -->

Resolving patch:
diff --git a/scripts/lib/hash_reqs.py b/scripts/lib/hash_reqs.py
--- a/scripts/lib/hash_reqs.py
+++ b/scripts/lib/hash_reqs.py
@@ -2,29 +2,20 @@
 import argparse
 import hashlib
 import os
+import subprocess
 import sys
-from typing import Iterable, List, MutableSet
+from typing import Iterable, List
 
 
-def expand_reqs_helper(fpath: str, visited: MutableSet[str]) -> List[str]:
-    if fpath in visited:
-        return []
-    else:
-        visited.add(fpath)
-
-    curr_dir = os.path.dirname(fpath)
+def expand_reqs_helper(fpath: str) -> List[str]:
     result = []  # type: List[str]
 
     for line in open(fpath):
-        if line.startswith('#'):
+        if line.strip().startswith(('#', '--hash')):
             continue
-        dep = line.split(" #", 1)[0].strip()  # remove comments and strip whitespace
+        dep = line.split(" \\", 1)[0].strip()
         if dep:
-            if dep.startswith('-r'):
-                child = os.path.join(curr_dir, dep[3:])
-                result += expand_reqs_helper(child, visited)
-            else:
-                result.append(dep)
+            result.append(dep)
     return result
 
 def expand_reqs(fpath: str) -> List[str]:
@@ -34,11 +25,17 @@ def expand_reqs(fpath: str) -> List[str]:
     `fpath` can be either an absolute path or a relative path.
     """
     absfpath = os.path.abspath(fpath)
-    output = expand_reqs_helper(absfpath, set())
+    output = expand_reqs_helper(absfpath)
     return sorted(set(output))
 
+def python_version() -> str:
+    """
+    Returns the Python version as string 'Python major.minor.patchlevel'
+    """
+    return subprocess.check_output(["/usr/bin/python3", "-VV"], universal_newlines=True)
+
 def hash_deps(deps: Iterable[str]) -> str:
-    deps_str = "\n".join(deps) + "\n"
+    deps_str = "\n".join(deps) + "\n" + python_version()
     return hashlib.sha1(deps_str.encode('utf-8')).hexdigest()
 
 def main() -> int:
diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py
--- a/scripts/lib/setup_venv.py
+++ b/scripts/lib/setup_venv.py
@@ -4,7 +4,7 @@
 import subprocess
 from typing import List, Optional, Set, Tuple
 
-from scripts.lib.hash_reqs import expand_reqs
+from scripts.lib.hash_reqs import expand_reqs, python_version
 from scripts.lib.zulip_tools import ENDC, WARNING, os_families, run, run_as_root
 
 ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -163,6 +163,7 @@ def try_to_copy_venv(venv_path: str, new_packages: Set[str]) -> bool:
     if not os.path.exists(VENV_CACHE_PATH):
         return False
 
+    desired_python_version = python_version()
     venv_name = os.path.basename(venv_path)
 
     overlaps = []  # type: List[Tuple[int, str, Set[str]]]
@@ -173,6 +174,15 @@ def try_to_copy_venv(venv_path: str, new_packages: Set[str]) -> bool:
                 not os.path.exists(get_index_filename(curr_venv_path))):
             continue
 
+        # Check the Python version in the venv matches the version we want to use.
+        venv_python3 = os.path.join(curr_venv_path, "bin", "python3")
+        if not os.path.exists(venv_python3):
+            continue
+        venv_python_version = subprocess.check_output([
+            venv_python3, "-VV"], universal_newlines=True)
+        if desired_python_version != venv_python_version:
+            continue
+
         old_packages = get_venv_packages(curr_venv_path)
         # We only consider using using old virtualenvs that only
         # contain packages that we want in our new virtualenv.
@@ -262,16 +272,19 @@ def do_patch_activate_script(venv_path: str) -> None:
     with open(script_path, 'w') as f:
         f.write("".join(lines))
 
+def generate_hash(requirements_file: str) -> str:
+    path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
+    output = subprocess.check_output([path, requirements_file], universal_newlines=True)
+    return output.split()[0]
+
 def setup_virtualenv(
     target_venv_path: Optional[str],
     requirements_file: str,
     patch_activate_script: bool = False,
 ) -> str:
 
+    sha1sum = generate_hash(requirements_file)
     # Check if a cached version already exists
-    path = os.path.join(ZULIP_PATH, 'scripts', 'lib', 'hash_reqs.py')
-    output = subprocess.check_output([path, requirements_file], universal_newlines=True)
-    sha1sum = output.split()[0]
     if target_venv_path is None:
         cached_venv_path = os.path.join(VENV_CACHE_PATH, sha1sum, 'venv')
     else:


Test patch:
diff --git a/tools/tests/test_hash_reqs.py b/tools/tests/test_hash_reqs.py
new file mode 100644
--- /dev/null
+++ b/tools/tests/test_hash_reqs.py
@@ -0,0 +1,25 @@
+import unittest
+
+import mock
+
+from scripts.lib.hash_reqs import expand_reqs, hash_deps
+from tools.setup.setup_venvs import DEV_REQS_FILE
+
+
+class TestHashCreation(unittest.TestCase):
+
+    def test_diff_hash_for_diff_python_version(self) -> None:
+        with mock.patch('scripts.lib.hash_reqs.python_version', return_value='Python 3.6.9'):
+            deps = expand_reqs(DEV_REQS_FILE)
+            hash1 = hash_deps(deps)
+
+        with mock.patch('scripts.lib.hash_reqs.python_version', return_value='Python 3.6.9'):
+            deps = expand_reqs(DEV_REQS_FILE)
+            hash2 = hash_deps(deps)
+
+        with mock.patch('scripts.lib.hash_reqs.python_version', return_value='Python 3.8.2'):
+            deps = expand_reqs(DEV_REQS_FILE)
+            hash3 = hash_deps(deps)
+
+        assert hash1 == hash2
+        assert hash1 != hash3