Repository: readthedocs/readthedocs.org

Issue:
Auto-wipe when env var is added/deleted
If an environment variable is added from the Admin, it could happen that it doesn't change anything since no file is modified and Sphinx could ignore them.

A way that we can implement to avoid this is by saving the names of the environment variables for a project into our `readthedocs-environment.json` file. By doing that our current checking system for obsolete environment will detect it and auto-wipe the version.


Discussion:
Working on it.
@humitos 
I was working on it and I have some doubts about this.
* Names of the environment variables can be saved in `readthedocs-environment.json` file but for which version?

* > If an environment variable is added from the Admin
    
    Here `Admin` is referred to project admin, i.e. this url - `readthedocs.org/dashboard/<slug>/environmentvariables/` or Django admin?
> Names of the environment variables can be saved in `readthedocs-environment.json` file but for which version?

Each version contains its own `.json` file.

Supposing I build `latest` today with no env vars. It succeed. Immediately after it finished, I add a env var from the admin and build `latest` again. At this point, RTD should check the env vars in the `.json` file for `latest` version and if they are different wipe the environment before building.

Consider also saving not only the names but a hash of the values also --or maybe even better just _one_ hash of all the names together with the values. We don't really need to know the names nor the values, we just want to detect if any of them has changed.

> Here `Admin` is referred to project admin, i.e. this url - `readthedocs.org/dashboard/<slug>/environmentvariables/` or Django admin?

The first URL you mentioned. Project's admin accessible by a regular user.

Resolving patch:
diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py
--- a/readthedocs/doc_builder/python_environments.py
+++ b/readthedocs/doc_builder/python_environments.py
@@ -3,6 +3,7 @@
 """An abstraction over virtualenv and Conda environments."""
 
 import copy
+import hashlib
 import itertools
 import json
 import logging
@@ -149,6 +150,7 @@ def is_obsolete(self):
         * the Python version (e.g. 2.7, 3, 3.6, etc)
         * the Docker image name
         * the Docker image hash
+        * the environment variables hash
 
         :returns: ``True`` when it's obsolete and ``False`` otherwise
 
@@ -174,6 +176,7 @@ def is_obsolete(self):
 
         env_python = environment_conf.get('python', {})
         env_build = environment_conf.get('build', {})
+        env_vars_hash = environment_conf.get('env_vars_hash', None)
 
         # By defaulting non-existent options to ``None`` we force a wipe since
         # we don't know how the environment was created
@@ -197,8 +200,27 @@ def is_obsolete(self):
             env_python_version != self.config.python_full_version,
             env_build_image != build_image,
             env_build_hash != image_hash,
+            env_vars_hash != self._get_env_vars_hash(),
         ])
 
+    def _get_env_vars(self):
+        """Return env vars with their values of the project."""
+        env_vars = self.version.project.environmentvariable_set.values_list('name', 'value')
+        return env_vars
+
+    def _get_env_vars_hash(self):
+        """
+        Returns the sha256 hash of all the environment variables and their values.
+
+        If there are no environment variables configured for the associated project,
+        it returns sha256 hash of empty string.
+        """
+        m = hashlib.sha256()
+        for variable, value in self._get_env_vars():
+            hash_str = f'_{variable}_{value}_'
+            m.update(hash_str.encode('utf-8'))
+        return m.hexdigest()
+
     def save_environment_json(self):
         """
         Save on builders disk data about the environment used to build docs.
@@ -208,11 +230,13 @@ def save_environment_json(self):
         - python.version
         - build.image
         - build.hash
+        - env_vars_hash
         """
         data = {
             'python': {
                 'version': self.config.python_full_version,
             },
+            'env_vars_hash': self._get_env_vars_hash(),
         }
 
         if isinstance(self.build_env, DockerBuildEnvironment):


Test patch:
diff --git a/readthedocs/rtd_tests/tests/test_doc_building.py b/readthedocs/rtd_tests/tests/test_doc_building.py
--- a/readthedocs/rtd_tests/tests/test_doc_building.py
+++ b/readthedocs/rtd_tests/tests/test_doc_building.py
@@ -5,6 +5,7 @@
 * raw subprocess calls like .communicate expects bytes
 * the Command wrappers encapsulate the bytes and expose unicode
 """
+import hashlib
 import json
 import os
 import re
@@ -30,7 +31,7 @@
 )
 from readthedocs.doc_builder.exceptions import BuildEnvironmentError
 from readthedocs.doc_builder.python_environments import Conda, Virtualenv
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, EnvironmentVariable
 from readthedocs.rtd_tests.mocks.environment import EnvironmentMockGroup
 from readthedocs.rtd_tests.mocks.paths import fake_paths_lookup
 from readthedocs.rtd_tests.tests.test_config_integration import create_load
@@ -1469,6 +1470,13 @@ def test_save_environment_json(self, load_config):
             config=config,
         )
 
+        self.assertFalse(self.pip.environmentvariable_set.all().exists())
+        get(EnvironmentVariable, project=self.pip, name='ABCD', value='1234')
+        env_var_str = '_ABCD_1234_'
+        m = hashlib.sha256()
+        m.update(env_var_str.encode('utf-8'))
+        env_vars_hash = m.hexdigest()
+
         with patch(
                 'readthedocs.doc_builder.python_environments.PythonEnvironment.environment_json_path',
                 return_value=tempfile.mktemp(suffix='envjson'),
@@ -1476,6 +1484,7 @@ def test_save_environment_json(self, load_config):
             python_env.save_environment_json()
             json_data = json.load(open(python_env.environment_json_path()))
 
+        
         expected_data = {
             'build': {
                 'image': 'readthedocs/build:2.0',
@@ -1484,6 +1493,7 @@ def test_save_environment_json(self, load_config):
             'python': {
                 'version': 2.7,
             },
+            'env_vars_hash': env_vars_hash
         }
         self.assertDictEqual(json_data, expected_data)
 
@@ -1610,7 +1620,15 @@ def test_is_obsolete_with_json_same_data_as_version(self, load_config):
             build_env=self.build_env,
             config=config,
         )
-        env_json_data = '{"build": {"image": "readthedocs/build:2.0", "hash": "a1b2c3"}, "python": {"version": 3.5}}'  # noqa
+
+        self.assertFalse(self.pip.environmentvariable_set.all().exists())
+        get(EnvironmentVariable, project=self.pip, name='HELLO', value='WORLD')
+        env_var_str = '_HELLO_WORLD_'
+        m = hashlib.sha256()
+        m.update(env_var_str.encode('utf-8'))
+        env_vars_hash = m.hexdigest()
+
+        env_json_data = '{{"build": {{"image": "readthedocs/build:2.0", "hash": "a1b2c3"}}, "python": {{"version": 3.5}}, "env_vars_hash": "{}"}}'.format(env_vars_hash)  # noqa
         with patch('os.path.exists') as exists, patch('readthedocs.doc_builder.python_environments.open', mock_open(read_data=env_json_data)) as _open:  # noqa
             exists.return_value = True
             self.assertFalse(python_env.is_obsolete)