Repository: readthedocs/readthedocs.org
Issue: GitHub edit link is aggressively stripped

Hi 

I'm trying to override the value of `github_repo` which is used to create the 'edit in git hub' link on the doco site. In my case this value isn't the default (which appears to be that of the RTD project name).

I've set `html_context` in my `conf.py` to override `github_repo` however I see in the build logs that RTD seems to extend this and blow away my changes:

``` python

# my config

html_context = {
    'github_repo': "esp.github.io",
}


... later in the same script ... 

###########################################################################
#          auto-created readthedocs.org specific configuration            #
###########################################################################


#
# The following code was added during an automated build on readthedocs.org
# It is auto created and injected for every build. The result is based on the
# conf.py.tmpl file found in the readthedocs.org codebase:
# https://github.com/rtfd/readthedocs.org/blob/master/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl
#

... later in the same script ... 

#Add project information to the template context.
context = {
    'using_theme': using_rtd_theme,
    'html_theme': html_theme,
    'current_version': "latest",
    'MEDIA_URL': "https://media.readthedocs.org/",
    'PRODUCTION_DOMAIN': "readthedocs.org",
    'versions': [
    ("latest", "/en/latest/"),
    ],
    'downloads': [ 
    ("pdf", "//readthedocs.org/projects/esp/downloads/pdf/latest/"),
    ("htmlzip", "//readthedocs.org/projects/esp/downloads/htmlzip/latest/"),
    ("epub", "//readthedocs.org/projects/esp/downloads/epub/latest/"),
    ],
    'slug': 'esp',
    'name': u'Evented State Processor (ESP)',
    'rtd_language': u'en',
    'canonical_url': 'http://esp.readthedocs.org/en/latest/',
    'analytics_code': '',
    'single_version': False,
    'conf_py_path': '/./',
    'api_host': 'https://readthedocs.org/',
    'github_user': 'esp',
    'github_repo': 'esp',
    'github_version': 'master',
    'display_github': True,
    'bitbucket_user': 'None',
    'bitbucket_repo': 'None',
    'bitbucket_version': 'master',
    'display_bitbucket': False,
    'READTHEDOCS': True,
    'using_theme': (html_theme == "default"),
    'new_theme': (html_theme == "sphinx_rtd_theme"),
    'source_suffix': SUFFIX,
    'user_analytics_code': '',
    'global_analytics_code': 'UA-17997319-1',

    'commit': '98303330',

}
if 'html_context' in globals():
    html_context.update(context)
else:
    html_context = context
```

I need to override this value as I'm using RTD to host a single doco site for 2 projects ([1](https://github.com/esp/esp-js),[2](https://github.com/esp/esp-net)).  
I need `github_repo` to be `esp.github.io` so it correctly points to the [doco src](https://github.com/esp/esp.github.io).

I think the fix to in rtfd's [conf.py](https://github.com/rtfd/readthedocs.org/blob/master/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl#L152-L155) may be something like this (assuming it's safe to replace html_context):

``` python
if 'html_context' in globals():
    context.update(html_context)

html_context = context

```

Possibly related to #743

Appreciate any help or suggestions on how to fix the link this

Keith


Changed files:
readthedocs/builds/models.py, readthedocs/rtd_tests/tests/test_repo_parsing.py, readthedocs/builds/utils.py, readthedocs/projects/constants.py

diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py
index e97f6002b..58f85321a 100644
--- a/readthedocs/builds/models.py
+++ b/readthedocs/builds/models.py
@@ -21,8 +21,7 @@ from taggit.managers import TaggableManager
 
 from readthedocs.core.utils import broadcast
 from readthedocs.projects.constants import (
-    BITBUCKET_REGEXS, BITBUCKET_URL, GITHUB_REGEXS, GITHUB_URL, GITLAB_REGEXS,
-    GITLAB_URL, PRIVACY_CHOICES, PRIVATE)
+    BITBUCKET_URL, GITHUB_URL, GITLAB_URL, PRIVACY_CHOICES, PRIVATE)
 from readthedocs.projects.models import APIProject, Project
 
 from .constants import (
@@ -30,6 +29,9 @@ from .constants import (
     NON_REPOSITORY_VERSIONS, STABLE, TAG, VERSION_TYPES)
 from .managers import VersionManager
 from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet
+from .utils import (
+    get_bitbucket_username_repo, get_github_username_repo,
+    get_gitlab_username_repo)
 from .version_slug import VersionSlugField
 
 DEFAULT_VERSION_PRIVACY_LEVEL = getattr(
@@ -277,12 +279,8 @@ class Version(models.Model):
         elif action == 'edit':
             action_string = 'edit'
 
-        for regex in GITHUB_REGEXS:
-            match = regex.search(repo_url)
-            if match:
-                user, repo = match.groups()
-                break
-        else:
+        user, repo = get_github_username_repo(repo_url)
+        if not user and not repo:
             return ''
         repo = repo.rstrip('/')
 
@@ -315,12 +313,8 @@ class Version(models.Model):
         elif action == 'edit':
             action_string = 'edit'
 
-        for regex in GITLAB_REGEXS:
-            match = regex.search(repo_url)
-            if match:
-                user, repo = match.groups()
-                break
-        else:
+        user, repo = get_gitlab_username_repo(repo_url)
+        if not user and not repo:
             return ''
         repo = repo.rstrip('/')
 
@@ -341,12 +335,8 @@ class Version(models.Model):
         if not docroot:
             return ''
 
-        for regex in BITBUCKET_REGEXS:
-            match = regex.search(repo_url)
-            if match:
-                user, repo = match.groups()
-                break
-        else:
+        user, repo = get_bitbucket_username_repo(repo_url)
+        if not user and not repo:
             return ''
         repo = repo.rstrip('/')
 
diff --git a/readthedocs/builds/utils.py b/readthedocs/builds/utils.py
index 1457a123c..7fcc245db 100644
--- a/readthedocs/builds/utils.py
+++ b/readthedocs/builds/utils.py
@@ -4,32 +4,13 @@
 from __future__ import (
     absolute_import, division, print_function, unicode_literals)
 
-import re
-
-GH_REGEXS = [
-    re.compile('github.com/(.+)/(.+)(?:\\.git){1}'),
-    re.compile('github.com/(.+)/(.+)'),
-    re.compile('github.com:(.+)/(.+).git'),
-]
-
-BB_REGEXS = [
-    re.compile('bitbucket.org/(.+)/(.+)/'),
-    re.compile('bitbucket.org/(.+)/(.+)'),
-    re.compile('bitbucket.org:(.+)/(.+)\\.git'),
-]
-
-# TODO: I think this can be different than `gitlab.com`
-# self.adapter.provider_base_url
-GL_REGEXS = [
-    re.compile('gitlab.com/(.+)/(.+)(?:\\.git){1}'),
-    re.compile('gitlab.com/(.+)/(.+)'),
-    re.compile('gitlab.com:(.+)/(.+)\\.git'),
-]
+from readthedocs.projects.constants import (
+    BITBUCKET_REGEXS, GITHUB_REGEXS, GITLAB_REGEXS)
 
 
 def get_github_username_repo(url):
     if 'github' in url:
-        for regex in GH_REGEXS:
+        for regex in GITHUB_REGEXS:
             match = regex.search(url)
             if match:
                 return match.groups()
@@ -38,7 +19,7 @@ def get_github_username_repo(url):
 
 def get_bitbucket_username_repo(url=None):
     if 'bitbucket' in url:
-        for regex in BB_REGEXS:
+        for regex in BITBUCKET_REGEXS:
             match = regex.search(url)
             if match:
                 return match.groups()
@@ -47,7 +28,7 @@ def get_bitbucket_username_repo(url=None):
 
 def get_gitlab_username_repo(url=None):
     if 'gitlab' in url:
-        for regex in GL_REGEXS:
+        for regex in GITLAB_REGEXS:
             match = regex.search(url)
             if match:
                 return match.groups()
diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py
index 897126239..8837f13b0 100644
--- a/readthedocs/projects/constants.py
+++ b/readthedocs/projects/constants.py
@@ -292,19 +292,20 @@ PROJECT_PK_REGEX = '(?:[-\\w]+)'
 PROJECT_SLUG_REGEX = '(?:[-\\w]+)'
 
 GITHUB_REGEXS = [
-    re.compile('github.com/(.+)/(.+)(?:\\.git){1}'),
+    re.compile('github.com/(.+)/(.+)(?:\\.git){1}$'),
     re.compile('github.com/(.+)/(.+)'),
-    re.compile('github.com:(.+)/(.+).git'),
+    re.compile('github.com:(.+)/(.+)\\.git$'),
 ]
 BITBUCKET_REGEXS = [
-    re.compile('bitbucket.org/(.+)/(.+).git'),
+    re.compile('@bitbucket.org/(.+)/(.+)\\.git$'),
     re.compile('bitbucket.org/(.+)/(.+)/'),
     re.compile('bitbucket.org/(.+)/(.+)'),
+    re.compile('bitbucket.org:(.+)/(.+)\\.git$'),
 ]
 GITLAB_REGEXS = [
-    re.compile('gitlab.com/(.+)/(.+)(?:\\.git){1}'),
+    re.compile('gitlab.com/(.+)/(.+)(?:\\.git){1}$'),
     re.compile('gitlab.com/(.+)/(.+)'),
-    re.compile('gitlab.com:(.+)/(.+).git'),
+    re.compile('gitlab.com:(.+)/(.+)\\.git$'),
 ]
 GITHUB_URL = (
     'https://github.com/{user}/{repo}/'
diff --git a/readthedocs/rtd_tests/tests/test_repo_parsing.py b/readthedocs/rtd_tests/tests/test_repo_parsing.py
index 33e4ac212..43231d680 100644
--- a/readthedocs/rtd_tests/tests/test_repo_parsing.py
+++ b/readthedocs/rtd_tests/tests/test_repo_parsing.py
@@ -22,9 +22,28 @@ class TestRepoParsing(TestCase):
         self.pip.repo = 'https://github.com/user/repo/'
         self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo/blob/master/docs/file.rst')
 
+        self.pip.repo = 'https://github.com/user/repo.github.io'
+        self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst')
+
+        self.pip.repo = 'https://github.com/user/repo.github.io/'
+        self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst')
+
         self.pip.repo = 'https://github.com/user/repo.git'
         self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo/blob/master/docs/file.rst')
 
+        self.pip.repo = 'https://github.com/user/repo.github.io.git'
+        self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst')
+
+        self.pip.repo = 'https://github.com/user/repo.git.git'
+        self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.git/blob/master/docs/file.rst')
+
+    def test_github_ssh(self):
+        self.pip.repo = 'git@github.com:user/repo.git'
+        self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo/blob/master/docs/file.rst')
+
+        self.pip.repo = 'git@github.com:user/repo.github.io.git'
+        self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst')
+
     def test_gitlab(self):
         self.pip.repo = 'https://gitlab.com/user/repo'
         self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst')
@@ -32,9 +51,28 @@ class TestRepoParsing(TestCase):
         self.pip.repo = 'https://gitlab.com/user/repo/'
         self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst')
 
+        self.pip.repo = 'https://gitlab.com/user/repo.gitlab.io'
+        self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst')
+
+        self.pip.repo = 'https://gitlab.com/user/repo.gitlab.io/'
+        self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst')
+
         self.pip.repo = 'https://gitlab.com/user/repo.git'
         self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst')
 
+        self.pip.repo = 'https://gitlab.com/user/repo.gitlab.io.git'
+        self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst')
+
+        self.pip.repo = 'https://gitlab.com/user/repo.git.git'
+        self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.git/blob/master/foo/bar/file.rst')
+
+    def test_gitlab_ssh(self):
+        self.pip.repo = 'git@gitlab.com:user/repo.git'
+        self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst')
+
+        self.pip.repo = 'git@gitlab.com:user/repo.gitlab.io.git'
+        self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst')
+
     def test_bitbucket(self):
         self.pip.repo = 'https://bitbucket.org/user/repo'
         self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst')
@@ -42,6 +80,31 @@ class TestRepoParsing(TestCase):
         self.pip.repo = 'https://bitbucket.org/user/repo/'
         self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst')
 
+        self.pip.repo = 'https://bitbucket.org/user/repo.gitbucket.io'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst')
+
+        self.pip.repo = 'https://bitbucket.org/user/repo.gitbucket.io/'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst')
+
         self.pip.repo = 'https://bitbucket.org/user/repo.git'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.git/src/master/foo/bar/file.rst')
+
+        self.pip.repo = 'https://bitbucket.org/user/repo.gitbucket.io.git'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io.git/src/master/foo/bar/file.rst')
+
+        self.pip.repo = 'https://bitbucket.org/user/repo.git.git'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.git.git/src/master/foo/bar/file.rst')
+
+    def test_bitbucket_https(self):
+        self.pip.repo = 'https://user@bitbucket.org/user/repo.git'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst')
+
+        self.pip.repo = 'https://user@bitbucket.org/user/repo.gitbucket.io.git'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst')
+
+    def test_bitbucket_ssh(self):
+        self.pip.repo = 'git@bitbucket.org:user/repo.git'
         self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst')
 
+        self.pip.repo = 'git@bitbucket.org:user/repo.gitbucket.io.git'
+        self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst')