Repository: getsentry/sentry

Issue:
feat(gitlab): Add support for integrating projects from entire GitLab instances
This PR is meant to follow after #15178.

Make the *GitLab Group Path* field optional when installing a GitLab integration. If the path is empty, then the integration will allow projects from the entire instance to be added, not just projects within a specific group. The installation's metadata will have `None` as the group id.

If the path is empty, the *Include Subgroups* option will have no effect. There's some help text indicating this.

The sort ordering for projects is also changed to sort by last activity, instead of date created.

Here's a screenshot with the UI changes on the installation form highlighted.

![Screen Shot 2019-10-18 at 10 07 53 PM](https://user-images.githubusercontent.com/27340/67137532-29604500-f1f4-11e9-8013-c741a98ff95d.png)

Closes #12774 and closes #14799.


Discussion:

Resolving patch:
diff --git a/src/sentry/integrations/gitlab/client.py b/src/sentry/integrations/gitlab/client.py
--- a/src/sentry/integrations/gitlab/client.py
+++ b/src/sentry/integrations/gitlab/client.py
@@ -30,6 +30,7 @@ class GitLabApiClientPath:
     project_hooks = "/projects/{project}/hooks"
     project_hook = "/projects/{project}/hooks/{hook_id}"
     project_search = "/projects/{project}/search"
+    projects = "/projects"
     user = "/user"
 
     @staticmethod
@@ -134,31 +135,40 @@ def get_user(self):
         """
         return self.get(GitLabApiClientPath.user)
 
-    def search_group_projects(self, group, query=None, simple=True):
-        """Get projects for a group
+    def search_projects(self, group=None, query=None, simple=True):
+        """Get projects
 
         See https://docs.gitlab.com/ee/api/groups.html#list-a-group-s-projects
+        and https://docs.gitlab.com/ee/api/projects.html#list-all-projects
         """
 
         def gen_params(page_number, page_size):
-            # simple param returns limited fields for the project.
+            # Simple param returns limited fields for the project.
             # Really useful, because we often don't need most of the project information
-            return {
+            params = {
                 "search": query,
                 "simple": simple,
-                "include_subgroups": self.metadata.get("include_subgroups", False),
+                "order_by": "last_activity_at",
                 "page": page_number + 1,  # page starts at 1
                 "per_page": page_size,
             }
+            if group:
+                extra_params = {"include_subgroups": self.metadata.get("include_subgroups", False)}
+            else:
+                extra_params = {"membership": True}
+
+            params.update(extra_params)
+            return params
 
         def get_results(resp):
             return resp
 
-        return self.get_with_pagination(
-            GitLabApiClientPath.group_projects.format(group=group),
-            gen_params=gen_params,
-            get_results=get_results,
-        )
+        if group:
+            path = GitLabApiClientPath.group_projects.format(group=group)
+        else:
+            path = GitLabApiClientPath.projects
+
+        return self.get_with_pagination(path, gen_params, get_results)
 
     def get_project(self, project_id):
         """Get project
diff --git a/src/sentry/integrations/gitlab/integration.py b/src/sentry/integrations/gitlab/integration.py
--- a/src/sentry/integrations/gitlab/integration.py
+++ b/src/sentry/integrations/gitlab/integration.py
@@ -112,7 +112,7 @@ def get_codeowner_file(self, repo, ref=None):
     def get_repositories(self, query=None):
         # Note: gitlab projects are the same things as repos everywhere else
         group = self.get_group_id()
-        resp = self.get_client().search_group_projects(group, query)
+        resp = self.get_client().search_projects(group, query)
         return [{"identifier": repo["id"], "name": repo["name_with_namespace"]} for repo in resp]
 
     def format_source_url(self, repo: Repository, filepath: str, branch: str) -> str:
@@ -126,7 +126,7 @@ def format_source_url(self, repo: Repository, filepath: str, branch: str) -> str
     def search_projects(self, query):
         client = self.get_client()
         group_id = self.get_group_id()
-        return client.search_group_projects(group_id, query)
+        return client.search_projects(group_id, query)
 
     def search_issues(self, project_id, query, iids):
         client = self.get_client()
@@ -151,7 +151,7 @@ class InstallationForm(forms.Form):
         label=_("GitLab URL"),
         help_text=_(
             "The base URL for your GitLab instance, including the host and protocol. "
-            "Do not include group path."
+            "Do not include the group path."
             "<br>"
             "If using gitlab.com, enter https://gitlab.com/"
         ),
@@ -160,16 +160,26 @@ class InstallationForm(forms.Form):
     group = forms.CharField(
         label=_("GitLab Group Path"),
         help_text=_(
-            "This can be found in the URL of your group's GitLab page. "
+            "This can be found in the URL of your group's GitLab page."
             "<br>"
-            "For example, if your group can be found at "
+            "For example, if your group URL is "
             "https://gitlab.com/my-group/my-subgroup, enter `my-group/my-subgroup`."
+            "<br>"
+            "If you are trying to integrate an entire self-managed GitLab instance, "
+            "leave this empty. Doing so will also allow you to select projects in "
+            "all group and user namespaces (such as users' personal repositories and forks)."
         ),
         widget=forms.TextInput(attrs={"placeholder": _("my-group/my-subgroup")}),
+        required=False,
     )
     include_subgroups = forms.BooleanField(
         label=_("Include Subgroups"),
-        help_text=_("Include projects in subgroups of the GitLab group."),
+        help_text=_(
+            "Include projects in subgroups of the GitLab group."
+            "<br>"
+            "Not applicable when integrating an entire GitLab instance. "
+            "All groups are included for instance-level integrations."
+        ),
         widget=forms.CheckboxInput(),
         required=False,
         initial=False,
@@ -343,11 +353,16 @@ def build_integration(self, state):
         }
 
         user = get_user_info(data["access_token"], state["installation_data"])
-        group = self.get_group_info(data["access_token"], state["installation_data"])
-        include_subgroups = state["installation_data"]["include_subgroups"]
         scopes = sorted(GitlabIdentityProvider.oauth_scopes)
         base_url = state["installation_data"]["url"]
 
+        if state["installation_data"].get("group"):
+            group = self.get_group_info(data["access_token"], state["installation_data"])
+            include_subgroups = state["installation_data"]["include_subgroups"]
+        else:
+            group = {}
+            include_subgroups = False
+
         hostname = urlparse(base_url).netloc
         verify_ssl = state["installation_data"]["verify_ssl"]
 
@@ -357,22 +372,22 @@ def build_integration(self, state):
         secret = sha1_text("".join([hostname, state["installation_data"]["client_id"]]))
 
         integration = {
-            "name": group["full_name"],
+            "name": group.get("full_name", hostname),
             # Splice the gitlab host and project together to
             # act as unique link between a gitlab instance, group + sentry.
             # This value is embedded then in the webhook token that we
             # give to gitlab to allow us to find the integration a hook came
             # from.
-            "external_id": "{}:{}".format(hostname, group["id"]),
+            "external_id": "{}:{}".format(hostname, group.get("id", "_instance_")),
             "metadata": {
-                "icon": group["avatar_url"],
+                "icon": group.get("avatar_url"),
                 "instance": hostname,
-                "domain_name": "{}/{}".format(hostname, group["full_path"]),
+                "domain_name": "{}/{}".format(hostname, group.get("full_path", "")).rstrip("/"),
                 "scopes": scopes,
                 "verify_ssl": verify_ssl,
                 "base_url": base_url,
                 "webhook_secret": secret.hexdigest(),
-                "group_id": group["id"],
+                "group_id": group.get("id"),
                 "include_subgroups": include_subgroups,
             },
             "user_identity": {


Test patch:
diff --git a/tests/sentry/integrations/gitlab/test_integration.py b/tests/sentry/integrations/gitlab/test_integration.py
--- a/tests/sentry/integrations/gitlab/test_integration.py
+++ b/tests/sentry/integrations/gitlab/test_integration.py
@@ -330,3 +330,122 @@ def test_get_stacktrace_link_use_default_if_version_404(self):
         assert (
             source_url == "https://gitlab.example.com/getsentry/example-repo/blob/master/README.md"
         )
+
+
+class GitlabIntegrationInstanceTest(IntegrationTestCase):
+    provider = GitlabIntegrationProvider
+    config = {
+        # Trailing slash is intentional to ensure that valid
+        # URLs are generated even if the user inputs a trailing /
+        "url": "https://gitlab.example.com/",
+        "name": "Test App",
+        "group": "",
+        "verify_ssl": True,
+        "client_id": "client_id",
+        "client_secret": "client_secret",
+        "include_subgroups": True,
+    }
+
+    def setUp(self):
+        super().setUp()
+        self.init_path_without_guide = f"{self.init_path}?completed_installation_guide"
+
+    def assert_setup_flow(self, user_id="user_id_1"):
+        resp = self.client.get(self.init_path)
+        assert resp.status_code == 200
+        self.assertContains(resp, "you will need to create a Sentry app in your GitLab instance")
+        resp = self.client.get(self.init_path_without_guide)
+        assert resp.status_code == 200
+        resp = self.client.post(self.init_path_without_guide, data=self.config)
+        assert resp.status_code == 302
+        redirect = urlparse(resp["Location"])
+        assert redirect.scheme == "https"
+        assert redirect.netloc == "gitlab.example.com"
+        assert redirect.path == "/oauth/authorize"
+
+        params = parse_qs(redirect.query)
+        assert params["state"]
+        assert params["redirect_uri"] == ["http://testserver/extensions/gitlab/setup/"]
+        assert params["response_type"] == ["code"]
+        assert params["client_id"] == ["client_id"]
+        # once we've asserted on it, switch to a singular values to make life easier
+        authorize_params = {k: v[0] for k, v in params.items()}
+
+        access_token = "xxxxx-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx"
+
+        responses.add(
+            responses.POST,
+            "https://gitlab.example.com/oauth/token",
+            json={"access_token": access_token},
+        )
+        responses.add(responses.GET, "https://gitlab.example.com/api/v4/user", json={"id": user_id})
+        responses.add(
+            responses.POST, "https://gitlab.example.com/api/v4/hooks", json={"id": "webhook-id-1"}
+        )
+
+        resp = self.client.get(
+            "{}?{}".format(
+                self.setup_path,
+                urlencode({"code": "oauth-code", "state": authorize_params["state"]}),
+            )
+        )
+
+        mock_access_token_request = responses.calls[0].request
+        req_params = parse_qs(mock_access_token_request.body)
+        assert req_params["grant_type"] == ["authorization_code"]
+        assert req_params["code"] == ["oauth-code"]
+        assert req_params["redirect_uri"] == ["http://testserver/extensions/gitlab/setup/"]
+        assert req_params["client_id"] == ["client_id"]
+        assert req_params["client_secret"] == ["client_secret"]
+
+        assert resp.status_code == 200
+
+        self.assertDialogSuccess(resp)
+
+    @responses.activate
+    @patch("sentry.integrations.gitlab.integration.sha1_text")
+    def test_basic_flow(self, mock_sha):
+        sha = Mock()
+        sha.hexdigest.return_value = "secret-token"
+        mock_sha.return_value = sha
+
+        self.assert_setup_flow()
+
+        integration = Integration.objects.get(provider=self.provider.key)
+
+        assert integration.external_id == "gitlab.example.com:_instance_"
+        assert integration.name == "gitlab.example.com"
+        assert integration.metadata == {
+            "instance": "gitlab.example.com",
+            "scopes": ["api"],
+            "icon": None,
+            "domain_name": "gitlab.example.com",
+            "verify_ssl": True,
+            "base_url": "https://gitlab.example.com",
+            "webhook_secret": "secret-token",
+            "group_id": None,
+            "include_subgroups": False,
+        }
+        oi = OrganizationIntegration.objects.get(
+            integration=integration, organization=self.organization
+        )
+        assert oi.config == {}
+
+        idp = IdentityProvider.objects.get(type="gitlab")
+        identity = Identity.objects.get(
+            idp=idp, user=self.user, external_id="gitlab.example.com:user_id_1"
+        )
+        assert identity.status == IdentityStatus.VALID
+        assert identity.data == {
+            "access_token": "xxxxx-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
+            "client_id": "client_id",
+            "client_secret": "client_secret",
+        }
+
+    @responses.activate
+    def test_get_group_id(self):
+        self.assert_setup_flow()
+        integration = Integration.objects.get(provider=self.provider.key)
+
+        installation = integration.get_installation(self.organization.id)
+        assert installation.get_group_id() is None
diff --git a/tests/sentry/integrations/gitlab/test_search.py b/tests/sentry/integrations/gitlab/test_search.py
--- a/tests/sentry/integrations/gitlab/test_search.py
+++ b/tests/sentry/integrations/gitlab/test_search.py
@@ -60,7 +60,7 @@ def test_finds_external_issue_results_with_iid(self):
     def test_finds_project_results(self):
         responses.add(
             responses.GET,
-            "https://example.gitlab.com/api/v4/groups/1/projects?search=GetSentry&simple=True&include_subgroups=False&page=1&per_page=100",
+            "https://example.gitlab.com/api/v4/groups/1/projects?search=GetSentry&simple=True&include_subgroups=False&page=1&per_page=100&order_by=last_activity_at",
             json=[
                 {
                     "id": "1",
@@ -150,7 +150,7 @@ def test_finds_no_external_issues_results_iid(self):
     def test_finds_no_project_results(self):
         responses.add(
             responses.GET,
-            "https://example.gitlab.com/api/v4/groups/1/projects?search=GetSentry&simple=True&include_subgroups=False&page=1&per_page=100",
+            "https://example.gitlab.com/api/v4/groups/1/projects?search=GetSentry&simple=True&include_subgroups=False&page=1&per_page=100&order_by=last_activity_at",
             json=[],
         )
         resp = self.client.get(self.url, data={"field": "project", "query": "GetSentry"})