Repository: learningequality/kolibri

Issue:
Handle baseurl on ContentNodeTree viewset to allow browsing of non-local topic trees
In parallel to #9382, we should be able to browse non-local topic trees by making a request to the baseurl.


Discussion:

Resolving patch:
diff --git a/kolibri/core/content/api.py b/kolibri/core/content/api.py
--- a/kolibri/core/content/api.py
+++ b/kolibri/core/content/api.py
@@ -14,7 +14,6 @@
 from django.db.models import Sum
 from django.db.models.aggregates import Count
 from django.http import Http404
-from django.http.request import HttpRequest
 from django.utils.cache import patch_response_headers
 from django.utils.decorators import method_decorator
 from django.utils.translation import ugettext as _
@@ -29,6 +28,7 @@
 from django_filters.rest_framework import UUIDFilter
 from le_utils.constants import content_kinds
 from le_utils.constants import languages
+from requests.exceptions import RequestException
 from rest_framework import mixins
 from rest_framework import status
 from rest_framework import viewsets
@@ -36,6 +36,7 @@
 from rest_framework.decorators import list_route
 from rest_framework.generics import get_object_or_404
 from rest_framework.response import Response
+from six.moves.urllib.parse import urljoin
 
 from kolibri.core.api import BaseValuesViewset
 from kolibri.core.api import ListModelMixin
@@ -67,6 +68,7 @@
 from kolibri.core.content.utils.stopwords import stopwords_set
 from kolibri.core.decorators import query_params_required
 from kolibri.core.device.models import ContentCacheKey
+from kolibri.core.discovery.utils.network.errors import ResourceGoneError
 from kolibri.core.lessons.models import Lesson
 from kolibri.core.logger.models import ContentSessionLog
 from kolibri.core.logger.models import ContentSummaryLog
@@ -74,12 +76,17 @@
 from kolibri.core.utils.pagination import ValuesViewsetCursorPagination
 from kolibri.core.utils.pagination import ValuesViewsetLimitOffsetPagination
 from kolibri.core.utils.pagination import ValuesViewsetPageNumberPagination
+from kolibri.utils.conf import OPTIONS
 
 
 logger = logging.getLogger(__name__)
 
 
-def cache_forever(some_func):
+def get_cache_key(*args, **kwargs):
+    return str(ContentCacheKey.get_cache_key())
+
+
+def metadata_cache(some_func):
     """
     Decorator for patch_response_headers function
     """
@@ -87,29 +94,94 @@ def cache_forever(some_func):
     # Source: https://stackoverflow.com/a/3001556/405682
     cache_timeout = 31556926
 
+    @etag(get_cache_key)
     def wrapper_func(*args, **kwargs):
         response = some_func(*args, **kwargs)
-        # This caching has the unfortunate effect of also caching the dynamically
-        # generated filters for recommendation, this quick hack checks if
-        # the request is any of those filters, and then applies less long running
-        # caching on it.
-        timeout = cache_timeout
         try:
             request = args[0]
             request = kwargs.get("request", request)
         except IndexError:
             request = kwargs.get("request", None)
-        if isinstance(request, HttpRequest):
-            if any(map(lambda x: x in request.path, ["next_steps", "resume"])):
-                timeout = 0
-            elif "popular" in request.path:
-                timeout = 600
-        patch_response_headers(response, cache_timeout=timeout)
+        if "contentCacheKey" in request.GET:
+            # Only do long running caching if the contentCacheKey is explicitly
+            # set in the URL, otherwise rely on the Etag to do a quick 304.
+            patch_response_headers(response, cache_timeout=cache_timeout)
+
         return response
 
     return session_exempt(wrapper_func)
 
 
+class RemoteMixin(object):
+    remote_url_param = "baseurl"
+
+    def _should_proxy_request(self, request):
+        return self.remote_url_param in request.GET
+
+    def _get_request_headers(self, request):
+        return {
+            "Accept": request.META.get("HTTP_ACCEPT"),
+            "Accept-Encoding": request.META.get("HTTP_ACCEPT_ENCODING"),
+            "Accept-Language": request.META.get("HTTP_ACCEPT_LANGUAGE"),
+            "Content-Type": request.META.get("CONTENT_TYPE"),
+            "If-None-Match": request.META.get("HTTP_IF_NONE_MATCH", ""),
+        }
+
+    def _get_response_headers(self, response):
+        return {
+            "Cache-Control": response.headers.get("cache-control"),
+            "Content-Type": response.headers.get("content-type"),
+            "Etag": response.headers.get("etag"),
+        }
+
+    def update_data(self, response_data, baseurl):
+        return response_data
+
+    def _hande_proxied_request(self, request):
+        full_path = request.get_full_path().split("?")[0]
+        remote_path = full_path.replace(
+            "/{}api/content/".format(
+                OPTIONS["Deployment"]["URL_PATH_PREFIX"].lstrip("/")
+            ),
+            "/api/public/v2/",
+        )
+        baseurl = request.GET[self.remote_url_param]
+        qs = request.GET.copy()
+        del qs[self.remote_url_param]
+        qs.pop("contentCacheKey", None)
+        remote_url = urljoin(baseurl, remote_path)
+        try:
+            response = requests.get(
+                remote_url, params=qs, headers=self._get_request_headers(request)
+            )
+            # If Etag is set on the response we have returned here, any further Etag will not be modified
+            # by the django etag decorator, so this should allow us to transparently proxy the remote etag.
+            try:
+                content = self.update_data(response.json(), baseurl)
+            except ValueError:
+                content = response.content
+            return Response(
+                content,
+                status=response.status_code,
+                headers=self._get_response_headers(response),
+            )
+        except RequestException:
+            # If any sort of error due to connection or timeout, raise a resource gone error
+            raise ResourceGoneError
+
+
+class RemoteViewSet(ReadOnlyValuesViewset, RemoteMixin):
+    def retrieve(self, request, *args, **kwargs):
+        if self._should_proxy_request(request):
+            return self._hande_proxied_request(request)
+        return super(RemoteViewSet, self).retrieve(request, *args, **kwargs)
+
+    def list(self, request, *args, **kwargs):
+        if self._should_proxy_request(request):
+            return self._hande_proxied_request(request)
+        return super(RemoteViewSet, self).list(request, *args, **kwargs)
+
+
 class ChannelMetadataFilter(FilterSet):
     available = BooleanFilter(method="filter_available", label="Available")
     has_exercise = BooleanFilter(method="filter_has_exercise", label="Has exercises")
@@ -135,8 +207,7 @@ def filter_available(self, queryset, name, value):
         return queryset.filter(root__available=value)
 
 
-@method_decorator(cache_forever, name="dispatch")
-class ChannelMetadataViewSet(ReadOnlyValuesViewset):
+class BaseChannelMetadataMixin(object):
     filter_backends = (DjangoFilterBackend,)
     filter_class = ChannelMetadataFilter
 
@@ -155,6 +226,8 @@ class ChannelMetadataViewSet(ReadOnlyValuesViewset):
         "root__available",
         "root__num_coach_contents",
         "public",
+        "total_resource_count",
+        "published_size",
     )
 
     field_map = {
@@ -167,6 +240,24 @@ class ChannelMetadataViewSet(ReadOnlyValuesViewset):
     def get_queryset(self):
         return models.ChannelMetadata.objects.all()
 
+    def consolidate(self, items, queryset):
+        included_languages = {}
+        for (
+            channel_id,
+            language_id,
+        ) in models.ChannelMetadata.included_languages.through.objects.filter(
+            channelmetadata__in=queryset
+        ).values_list(
+            "channelmetadata_id", "language_id"
+        ):
+            if channel_id not in included_languages:
+                included_languages[channel_id] = []
+            included_languages[channel_id].append(language_id)
+        for item in items:
+            item["included_languages"] = included_languages.get(item["id"], [])
+            item["last_published"] = item["last_updated"]
+        return items
+
     @list_route(methods=["get"])
     def filter_options(self, request, **kwargs):
         channel_id = self.request.query_params.get("id")
@@ -198,6 +289,11 @@ def filter_options(self, request, **kwargs):
         return Response(data)
 
 
+@method_decorator(metadata_cache, name="dispatch")
+class ChannelMetadataViewSet(BaseChannelMetadataMixin, RemoteViewSet):
+    pass
+
+
 class IdFilter(FilterSet):
     ids = CharFilter(method="filter_ids")
 
@@ -553,6 +649,40 @@ def consolidate(self, items, queryset):
                 output.append(item)
         return output
 
+    def update_data(self, response_data, baseurl):
+        if type(response_data) is dict:
+            if "more" in response_data and "results" in response_data:
+                # This is a paginated object
+                if response_data["more"] is not None:
+                    if type(response_data["more"].get("params", None)) is dict:
+                        response_data["more"]["params"][self.remote_url_param] = baseurl
+                    else:
+                        response_data["more"][self.remote_url_param] = baseurl
+                response_data["results"] = self.update_data(
+                    response_data["results"], baseurl
+                )
+            else:
+                response_data = self.add_base_url_to_node(response_data, baseurl)
+                if "children" in response_data:
+                    response_data["children"] = self.update_data(
+                        response_data["children"], baseurl
+                    )
+        elif type(response_data) is list:
+            data = []
+            for node in response_data:
+                data.append(self.update_data(node, baseurl))
+            response_data = data
+        return response_data
+
+    def add_base_url_to_node(self, node, baseurl):
+        baseurl_querystring = "?{}={}".format(self.remote_url_param, baseurl)
+        if node["thumbnail"]:
+            node["thumbnail"] += baseurl_querystring
+        for file in node["files"]:
+            if file["storage_url"]:
+                file["storage_url"] += baseurl_querystring
+        return node
+
 
 class OptionalPagination(ValuesViewsetCursorPagination):
     ordering = ("lft", "id")
@@ -598,23 +728,8 @@ def get_paginated_response_schema(self, schema):
         }
 
 
-def get_resume_queryset(request, queryset):
-    user = request.user
-    # if user is anonymous, don't return any nodes
-    # if person requesting is not the data they are requesting for, also return no nodes
-    if not user.is_facility_user:
-        return queryset.none()
-    # get the most recently viewed, but not finished, content nodes
-    content_ids = (
-        ContentSummaryLog.objects.filter(user=user, progress__gt=0)
-        .exclude(progress=1)
-        .values_list("content_id", flat=True)
-    )
-    return queryset.filter(content_id__in=content_ids)
-
-
-@method_decorator(cache_forever, name="dispatch")
-class ContentNodeViewset(BaseContentNodeMixin, ReadOnlyValuesViewset):
+@method_decorator(metadata_cache, name="dispatch")
+class ContentNodeViewset(BaseContentNodeMixin, RemoteViewSet):
     pagination_class = OptionalContentNodePagination
 
     @list_route(methods=["get"])
@@ -787,126 +902,6 @@ def recommendations_for(self, request, **kwargs):
         )
         return Response(self.serialize(queryset))
 
-    @list_route(methods=["get"])
-    def next_steps(self, request, **kwargs):
-        """
-        Recommend content that has user completed content as a prerequisite, or leftward sibling.
-
-        :param request: request object
-        :return: uncompleted content nodes, or empty queryset if user is anonymous
-        """
-        user = request.user
-        queryset = self.get_queryset()
-        # if user is anonymous, don't return any nodes
-        # if person requesting is not the data they are requesting for, also return no nodes
-        if not user.is_facility_user:
-            queryset = queryset.none()
-        else:
-            completed_content_ids = ContentSummaryLog.objects.filter(
-                user=user, progress=1
-            ).values_list("content_id", flat=True)
-
-            # If no logs, don't bother doing the other queries
-            if not completed_content_ids.exists():
-                queryset = queryset.none()
-            else:
-                completed_content_nodes = queryset.filter_by_content_ids(
-                    completed_content_ids
-                ).order_by()
-
-                # Filter to only show content that the user has not engaged in, so as not to be redundant with resume
-                queryset = (
-                    queryset.exclude_by_content_ids(
-                        ContentSummaryLog.objects.filter(user=user).values_list(
-                            "content_id", flat=True
-                        ),
-                        validate=False,
-                    )
-                    .filter(
-                        Q(has_prerequisite__in=completed_content_nodes)
-                        | Q(
-                            lft__in=[
-                                rght + 1
-                                for rght in completed_content_nodes.values_list(
-                                    "rght", flat=True
-                                )
-                            ]
-                        )
-                    )
-                    .order_by()
-                )
-                if not (
-                    user.roles.exists() or user.is_superuser
-                ):  # must have coach role or higher
-                    queryset = queryset.exclude(coach_content=True)
-
-        return Response(self.serialize(queryset))
-
-    @list_route(methods=["get"])
-    def popular(self, request, **kwargs):
-        """
-        Recommend content that is popular with all users.
-
-        :param request: request object
-        :return: 10 most popular content nodes
-        """
-        cache_key = "popular_content"
-
-        if cache.get(cache_key) is not None:
-            return Response(cache.get(cache_key))
-
-        queryset = self.filter_queryset(self.get_queryset())
-
-        if ContentSessionLog.objects.count() < 50:
-            # return 25 random content nodes if not enough session logs
-            pks = queryset.values_list("pk", flat=True).exclude(
-                kind=content_kinds.TOPIC
-            )
-            # .count scales with table size, so can get slow on larger channels
-            count_cache_key = "content_count_for_popular"
-            count = cache.get(count_cache_key) or min(pks.count(), 25)
-            queryset = queryset.filter_by_uuids(
-                sample(list(pks), count), validate=False
-            )
-        else:
-            # get the most accessed content nodes
-            # search for content nodes that currently exist in the database
-            content_nodes = models.ContentNode.objects.filter(available=True)
-            content_counts_sorted = (
-                ContentSessionLog.objects.filter(
-                    content_id__in=content_nodes.values_list(
-                        "content_id", flat=True
-                    ).distinct()
-                )
-                .values_list("content_id", flat=True)
-                .annotate(Count("content_id"))
-                .order_by("-content_id__count")
-            )
-
-            most_popular = queryset.filter_by_content_ids(
-                list(content_counts_sorted[:20]), validate=False
-            )
-            queryset = most_popular.dedupe_by_content_id(use_distinct=False)
-
-        data = self.serialize(queryset)
-
-        # cache the popular results queryset for 10 minutes, for efficiency
-        cache.set(cache_key, data, 60 * 10)
-
-        return Response(data)
-
-    @list_route(methods=["get"])
-    def resume(self, request, **kwargs):
-        """
-        Recommend content that the user has recently engaged with, but not finished.
-
-        :param request: request object
-        :return: 10 most recently viewed content nodes
-        """
-        queryset = get_resume_queryset(request, self.get_queryset())
-
-        return Response(self.serialize(queryset))
-
 
 # The max recursed page size should be less than 25 for a couple of reasons:
 # 1. At this size the query appears to be relatively performant, and will deliver most of the tree
@@ -991,8 +986,9 @@ def get_tree_queryset(self, request, pk):
         )
 
 
-@method_decorator(cache_forever, name="dispatch")
-class ContentNodeTreeViewset(BaseContentNodeMixin, TreeQueryMixin, BaseValuesViewset):
+class BaseContentNodeTreeViewset(
+    BaseContentNodeMixin, TreeQueryMixin, BaseValuesViewset
+):
     def retrieve(self, request, pk=None):
         """
         A nested, paginated representation of the children and grandchildren of a specific node
@@ -1089,6 +1085,14 @@ def retrieve(self, request, pk=None):
         return Response(parent)
 
 
+@method_decorator(metadata_cache, name="dispatch")
+class ContentNodeTreeViewset(BaseContentNodeTreeViewset, RemoteMixin):
+    def retrieve(self, request, *args, **kwargs):
+        if self._should_proxy_request(request):
+            return self._hande_proxied_request(request)
+        return super(ContentNodeTreeViewset, self).retrieve(request, *args, **kwargs)
+
+
 # return the result of and-ing a list of queries
 def intersection(queries):
     if queries:
@@ -1289,10 +1293,6 @@ def consolidate(self, items, queryset):
         return sorted_items
 
 
-def get_cache_key(*args, **kwargs):
-    return str(ContentCacheKey.get_cache_key())
-
-
 @method_decorator(etag(get_cache_key), name="retrieve")
 class ContentNodeGranularViewset(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     serializer_class = serializers.ContentNodeGranularSerializer
@@ -1356,6 +1356,8 @@ def retrieve(self, request, pk):
 class UserContentNodeFilter(ContentNodeFilter):
     lesson = UUIDFilter(method="filter_by_lesson")
     resume = BooleanFilter(method="filter_by_resume")
+    next_steps = BooleanFilter(method="filter_by_next_steps")
+    popular = BooleanFilter(method="filter_by_popular")
 
     def filter_by_lesson(self, queryset, name, value):
         try:
@@ -1366,7 +1368,111 @@ def filter_by_lesson(self, queryset, name, value):
             return queryset.none()
 
     def filter_by_resume(self, queryset, name, value):
-        return get_resume_queryset(self.request, queryset)
+        user = self.request.user
+        # if user is anonymous, don't return any nodes
+        if not user.is_facility_user:
+            return queryset.none()
+        # get the most recently viewed, but not finished, content nodes
+        content_ids = (
+            ContentSummaryLog.objects.filter(user=user, progress__gt=0)
+            .exclude(progress=1)
+            .values_list("content_id", flat=True)
+        )
+        return queryset.filter(content_id__in=content_ids)
+
+    def filter_by_next_steps(self, queryset, name, value):
+        """
+        Recommend content that has user completed content as a prerequisite, or leftward sibling.
+
+        :param request: request object
+        :return: uncompleted content nodes, or empty queryset if user is anonymous
+        """
+        user = self.request.user
+        # if user is anonymous, don't return any nodes
+        # if person requesting is not the data they are requesting for, also return no nodes
+        if not user.is_facility_user:
+            return queryset.none()
+        completed_content_ids = ContentSummaryLog.objects.filter(
+            user=user, progress=1
+        ).values_list("content_id", flat=True)
+
+        # If no logs, don't bother doing the other queries
+        if not completed_content_ids.exists():
+            return queryset.none()
+        completed_content_nodes = queryset.filter_by_content_ids(
+            completed_content_ids
+        ).order_by()
+
+        # Filter to only show content that the user has not engaged in, so as not to be redundant with resume
+        queryset = (
+            queryset.exclude_by_content_ids(
+                ContentSummaryLog.objects.filter(user=user).values_list(
+                    "content_id", flat=True
+                ),
+                validate=False,
+            )
+            .filter(
+                Q(has_prerequisite__in=completed_content_nodes)
+                | Q(
+                    lft__in=[
+                        rght + 1
+                        for rght in completed_content_nodes.values_list(
+                            "rght", flat=True
+                        )
+                    ]
+                )
+            )
+            .order_by()
+        )
+        if not (
+            user.roles.exists() or user.is_superuser
+        ):  # must have coach role or higher
+            queryset = queryset.exclude(coach_content=True)
+
+        return queryset
+
+    def filter_by_popular(self, queryset, name, value):
+        """
+        Recommend content that is popular with all users.
+
+        :param request: request object
+        :return: 10 most popular content nodes
+        """
+        cache_key = "popular_content"
+
+        content_ids = cache.get(cache_key)
+
+        if content_ids is None:
+            if len(ContentSessionLog.objects.values_list("pk")[:50]) < 50:
+                # return 25 random content nodes if not enough session logs
+                pks = queryset.values_list("pk", flat=True).exclude(
+                    kind=content_kinds.TOPIC
+                )
+                # .count scales with table size, so can get slow on larger channels
+                count_cache_key = "content_count_for_popular"
+                count = cache.get(count_cache_key) or min(pks.count(), 25)
+                return queryset.filter_by_uuids(
+                    sample(list(pks), count), validate=False
+                )
+            # get the most accessed content nodes
+            # search for content nodes that currently exist in the database
+            content_nodes = models.ContentNode.objects.filter(available=True)
+            content_counts_sorted = (
+                ContentSessionLog.objects.filter(
+                    content_id__in=content_nodes.values_list(
+                        "content_id", flat=True
+                    ).distinct()
+                )
+                .values_list("content_id", flat=True)
+                .annotate(Count("content_id"))
+                .order_by("-content_id__count")
+            )
+
+            content_ids = list(content_counts_sorted[:20])
+            # cache the popular results content_ids for 10 minutes, for efficiency
+            cache.set(cache_key, content_ids, 60 * 10)
+
+        return queryset.filter_by_content_ids(content_ids, validate=False)
 
     class Meta:
         model = models.ContentNode
diff --git a/kolibri/core/public/api.py b/kolibri/core/public/api.py
--- a/kolibri/core/public/api.py
+++ b/kolibri/core/public/api.py
@@ -9,6 +9,7 @@
 from django.http import HttpResponseBadRequest
 from django.http import HttpResponseNotFound
 from django.utils import timezone
+from django.utils.decorators import method_decorator
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.gzip import gzip_page
 from morango.constants import transfer_statuses
@@ -21,7 +22,13 @@
 from .. import error_constants
 from .constants.user_sync_statuses import QUEUED
 from .constants.user_sync_statuses import SYNC
+from kolibri.core.api import ReadOnlyValuesViewset
 from kolibri.core.auth.models import FacilityUser
+from kolibri.core.content.api import BaseChannelMetadataMixin
+from kolibri.core.content.api import BaseContentNodeMixin
+from kolibri.core.content.api import BaseContentNodeTreeViewset
+from kolibri.core.content.api import metadata_cache
+from kolibri.core.content.api import OptionalContentNodePagination
 from kolibri.core.content.models import ChannelMetadata
 from kolibri.core.content.models import ContentNode
 from kolibri.core.content.models import LocalFile
@@ -97,6 +104,26 @@ def _get_channel_list_v1(params, identifier=None):
     return channels.filter(root__available=True).distinct()
 
 
+@method_decorator(metadata_cache, name="dispatch")
+class PublicChannelMetadataViewSet(BaseChannelMetadataMixin, ReadOnlyValuesViewset):
+    def get_queryset(self):
+        return (
+            ChannelMetadata.objects.all()
+            if allow_peer_unlisted_channel_import()
+            else ChannelMetadata.objects.filter(public=True)
+        )
+
+
+@method_decorator(metadata_cache, name="dispatch")
+class PublicContentNodeViewSet(BaseContentNodeMixin, ReadOnlyValuesViewset):
+    pagination_class = OptionalContentNodePagination
+
+
+@method_decorator(metadata_cache, name="dispatch")
+class PublicContentNodeTreeViewSet(BaseContentNodeTreeViewset):
+    pass
+
+
 @api_view(["GET"])
 def get_public_channel_list(request, version):
     """ Endpoint: /public/<version>/channels/?=<query params> """
diff --git a/kolibri/core/public/api_urls.py b/kolibri/core/public/api_urls.py
--- a/kolibri/core/public/api_urls.py
+++ b/kolibri/core/public/api_urls.py
@@ -26,8 +26,12 @@
 from .api import get_public_channel_lookup
 from .api import get_public_file_checksums
 from .api import InfoViewSet
+from .api import PublicChannelMetadataViewSet
+from .api import PublicContentNodeTreeViewSet
+from .api import PublicContentNodeViewSet
 from .api import SyncQueueViewSet
 
+
 router = routers.SimpleRouter()
 
 router.register(r"v1/facility", PublicFacilityViewSet, base_name="publicfacility")
@@ -36,9 +40,23 @@
 router.register(r"info", InfoViewSet, base_name="info")
 router.register(r"syncqueue", SyncQueueViewSet, base_name="syncqueue")
 
+public_content_v2_router = routers.SimpleRouter()
+public_content_v2_router.register(
+    r"channel", PublicChannelMetadataViewSet, base_name="publicchannel"
+)
+public_content_v2_router.register(
+    r"contentnode", PublicContentNodeViewSet, base_name="publiccontentnode"
+)
+public_content_v2_router.register(
+    r"contentnode_tree",
+    PublicContentNodeTreeViewSet,
+    base_name="publiccontentnode_tree",
+)
+
 # Add public api endpoints
 urlpatterns = [
     url(r"^", include(router.urls)),
+    url(r"v2/", include(public_content_v2_router.urls)),
     url(
         r"(?P<version>[^/]+)/channels/lookup/(?P<identifier>[^/]+)",
         get_public_channel_lookup,


Test patch:
diff --git a/kolibri/core/content/test/test_content_app.py b/kolibri/core/content/test/test_content_app.py
--- a/kolibri/core/content/test/test_content_app.py
+++ b/kolibri/core/content/test/test_content_app.py
@@ -10,6 +10,7 @@
 from django.conf import settings
 from django.core.cache import cache
 from django.core.urlresolvers import reverse
+from django.test import LiveServerTestCase
 from django.test import TestCase
 from django.utils import timezone
 from le_utils.constants import content_kinds
@@ -25,6 +26,7 @@
 from kolibri.core.device.models import DeviceSettings
 from kolibri.core.logger.models import ContentSessionLog
 from kolibri.core.logger.models import ContentSummaryLog
+from kolibri.utils.tests.helpers import override_option
 
 DUMMY_PASSWORD = "password"
 
@@ -214,13 +216,11 @@ def infer_learning_activity(kind):
     return []
 
 
-class ContentNodeAPITestCase(APITestCase):
-    """
-    Testcase for content API methods
-    """
+class ContentNodeAPIBase(object):
 
     fixtures = ["content_test.json"]
     the_channel_id = "6199dde695db4ee4ab392222d5af1e5c"
+    baseurl = None
 
     @classmethod
     def setUpTestData(cls):
@@ -231,26 +231,8 @@ def setUpTestData(cls):
         cls.admin.save()
         cls.facility.add_admin(cls.admin)
 
-    def test_prerequisite_for_filter(self):
-        c1_id = content.ContentNode.objects.get(title="c1").id
-        response = self.client.get(
-            reverse("kolibri:core:contentnode-list"), data={"prerequisite_for": c1_id}
-        )
-        self.assertEqual(response.data[0]["title"], "root")
-
-    def test_has_prerequisite_filter(self):
-        root_id = content.ContentNode.objects.get(title="root").id
-        response = self.client.get(
-            reverse("kolibri:core:contentnode-list"), data={"has_prerequisite": root_id}
-        )
-        self.assertEqual(response.data[0]["title"], "c1")
-
-    def test_related_filter(self):
-        c1_id = content.ContentNode.objects.get(title="c1").id
-        response = self.client.get(
-            reverse("kolibri:core:contentnode-list"), data={"related": c1_id}
-        )
-        self.assertEqual(response.data[0]["title"], "c2")
+    def _get(self, *args, **kwargs):
+        return self.client.get(*args, **kwargs)
 
     def map_language(self, lang):
         if lang:
@@ -304,9 +286,13 @@ def _assert_node(self, actual, expected):
                 file[field] = getattr(f.local_file, field)
             file["lang"] = self.map_language(f.lang)
             file["storage_url"] = f.get_storage_url()
+            if self.baseurl and file["storage_url"]:
+                file["storage_url"] += "?baseurl={}".format(self.baseurl)
             files.append(file)
             if f.thumbnail:
                 thumbnail = f.get_storage_url()
+                if self.baseurl and thumbnail:
+                    thumbnail += "?baseurl={}".format(self.baseurl)
         self.assertEqual(
             actual,
             {
@@ -369,7 +355,7 @@ def test_contentnode_list(self):
         root = content.ContentNode.objects.get(title="root")
         nodes = root.get_descendants(include_self=True).filter(available=True)
         expected_output = len(nodes)
-        response = self.client.get(reverse("kolibri:core:contentnode-list"))
+        response = self._get(reverse("kolibri:core:contentnode-list"))
         self.assertEqual(len(response.data), expected_output)
         self._assert_nodes(response.data, nodes)
 
@@ -386,7 +372,7 @@ def test_contentnode_list_long(self):
         nodes = content.ContentNode.objects.filter(available=True)
         expected_output = len(nodes)
         self.assertGreater(expected_output, 1000)
-        response = self.client.get(reverse("kolibri:core:contentnode-list"))
+        response = self._get(reverse("kolibri:core:contentnode-list"))
         self.assertEqual(len(response.data), expected_output)
         self._assert_nodes(response.data, nodes)
 
@@ -409,6 +395,10 @@ def _recurse_and_assert(self, data, nodes, recursion_depth=0):
                     self.assertEqual(
                         children["more"]["params"]["depth"], 2 - recursion_depth
                     )
+                    if self.baseurl:
+                        self.assertEqual(
+                            children["more"]["params"]["baseurl"], self.baseurl
+                        )
                 self._recurse_and_assert(
                     children["results"],
                     child_nodes,
@@ -417,7 +407,7 @@ def _recurse_and_assert(self, data, nodes, recursion_depth=0):
 
     def test_contentnode_tree(self):
         root = content.ContentNode.objects.get(title="root")
-        response = self.client.get(
+        response = self._get(
             reverse("kolibri:core:contentnode_tree-detail", kwargs={"pk": root.id})
         )
         self._recurse_and_assert([response.data], [root])
@@ -432,14 +422,14 @@ def test_contentnode_tree_long(self):
         builder.insert_into_default_db()
         content.ContentNode.objects.all().update(available=True)
         root = content.ContentNode.objects.get(id=builder.root_node["id"])
-        response = self.client.get(
+        response = self._get(
             reverse("kolibri:core:contentnode_tree-detail", kwargs={"pk": root.id})
         )
         self._recurse_and_assert([response.data], [root])
 
     def test_contentnode_tree_depth_1(self):
         root = content.ContentNode.objects.get(title="root")
-        response = self.client.get(
+        response = self._get(
             reverse("kolibri:core:contentnode_tree-detail", kwargs={"pk": root.id}),
             data={"depth": 1},
         )
@@ -456,7 +446,7 @@ def test_contentnode_tree_next__gt(self):
         content.ContentNode.objects.all().update(available=True)
         root = content.ContentNode.objects.get(id=builder.root_node["id"])
         next__gt = content.ContentNode.objects.filter(parent=root)[11].rght
-        response = self.client.get(
+        response = self._get(
             reverse("kolibri:core:contentnode_tree-detail", kwargs={"pk": root.id}),
             data={"next__gt": next__gt},
         )
@@ -477,12 +467,12 @@ def test_contentnode_tree_more(self):
         builder.insert_into_default_db()
         content.ContentNode.objects.all().update(available=True)
         root = content.ContentNode.objects.get(id=builder.root_node["id"])
-        response = self.client.get(
+        response = self._get(
             reverse("kolibri:core:contentnode_tree-detail", kwargs={"pk": root.id})
         )
         first_child = response.data["children"]["results"][0]
         self.assertEqual(first_child["children"]["more"]["params"]["depth"], 1)
-        nested_page_response = self.client.get(
+        nested_page_response = self._get(
             reverse(
                 "kolibri:core:contentnode_tree-detail",
                 kwargs={"pk": first_child["children"]["more"]["id"]},
@@ -492,6 +482,33 @@ def test_contentnode_tree_more(self):
         self.assertEqual(len(nested_page_response.data["children"]["results"]), 5)
         self.assertIsNone(nested_page_response.data["children"]["more"])
 
+
+class ContentNodeAPITestCase(ContentNodeAPIBase, APITestCase):
+    """
+    Testcase for content API methods
+    """
+
+    def test_prerequisite_for_filter(self):
+        c1_id = content.ContentNode.objects.get(title="c1").id
+        response = self.client.get(
+            reverse("kolibri:core:contentnode-list"), data={"prerequisite_for": c1_id}
+        )
+        self.assertEqual(response.data[0]["title"], "root")
+
+    def test_has_prerequisite_filter(self):
+        root_id = content.ContentNode.objects.get(title="root").id
+        response = self.client.get(
+            reverse("kolibri:core:contentnode-list"), data={"has_prerequisite": root_id}
+        )
+        self.assertEqual(response.data[0]["title"], "c1")
+
+    def test_related_filter(self):
+        c1_id = content.ContentNode.objects.get(title="c1").id
+        response = self.client.get(
+            reverse("kolibri:core:contentnode-list"), data={"related": c1_id}
+        )
+        self.assertEqual(response.data[0]["title"], "c2")
+
     @mock.patch("kolibri.core.content.api.get_channel_stats_from_studio")
     def test_contentnode_granular_network_import(self, stats_mock):
         c1 = content.ContentNode.objects.get(title="root")
@@ -1361,7 +1378,9 @@ def _create_session_logs(self):
 
     def test_popular(self):
         expected_content_ids = self._create_session_logs()
-        response = self.client.get(reverse("kolibri:core:contentnode-popular"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"popular": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
@@ -1372,8 +1391,8 @@ def test_popular_no_coach_content(self):
         node.save()
         expected_content_ids = expected_content_ids[1:]
         response = self.client.get(
-            reverse("kolibri:core:contentnode-popular"),
-            data={"include_coach_content": False},
+            reverse("kolibri:core:usercontentnode-list"),
+            data={"popular": True, "include_coach_content": False},
         )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
@@ -1389,17 +1408,12 @@ def test_popular_coach_has_coach_content(self):
         node.save()
         self.client.login(username="coach", password=DUMMY_PASSWORD)
         response = self.client.get(
-            reverse("kolibri:core:contentnode-popular"),
-            data={"include_coach_content": True},
+            reverse("kolibri:core:usercontentnode-list"),
+            data={"popular": True, "include_coach_content": True},
         )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
-    def test_popular_ten_minute_cache(self):
-        self._create_session_logs()
-        response = self.client.get(reverse("kolibri:core:contentnode-popular"))
-        self.assertEqual(response["Cache-Control"], "max-age=600")
-
     def _create_summary_logs(self):
         facility = Facility.objects.create(name="MyFac")
         user = FacilityUser.objects.create(username="user", facility=facility)
@@ -1441,14 +1455,18 @@ def _create_summary_logs(self):
     def test_resume(self):
         user, expected_content_ids = self._create_summary_logs()
         self.client.login(username=user.username, password=DUMMY_PASSWORD)
-        response = self.client.get(reverse("kolibri:core:contentnode-resume"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"resume": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
     def test_resume_zero_cache(self):
         user, expected_content_ids = self._create_summary_logs()
         self.client.login(username=user.username, password=DUMMY_PASSWORD)
-        response = self.client.get(reverse("kolibri:core:contentnode-resume"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"resume": True}
+        )
         self.assertEqual(response["Cache-Control"], "max-age=0")
 
     def test_next_steps_prereq(self):
@@ -1468,7 +1486,9 @@ def test_next_steps_prereq(self):
         self.client.login(username=user.username, password=DUMMY_PASSWORD)
         post_req = root.prerequisite_for.first()
         expected_content_ids = (post_req.content_id,)
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
@@ -1487,7 +1507,9 @@ def test_next_steps_prereq_zero_cache(self):
         user.set_password(DUMMY_PASSWORD)
         user.save()
         self.client.login(username=user.username, password=DUMMY_PASSWORD)
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         self.assertEqual(response["Cache-Control"], "max-age=0")
 
     def test_next_steps_prereq_in_progress(self):
@@ -1515,7 +1537,9 @@ def test_next_steps_prereq_in_progress(self):
             kind="audio",
         )
         expected_content_ids = []
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
@@ -1537,7 +1561,9 @@ def test_next_steps_prereq_coach_content_not_coach(self):
         post_req = root.prerequisite_for.first()
         post_req.coach_content = True
         post_req.save()
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(), response_content_ids)
 
@@ -1561,7 +1587,9 @@ def test_next_steps_prereq_coach_content_coach(self):
         post_req.coach_content = True
         post_req.save()
         expected_content_ids = (post_req.content_id,)
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
@@ -1584,7 +1612,9 @@ def test_next_steps_sibling(self):
         self.client.login(username=user.username, password=DUMMY_PASSWORD)
         sibling = node.get_next_sibling()
         expected_content_ids = (sibling.content_id,)
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
@@ -1615,7 +1645,9 @@ def test_next_steps_sibling_in_progress(self):
             kind="audio",
         )
         expected_content_ids = []
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
@@ -1639,7 +1671,9 @@ def test_next_steps_sibling_coach_content_not_coach(self):
         sibling = node.get_next_sibling()
         sibling.coach_content = True
         sibling.save()
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(), response_content_ids)
 
@@ -1665,7 +1699,9 @@ def test_next_steps_sibling_coach_content_coach(self):
         sibling.coach_content = True
         sibling.save()
         expected_content_ids = (sibling.content_id,)
-        response = self.client.get(reverse("kolibri:core:contentnode-next-steps"))
+        response = self.client.get(
+            reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True}
+        )
         response_content_ids = {node["content_id"] for node in response.json()}
         self.assertSetEqual(set(expected_content_ids), response_content_ids)
 
@@ -1768,3 +1804,22 @@ def test_channel_list_offline(self, mock_get):
 
     def tearDown(self):
         cache.clear()
+
+
+class ProxyContentMetadataTestCase(ContentNodeAPIBase, LiveServerTestCase):
+    @property
+    def baseurl(self):
+        return self.live_server_url
+
+    def _get(self, *args, **kwargs):
+        if "data" not in kwargs:
+            kwargs["data"] = {}
+        kwargs["data"]["baseurl"] = self.baseurl
+        return self.client.get(*args, **kwargs)
+
+
+@override_option("Deployment", "URL_PATH_PREFIX", "test")
+class PrefixedProxyContentMetadataTestCase(ProxyContentMetadataTestCase):
+    @property
+    def baseurl(self):
+        return self.live_server_url + "/test/"