Repository: django-cms/django-cms

Issue:
improve support for multilanguage sitemaps
Working on a multilanguage web site and I needed a `sitemap.xml` that includes correct URLs for each page, in each language.

Currently `django-cms` generates URLs without any language prefix, even if I have the multilanguage URL middleware enabled. It also includes all pages at once, but I get the slugs in only one language (the current site language). So it's all a mix up: English-only pages end up getting French slugs and so on.

This pull request does the following:
- `CMSSitemap` only includes URLs for the current site language, by default. If I am not mistaken, this allows you to serve multiple language-specific `sitemap.xml` files - just point to `/en/sitemap.xml`, or `/other-language/sitemap.xml` from `robots.txt` and make sure you correctly setup your `urls.py`.
- I included a fix for `Title.objects.public()` which doesn't seem to work.
- Included a new `sitemaps/utils.py` that people can use to generate a single `sitemap.xml` with all the pages in all languages. Example:

``` python
from cms.sitemaps import CMSSitemap
from cms.sitemaps.utils import MakeMultilanguageSitemap
sitemaps = { 'sitemaps': MakeMultilanguageSitemap({
    'cmspages': CMSSitemap,
}),}
```

I use this to generate correct URLs for all multilanguage pages and apps. This utility function works for me with other sitemap classes as well.

Please let me know if further changes are needed. Thank you!



Discussion:
Please make a test for this change. And run the testsuite. You will see that at the moment this build is broken because of the queryset changes.

@digi604, AFAIK this is supersed in 2.4+ by using `i18n_patterns`

Closing, supersed in 2.4

Hmmm, so, if this was superseded in 2.4, then why do the sitemaps still only include the EN pages?

sure?

Hmm, well, if I open //sitemaps/sitemap-cmspages.xml, I get **only** the /en/ urls.

If I then go visit an /es/ url, then come back to the same sitemaps, I get **only** the /es/ urls.

Is this how it is supposed to work?

nope


Resolving patch:
diff --git a/cms/sitemaps/cms_sitemap.py b/cms/sitemaps/cms_sitemap.py
--- a/cms/sitemaps/cms_sitemap.py
+++ b/cms/sitemaps/cms_sitemap.py
@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 from django.contrib.sitemaps import Sitemap
+from django.utils import translation
+from cms.models import Title
+
 
 def from_iterable(iterables):
     """
@@ -9,21 +12,25 @@ def from_iterable(iterables):
         for element in it:
             yield element
 
+
 class CMSSitemap(Sitemap):
     changefreq = "monthly"
     priority = 0.5
 
     def items(self):
-        from cms.utils.page_resolver import get_page_queryset
-        page_queryset = get_page_queryset(None)
-        all_pages = page_queryset.published().filter(login_required=False)
-        return all_pages
+        all_titles = Title.objects.public().filter(page__login_required=False)
+        return all_titles
 
-    def lastmod(self, page):
-        modification_dates = [page.changed_date, page.publication_date]
+    def lastmod(self, title):
+        modification_dates = [title.page.changed_date, title.page.publication_date]
         plugins_for_placeholder = lambda placeholder: placeholder.get_plugins()
-        plugins = from_iterable(map(plugins_for_placeholder, page.placeholders.all()))
+        plugins = from_iterable(map(plugins_for_placeholder, title.page.placeholders.all()))
         plugin_modification_dates = map(lambda plugin: plugin.changed_date, plugins)
         modification_dates.extend(plugin_modification_dates)
         return max(modification_dates)
-    
+
+    def location(self, title):
+        translation.activate(title.language)
+        url = title.page.get_absolute_url(title.language)
+        translation.deactivate()
+        return url


Test patch:
diff --git a/cms/tests/__init__.py b/cms/tests/__init__.py
--- a/cms/tests/__init__.py
+++ b/cms/tests/__init__.py
@@ -26,6 +26,7 @@
 from cms.tests.security import *
 from cms.tests.settings import *
 from cms.tests.site import *
+from cms.tests.sitemap import *
 from cms.tests.static_placeholder import *
 from cms.tests.staticfiles import *
 from cms.tests.templatetags import *
diff --git a/cms/tests/page.py b/cms/tests/page.py
--- a/cms/tests/page.py
+++ b/cms/tests/page.py
@@ -12,7 +12,7 @@
 from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
 from django.utils import timezone
 
-from cms.admin.forms import PageForm, AdvancedSettingsForm
+from cms.admin.forms import AdvancedSettingsForm
 from cms.admin.pageadmin import PageAdmin
 from cms.api import create_page, add_plugin
 from cms.middleware.user import CurrentUserMiddleware
@@ -333,7 +333,6 @@ def test_copy_page(self):
 
         self.assertEqual(Page.objects.drafts().count() - count, 3)
 
-
     def test_language_change(self):
         superuser = self.get_superuser()
         with self.login_user_context(superuser):
@@ -417,7 +416,6 @@ def test_move_page_inherit(self):
         child.move_page(parent, 'left')
         self.assertEqual(child.get_template(), parent.get_template())
 
-
     def test_add_placeholder(self):
         # create page
         page = create_page("Add Placeholder", "nav_playground.html", "en",
@@ -467,11 +465,12 @@ def test_sitemap_uses_publication_date_when_later_than_modification(self):
         now -= datetime.timedelta(microseconds=now.microsecond)
         one_day_ago = now - datetime.timedelta(days=1)
         page = create_page("page", "nav_playground.html", "en", published=True, publication_date=now)
+        title = page.get_title_obj('en')
         page.creation_date = one_day_ago
         page.changed_date = one_day_ago
         sitemap = CMSSitemap()
-        actual_last_modification_time = sitemap.lastmod(page)
-        self.assertEqual(actual_last_modification_time, now)
+        actual_last_modification_time = sitemap.lastmod(title)
+        self.assertEqual(actual_last_modification_time.date(), now.date())
 
     def test_edit_page_other_site_and_language(self):
         """
diff --git a/cms/tests/plugins.py b/cms/tests/plugins.py
--- a/cms/tests/plugins.py
+++ b/cms/tests/plugins.py
@@ -740,16 +740,15 @@ def test_editing_plugin_changes_page_modification_time_in_sitemap(self):
         now = timezone.now()
         one_day_ago = now - datetime.timedelta(days=1)
         page = create_page("page", "nav_playground.html", "en", published=True, publication_date=now)
+        title = page.get_title_obj('en')
         page.creation_date = one_day_ago
         page.changed_date = one_day_ago
-
         plugin_id = self._create_text_plugin_on_page(page)
         plugin = self._edit_text_plugin(plugin_id, "fnord")
 
-        actual_last_modification_time = CMSSitemap().lastmod(page)
-        self.assertEqual(plugin.changed_date - datetime.timedelta(microseconds=plugin.changed_date.microsecond),
-                         actual_last_modification_time - datetime.timedelta(
-                             microseconds=actual_last_modification_time.microsecond))
+        actual_last_modification_time = CMSSitemap().lastmod(title)
+        actual_last_modification_time -= datetime.timedelta(microseconds=actual_last_modification_time.microsecond)
+        self.assertEqual(plugin.changed_date.date(), actual_last_modification_time.date())
 
     def test_moving_plugin_to_different_placeholder(self):
         plugin_pool.register_plugin(DumbFixturePlugin)
diff --git a/cms/tests/sitemap.py b/cms/tests/sitemap.py
new file mode 100644
--- /dev/null
+++ b/cms/tests/sitemap.py
@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+from django.utils.translation import ugettext_lazy as _
+from cms.models import Title, Page
+from cms.sitemaps import CMSSitemap
+from cms.test_utils.testcases import CMSTestCase
+from cms.api import create_page, create_title
+from cms.test_utils.util.context_managers import SettingsOverride
+
+
+class SitemapTestCase(CMSTestCase):
+    def create_fixtures(self):
+        """
+        Tree from fixture:
+
+            + P1 (de, en)
+            | + P2 (de, en)
+            |   + P3 (de, en)
+            | + P9 (de unpublished, en)
+            |   + P10 unpublished (de, en)
+            |   + P11 (en)
+            + P4 (de, en)
+            | + P5 (de, en)
+            + P6 (de, en) (not in menu)
+              + P7 (de, en)
+              + P8 (de, en)
+        """
+        defaults = {
+            'template': 'nav_playground.html',
+            'language': 'en',
+        }
+        with SettingsOverride(CMS_MODERATOR=False, CMS_PERMISSION=False):
+            p1 = create_page('P1', published=True, in_navigation=True, **defaults)
+            create_title(language='de', title="other title %s" % p1.get_title('en'), page=p1)
+            p4 = create_page('P4', published=True, in_navigation=True, **defaults)
+            create_title(language='de', title="other title %s" % p4.get_title('en'), page=p4)
+            p6 = create_page('P6', published=True, in_navigation=False, **defaults)
+            create_title(language='de', title="other title %s" % p6.get_title('en'), page=p6)
+            p2 = create_page('P2', published=True, in_navigation=True, parent=p1, **defaults)
+            create_title(language='de', title="other title %s" % p2.get_title('en'), page=p2)
+            p3 = create_page('P3', published=True, in_navigation=True, parent=p2, **defaults)
+            create_title(language='de', title="other title %s" % p3.get_title('en'), page=p3)
+            p5 = create_page('P5', published=True, in_navigation=True, parent=p4, **defaults)
+            create_title(language='de', title="other title %s" % p5.get_title('en'), page=p5)
+            p7 = create_page('P7', published=True, in_navigation=True, parent=p6, **defaults)
+            create_title(language='de', title="other title %s" % p7.get_title('en'), page=p7)
+            p8 = create_page('P8', published=True, in_navigation=True, parent=p6, **defaults)
+            create_title(language='de', title="other title %s" % p8.get_title('en'), page=p8)
+            p9 = create_page('P9', published=True, in_navigation=True, parent=p1, **defaults)
+            create_title(language='de', title="other title %s" % p9.get_title('en'), page=p9)
+            p10 = create_page('P10', published=False, in_navigation=True, parent=p9, **defaults)
+            create_title(language='de', title="other title %s" % p10.get_title('en'), page=p10)
+            p11 = create_page('P11', published=True, in_navigation=True, parent=p9, **defaults)
+            p8.publish()
+            p7.publish()
+            p5.publish()
+            p3.publish()
+            p2.publish()
+            p6.publish()
+            p4.publish()
+            p1.publish()
+
+    def test_sitemap_count(self):
+        """
+        Has the sitemap the correct number of elements?
+        """
+        sitemap = CMSSitemap()
+        # 8 pages with en and de titles published
+        # 1 page published only in english(with existsing de title)
+        # 1 page with both titles but unpublished
+        # 1 page with only english title
+        self.assertEqual(sitemap.items().count(), 18)
+
+    def test_sitemap_items_location(self):
+        """
+        Check the correct URL in location, recreating it according to the title
+        attributes (instead of using Page.get_absolute_url) for a lower level
+        check
+        """
+        sitemap = CMSSitemap()
+        urlset = sitemap.get_urls()
+        for item in urlset:
+            if item['item'].path:
+                url = 'http://example.com/%s/%s/' % (item['item'].language, item['item'].path)
+            else:
+                url = 'http://example.com/%s/%s' % (item['item'].language, item['item'].path)
+            self.assertEqual(item['location'], url)
+
+    def test_sitemap_published_titles(self):
+        """
+        Check that published titles are in the urls
+        """
+        sitemap = CMSSitemap()
+        locations = []
+        urlset = sitemap.get_urls()
+        for item in urlset:
+            locations.append(item['location'])
+        for title in Title.objects.public():
+            page = title.page.get_public_object()
+            if title.path:
+                url = 'http://example.com/%s/%s/' % (title.language, title.path)
+            else:
+                url = 'http://example.com/%s/%s' % (title.language, title.path)
+            if page.published and not page.publisher_is_draft:
+                self.assertTrue(url in locations)
+            else:
+                self.assertFalse(url in locations)
+
+    def test_sitemap_unpublished_titles(self):
+        """
+        Check that titles attached to unpublished pages are not in the urlset.
+        As titles are 'published' depending on their attached page, we create a
+        set of unpublished titles by checking titles attached to the draft and
+        public version of each page
+        """
+        sitemap = CMSSitemap()
+        locations = []
+        urlset = sitemap.get_urls()
+        unpublished_titles = set()
+        for item in urlset:
+            locations.append(item['location'])
+        for page in Page.objects.drafts():
+            if page.get_public_object():
+                set1 = set(page.get_public_object().title_set.values_list('path', flat=True))
+                set2 = set(page.title_set.values_list('path', flat=True))
+                unpublished_titles.update(set2.difference(set1))
+            else:
+                unpublished_titles.update(page.title_set.values_list('path', flat=True))
+
+        for path in unpublished_titles:
+            title = Title.objects.get(path=path)
+            if title.path:
+                url = 'http://example.com/%s/%s/' % (title.language, title.path)
+            else:
+                url = 'http://example.com/%s/%s' % (title.language, title.path)
+            self.assertFalse(url in locations)