Repository: akvo/akvo-rsr

Issue:
Fix problems with Location advanced filter in project directory
`ProjectLocation`s associated with projects were being used to populate the list of options in the advanced filter for locations, while `RecipientCountry` locations are shown in the project directory, when showing information about each project.  This made the search buggy, and also hard to understand.  To make it easier for the user to understand how search is working, this code ignores the `ProjectLocation` countries completely, and simply uses the `RecipientCountry` values.

- [ ] Test plan | Unit test | Integration test 
- [ ] Copyright header 
- [ ] Code formatting 
- [ ] Documentation 
- [ ] Change log entry

Closes #2623 


Discussion:

Resolving patch:
diff --git a/akvo/rsr/filters.py b/akvo/rsr/filters.py
--- a/akvo/rsr/filters.py
+++ b/akvo/rsr/filters.py
@@ -6,15 +6,17 @@
 see < http://www.gnu.org/licenses/agpl.html >.
 """
 
+from copy import deepcopy
 
 import django_filters
-
-from copy import deepcopy
+from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
+
 from akvo.codelists.store.codelists_v202 import ACTIVITY_STATUS, SECTOR_CATEGORY
 from akvo.utils import codelist_choices
 from .models import (Category, Keyword, Organisation, OrganisationLocation,
-                     Project, ProjectLocation, ProjectUpdate, ProjectUpdateLocation)
+                     Project, ProjectLocation, ProjectUpdate, ProjectUpdateLocation,
+                     RecipientCountry)
 from .m49 import M49_CODES, M49_HIERARCHY
 
 ANY_CHOICE = (('', _('All')), )
@@ -50,10 +52,7 @@ def walk(node):
     elif isinstance(node, int):
         return walk(deepcopy(M49_HIERARCHY)[node])
     else:
-        if node:
-            return (walk(node.pop()) + walk(node))
-        else:
-            return []
+        return (walk(node.pop()) + walk(node)) if node else []
 
 
 def filter_m49(queryset, value):
@@ -61,7 +60,12 @@ def filter_m49(queryset, value):
     if not value:
         return queryset
     countries = walk(deepcopy(M49_HIERARCHY)[int(value)])
-    return queryset.filter(recipient_countries__country__in=countries)
+    countries_lower = [c.lower() for c in countries]
+    filter_ = (
+        Q(recipient_countries__country__in=countries) |
+        Q(locations__country__iso_code__in=countries_lower)
+    )
+    return queryset.filter(filter_)
 
 
 def filter_m49_orgs(queryset, value):
@@ -73,51 +77,80 @@ def filter_m49_orgs(queryset, value):
 
 
 def get_id_for_iso(i):
-    """From an iso_code e.g. 'SE' get the identifier."""
+    """From an iso_code e.g. 'SE' get the identifier.
+
+    NOTE: If i is already an id, the parent id of the given id is returned!
+
+    """
     i = [k for k, v in M49_HIERARCHY.iteritems() if i in v]
-    if not i:
-        return None
-    else:
-        return i.pop()
+    return None if not i else i.pop()
 
 
-def get_locations(location, locations):
-    """Based on one location (country or group as Europe) get all the"""
+def get_location_hierarchy(location, locations=None):
+    """Return the location > parent > ... > continent hierarchy for a location."""
+    if locations is None:
+        locations = [location]
+    # FIXME: Actually returns parent id, when location is already an id!
     l = get_id_for_iso(location)
-    if isinstance(l, basestring):
-        return locations
-    elif l is 1:
-        return locations
-    elif l is None:
+    if isinstance(l, basestring) or l is 1 or l is None:
         return locations
     else:
         locations.append(l)
-        return get_locations(l, locations)
+        return get_location_hierarchy(l, locations)
 
 
 def location_choices(qs):
-    """From a queryset get possible location filter choices"""
+    """Return a filterd list of locations from M49_CODES based on queryset."""
+
+    country_ids = get_country_ids(qs)
+
+    location_ids = {
+        unicode(location)
+        for country_id in country_ids
+        for location in get_location_hierarchy(country_id)
+    }
+
+    # Add World to locations
+    location_ids.add("")
+
+    return filter(lambda (id_, name): id_ in location_ids, M49_CODES)
+
+
+def get_country_ids(qs):
+    """Return country ids for locations associated with the queryset items."""
+
+    country_ids = get_location_country_ids(qs)
+    if qs.model is Project:
+        country_ids = get_recipient_country_ids(qs) + country_ids
+
+    return set(country_ids)
+
+
+def get_recipient_country_ids(projects):
+    """Return countries based on recipient country of projects."""
+    countries = RecipientCountry.objects.filter(project__in=projects)
+    return [get_id_for_iso(country.country.upper()) for country in countries]
+
+
+def get_location_country_ids(qs):
+    """Return countries for locations associated with objects in the queryset."""
 
     if qs.model is Project:
         location_model = ProjectLocation
+
     elif qs.model is ProjectUpdate:
         location_model = ProjectUpdateLocation
+
     elif qs.model is Organisation:
         location_model = OrganisationLocation
 
     locations_qs = location_model.objects.filter(
         location_target__in=qs).order_by('country__id').distinct('country__id')
 
-    locations = []
-    for location in locations_qs:
-        if location.country:
-            country = get_id_for_iso(location.country.iso_code.upper())
-            locations.append(country)
-            locations.extend(get_locations(country, []))
-
-    choices = [tup for tup in M49_CODES if any(
-        unicode(i) in tup for i in locations)]
-    return [M49_CODES[0]] + choices  # Add the world to the choices
+    return [
+        get_id_for_iso(location.country.iso_code.upper())
+        for location in locations_qs if location.country
+    ]
 
 
 def build_choices(qs):
@@ -134,11 +167,6 @@ class BaseProjectFilter(django_filters.FilterSet):
         label=_(u'category'),
         name='categories__id')
 
-    location = django_filters.ChoiceFilter(
-        choices=M49_CODES,
-        label=_(u'location'),
-        action=filter_m49)
-
     sector = django_filters.ChoiceFilter(
         initial=_('All'),
         choices=([('', _('All'))] + sectors()),
@@ -160,18 +188,8 @@ class BaseProjectFilter(django_filters.FilterSet):
         label=_(u'Search'),
         name='title')
 
-    organisation = django_filters.ChoiceFilter(
-        choices=get_orgs(),
-        label=_(u'organisation'),
-        name='partners__id')
 
-    class Meta:
-        model = Project
-        fields = ['status', 'iati_status', 'location', 'organisation', 'category',
-                  'sector', 'title', ]
-
-
-def create_project_filter_class(request):
+def create_project_filter_class(request, projects):
     """Create ProjectFilter class based on request attributes."""
 
     def keywords():
@@ -182,12 +200,51 @@ def keywords():
         keywords = list(keywords.values_list('id', 'label'))
         return [('', _('All'))] + keywords
 
+    def locations():
+        if request.rsr_page is not None:
+            return location_choices(projects)
+        else:
+            return M49_CODES
+
+    def organisations():
+        if request.rsr_page is not None:
+            return build_choices(request.rsr_page.partners())
+        else:
+            return get_orgs()
+
     class ProjectFilter(BaseProjectFilter):
+
         keyword = django_filters.ChoiceFilter(
             initial=_('All'),
             choices=keywords(),
             label=_(u'keyword'),
-            name='keywords')
+            name='keywords',
+        )
+
+        location = django_filters.ChoiceFilter(
+            choices=locations(),
+            label=_(u'location'),
+            action=filter_m49,
+        )
+
+        organisation = django_filters.ChoiceFilter(
+            choices=organisations(),
+            label=_(u'organisation'),
+            name='partners__id',
+        )
+
+        class Meta:
+            model = Project
+            fields = [
+                'title',
+                'keyword',
+                'location',
+                'status',
+                'iati_status',
+                'organisation',
+                'category',
+                'sector',
+            ]
 
     return ProjectFilter
 
diff --git a/akvo/rsr/models/project.py b/akvo/rsr/models/project.py
--- a/akvo/rsr/models/project.py
+++ b/akvo/rsr/models/project.py
@@ -644,6 +644,18 @@ def set_reporting_org(self, organisation):
                 iati_organisation_role=Partnership.IATI_REPORTING_ORGANISATION
             )
 
+    def countries(self):
+        """Return a list of countries for the project."""
+
+        country_codes = set([c.country.lower() for c in self.recipient_countries.all()])
+        return (
+            [country for country in self.recipient_countries.all()] +
+            [
+                location.country for location in self.locations.all()
+                if location.country and location.country.iso_code not in country_codes
+            ]
+        )
+
     class QuerySet(DjangoQuerySet):
         def of_partner(self, organisation):
             "return projects that have organisation as partner"
@@ -953,6 +965,18 @@ def is_published(self):
         return False
     is_published.boolean = True
 
+    def publish(self):
+        """Set the publishing status to published."""
+
+        self.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED
+        self.publishingstatus.save()
+
+    def unpublish(self):
+        """Set the publishing status to unpublished."""
+
+        self.publishingstatus.status = PublishingStatus.STATUS_UNPUBLISHED
+        self.publishingstatus.save()
+
     def is_empty(self):
         exclude_fields = ['benchmarks', 'categories', 'created_at', 'crsadd', 'currency',
                           'custom_fields', 'fss', 'iati_checks', 'iati_project_exports',
diff --git a/akvo/rsr/views/project.py b/akvo/rsr/views/project.py
--- a/akvo/rsr/views/project.py
+++ b/akvo/rsr/views/project.py
@@ -7,7 +7,6 @@
 < http://www.gnu.org/licenses/agpl.html >.
 """
 
-import django_filters
 import json
 from datetime import datetime
 from sorl.thumbnail import get_thumbnail
@@ -18,16 +17,13 @@
 from django.core.urlresolvers import reverse
 from django.http import Http404, HttpResponse, HttpResponseRedirect
 from django.shortcuts import get_object_or_404, redirect, render
-from django.utils.translation import ugettext_lazy as _
 from lxml import etree
 
 from ..forms import ProjectUpdateForm
-from ..filters import (build_choices, location_choices, create_project_filter_class,
-                       remove_empty_querydict_items)
+from ..filters import (create_project_filter_class, remove_empty_querydict_items)
 from ..models import Project, ProjectUpdate
 from ...utils import pagination, filter_query_string
 from ...iati.exports.iati_export import IatiXML
-from .organisation import _page_organisations
 from akvo.codelists.models import SectorCategory, Sector, Version
 
 
@@ -65,19 +61,8 @@ def directory(request):
 
     # Yank project collection
     all_projects = _project_directory_coll(request)
-    f = create_project_filter_class(request)(qs, queryset=all_projects)
-
-    # Change filter options further when on an Akvo Page
-    if request.rsr_page:
-        # Filter location filter list to only populated locations
-        f.filters['location'].extra['choices'] = location_choices(all_projects)
-
-        # Swap to choice filter for RSR pages
-        f.filters['organisation'] = django_filters.ChoiceFilter(
-            choices=build_choices(_page_organisations(request.rsr_page)),
-            label=_(u'organisation'),
-            name='partners__id')
-
+    project_filter = create_project_filter_class(request, all_projects)
+    f = project_filter(qs, queryset=all_projects)
     sorted_projects = f.qs.distinct().order_by(sorting)
 
     # Build page
@@ -105,6 +90,8 @@ def directory(request):
     page.object_list = page.object_list.prefetch_related(
         'publishingstatus',
         'recipient_countries',
+        'locations',
+        'locations__country',
         'sectors',
         'budget_items',
         'partnerships__organisation',


Test patch:
diff --git a/akvo/rsr/tests/models/test_partner_site.py b/akvo/rsr/tests/models/test_partner_site.py
--- a/akvo/rsr/tests/models/test_partner_site.py
+++ b/akvo/rsr/tests/models/test_partner_site.py
@@ -8,7 +8,7 @@
 from django.test import TestCase
 
 from akvo.rsr.models import (
-    Organisation, Partnership, PartnerSite, Project, ProjectUpdate, PublishingStatus
+    Organisation, Partnership, PartnerSite, Project, ProjectUpdate
 )
 
 User = get_user_model()
@@ -23,11 +23,9 @@ def setUp(self):
 
         # Setup a project with results framework and a user
         self.project_1 = Project.objects.create(title="Test project 1")
-        self.project_1.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED
-        self.project_1.publishingstatus.save()
+        self.project_1.publish()
         self.project_2 = Project.objects.create(title="Test project 2")
-        self.project_2.publishingstatus.status = PublishingStatus.STATUS_PUBLISHED
-        self.project_2.publishingstatus.save()
+        self.project_2.publish()
         self.org_1 = Organisation.objects.create(name='Org 1', long_name='Organisation 1')
         self.org_2 = Organisation.objects.create(name='Org 2', long_name='Organisation 2')
         self.user = User.objects.create(username='user1@com.com', email='user1@com.com')
diff --git a/akvo/rsr/tests/views/test_account.py b/akvo/rsr/tests/views/test_account.py
--- a/akvo/rsr/tests/views/test_account.py
+++ b/akvo/rsr/tests/views/test_account.py
@@ -36,8 +36,7 @@ def setUp(self):
         self.org2 = Organisation.objects.create(name='icco', long_name='icco foundation')
         for org in (self.org1, self.org2):
             project = Project.objects.create()
-            project.publishingstatus.status = 'published'
-            project.publishingstatus.save()
+            project.publish()
             Partnership.objects.create(project=project, organisation=org)
         self.c = Client(HTTP_HOST=settings.RSR_DOMAIN)
 
diff --git a/akvo/rsr/tests/views/test_project.py b/akvo/rsr/tests/views/test_project.py
--- a/akvo/rsr/tests/views/test_project.py
+++ b/akvo/rsr/tests/views/test_project.py
@@ -15,7 +15,10 @@
 from django.conf import settings
 from django.test import Client, TestCase
 
-from akvo.rsr.models import Keyword, Organisation, PartnerSite, Partnership, Project
+from akvo.rsr.iso3166 import ISO_3166_COUNTRIES
+from akvo.rsr.models import (
+    Country, Keyword, Organisation, PartnerSite, Partnership, Project, ProjectLocation, RecipientCountry
+)
 
 
 @skip('Needs Django >= 1.8')
@@ -101,8 +104,7 @@ def test_should_show_keyword_projects_in_partner_site(self):
         project_title = '{} awesome project'.format(hostname)
         project = Project.objects.create(title=project_title)
         project.keywords.add(keyword)
-        project.publishingstatus.status = 'published'
-        project.publishingstatus.save()
+        project.publish()
         url = '/en/projects/'
         self.c = Client(HTTP_HOST='{}.{}'.format(hostname, settings.AKVOAPP_DOMAIN))
 
@@ -125,13 +127,11 @@ def test_should_show_all_partner_projects(self):
         )
         project_title1 = '{} awesome project {}'.format(hostname, 1)
         project1 = Project.objects.create(title=project_title1)
-        project1.publishingstatus.status = 'published'
-        project1.publishingstatus.save()
+        project1.publish()
         Partnership.objects.create(organisation=org, project=project1)
         project_title2 = '{} awesome project {}'.format(hostname, 2)
         project2 = Project.objects.create(title=project_title2)
-        project2.publishingstatus.status = 'published'
-        project2.publishingstatus.save()
+        project2.publish()
         Partnership.objects.create(organisation=org, project=project2)
         url = '/en/projects/'
         self.c = Client(HTTP_HOST='{}.{}'.format(hostname, settings.AKVOAPP_DOMAIN))
@@ -159,13 +159,11 @@ def test_should_show_partner_projects_with_keyword(self):
         project_title1 = '{} awesome project {}'.format(hostname, 1)
         project1 = Project.objects.create(title=project_title1)
         project1.keywords.add(keyword)
-        project1.publishingstatus.status = 'published'
-        project1.publishingstatus.save()
+        project1.publish()
         Partnership.objects.create(organisation=org, project=project1)
         project_title2 = '{} awesome project {}'.format(hostname, 2)
         project2 = Project.objects.create(title=project_title2)
-        project2.publishingstatus.status = 'published'
-        project2.publishingstatus.save()
+        project2.publish()
         Partnership.objects.create(organisation=org, project=project2)
         url = '/en/projects/'
         self.c = Client(HTTP_HOST='{}.{}'.format(hostname, settings.AKVOAPP_DOMAIN))
@@ -176,3 +174,43 @@ def test_should_show_partner_projects_with_keyword(self):
         # Then
         self.assertIn(project_title1, response.content)
         self.assertNotIn(project_title2, response.content)
+
+    def test_should_show_all_country_projects(self):
+        # Given
+        project_title1 = 'Project 1'
+        project_title2 = 'Project 2'
+        project_title3 = 'Project 3'
+        url = '/en/projects/?location=262'
+        latitude, longitude = ('11.8948112', '42.5807153')
+        country_code = 'DJ'
+        # No recipient country
+        project1 = Project.objects.create(title=project_title1)
+        project1.publish()
+        # Recipient Country - DJ
+        project2 = Project.objects.create(title=project_title2)
+        project2.publish()
+        RecipientCountry.objects.create(project=project2, country=country_code)
+        # ProjectLocation in DJ
+        self.setup_country_objects()
+        project3 = Project.objects.create(title=project_title3)
+        project3.publish()
+        project_location = ProjectLocation.objects.create(location_target=project3,
+                                                          latitude=latitude,
+                                                          longitude=longitude)
+        # ProjectLocation with no country
+        ProjectLocation.objects.create(location_target=project3,
+                                       latitude=None,
+                                       longitude=None)
+
+        # When
+        response = self.c.get(url, follow=True)
+
+        # Then
+        self.assertNotIn(project_title1, response.content)
+        self.assertIn(project_title2, response.content)
+        self.assertEqual(project_location.country.iso_code, country_code.lower())
+        self.assertIn(project_title3, response.content)
+
+    def setup_country_objects(self):
+        for iso_code, name in ISO_3166_COUNTRIES:
+            Country.objects.create(name=name, iso_code=iso_code)