Repository: akvo/akvo-rsr

Issue:
Disaggregations validation
The accumulated value of a disaggregation category should not be greater than the period update value.


Discussion:

Resolving patch:
diff --git a/akvo/rest/serializers/disaggregation.py b/akvo/rest/serializers/disaggregation.py
--- a/akvo/rest/serializers/disaggregation.py
+++ b/akvo/rest/serializers/disaggregation.py
@@ -37,6 +37,25 @@ class Meta:
             'last_modified_at',
         )
 
+    # NOTE: this validation should be necessary when doing operation directly on disaggregations endpoint but
+    # might cause problems when bulk update disaggregations values through the IndicatorPeriodDataFrameworkSerializer.
+    # One way to solve this it to not expose the disaggregations endpoint.
+    # def validate_value(self, value):
+    #     from django.db.models import Sum
+    #     from akvo.rsr.models import IndicatorPeriodData, IndicatorDimensionValue
+    #     data = self.get_initial()
+    #     update = IndicatorPeriodData.objects.get(pk=data['update'])
+    #     type = IndicatorDimensionValue.objects.prefetch_related('name').get(pk=data['dimension_value'])
+    #     category = type.name
+    #     subtotal = Disaggregation.objects\
+    #         .filter(update=update, dimension_value__name=category)\
+    #         .exclude(dimension_value=type)\
+    #         .aggregate(values=Sum('value')).get('values')
+    #     total = subtotal + value if subtotal else value
+    #     if total > update.value:
+    #         raise serializers.ValidationError("The accumulated disaggregations value should not exceed update value")
+    #     return value
+
 
 class DisaggregationReadOnlySerializer(BaseRSRSerializer):
 
diff --git a/akvo/rest/serializers/indicator_period_data.py b/akvo/rest/serializers/indicator_period_data.py
--- a/akvo/rest/serializers/indicator_period_data.py
+++ b/akvo/rest/serializers/indicator_period_data.py
@@ -4,12 +4,14 @@
 # See more details in the license.txt file located at the root folder of the Akvo RSR module.
 # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
 from rest_framework import serializers
+from django.db.models import Sum
 
 from akvo.rest.serializers.disaggregation import DisaggregationSerializer, DisaggregationReadOnlySerializer
 from akvo.rest.serializers.rsr_serializer import BaseRSRSerializer
 from akvo.rest.serializers.user import UserDetailsSerializer
 from akvo.rsr.models import (
     IndicatorPeriod, IndicatorPeriodData, IndicatorPeriodDataComment, IndicatorPeriodDataFile, IndicatorPeriodDataPhoto,
+    IndicatorDimensionValue, Disaggregation
 )
 from akvo.utils import ensure_decimal
 
@@ -95,6 +97,12 @@ class Meta:
         read_only_fields = ['user']
 
     def create(self, validated_data):
+        self._validate_disaggregations(
+            self._disaggregations_data,
+            value=ensure_decimal(validated_data.get('value', 0)),
+            numerator=ensure_decimal(validated_data.get('numerator', None)),
+            denominator=ensure_decimal(validated_data.get('denominator', None))
+        )
         """Over-ridden to handle nested writes."""
         files = validated_data.pop('files', [])
         photos = validated_data.pop('photos', [])
@@ -114,6 +122,13 @@ def create(self, validated_data):
         return update
 
     def update(self, instance, validated_data):
+        self._validate_disaggregations(
+            self._disaggregations_data,
+            value=ensure_decimal(validated_data.get('value', instance.value)),
+            numerator=ensure_decimal(validated_data.get('numerator', instance.numerator)),
+            denominator=ensure_decimal(validated_data.get('denominator', instance.denominator)),
+            update=instance
+        )
         """Over-ridden to handle nested updates."""
         files = validated_data.pop('files', [])
         photos = validated_data.pop('photos', [])
@@ -141,6 +156,29 @@ def update(self, instance, validated_data):
             'disaggregations',
         ).get(id=instance.id)
 
+    def _validate_disaggregations(self, disaggregations, value, numerator=None, denominator=None, update=None):
+        adjustments = {}
+        for disaggregation in disaggregations:
+            if denominator is not None:
+                disaggregation_denominator = ensure_decimal(disaggregation.get('denominator', 0))
+                if disaggregation_denominator > denominator:
+                    raise serializers.ValidationError("disaggregations denominator should not exceed update denominator")
+            category = IndicatorDimensionValue.objects.get(pk=disaggregation['dimension_value']).name
+            if category.id not in adjustments:
+                adjustments[category.id] = {'values': 0, 'numerators': 0, 'type_ids': []}
+            adjustments[category.id]['values'] += ensure_decimal(disaggregation.get('value', 0))
+            adjustments[category.id]['numerators'] += ensure_decimal(disaggregation.get('numerator', 0))
+            adjustments[category.id]['type_ids'].append(disaggregation['dimension_value'])
+        for key, adjustment in adjustments.items():
+            unmodifieds = Disaggregation.objects.filter(update=update, dimension_value__name=key)\
+                .exclude(dimension_value__in=adjustment['type_ids'])\
+                .aggregate(values=Sum('value'))
+            total = adjustment['values'] + ensure_decimal(unmodifieds['values'])
+            if numerator is not None and adjustment['numerators'] > numerator:
+                raise serializers.ValidationError("The disaggregation numerator should not exceed update numerator")
+            if total > value:
+                raise serializers.ValidationError("The accumulated disaggregations value should not exceed update value")
+
     def is_valid(self, raise_exception=False):
         # HACK to allow nested posting...
         self._disaggregations_data = self.initial_data.pop('disaggregations', [])


Test patch:
diff --git a/akvo/rsr/tests/rest/test_indicator_period_data.py b/akvo/rsr/tests/rest/test_indicator_period_data.py
--- a/akvo/rsr/tests/rest/test_indicator_period_data.py
+++ b/akvo/rsr/tests/rest/test_indicator_period_data.py
@@ -17,6 +17,7 @@
     IndicatorDimensionName, IndicatorDimensionValue,
     IndicatorPeriodData, IndicatorPeriodDataFile, IndicatorPeriodDataPhoto
 )
+from akvo.rsr.models.result.utils import QUANTITATIVE, PERCENTAGE_MEASURE
 from akvo.rsr.tests.base import BaseTestCase
 from akvo.rsr.tests.utils import ProjectFixtureBuilder
 
@@ -495,18 +496,268 @@ def setup_results_framework(self):
         )
 
 
-class IndicatorPeriodDataAttachmentsTestCase(BaseTestCase):
+def create_org_user(username='test@akvo.org', password='password', org='Acme'):
+    user = BaseTestCase.create_user(username, password)
+    org = BaseTestCase.create_organisation(org)
+    BaseTestCase.make_org_project_editor(user, org)
+    return org, user
+
+
+class IndicatorPeriodDataDisaggregationsValidationTestCase(BaseTestCase):
+    def test_unit_type_indicator_new_update_invalid_case(self):
+        # Given
+        username, password = 'test@akvo.org', 'password'
+        org, user = create_org_user(username, password)
+        project = ProjectFixtureBuilder()\
+            .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
+            .with_disaggregations({
+                'Gender': ['Male', 'Female'],
+            })\
+            .with_results([{
+                'title': 'Result  #1',
+                'indicators': [{
+                    'title': 'Indicator #1',
+                    'periods': [{
+                        'period_start': date(2010, 1, 1),
+                        'period_end': date(2010, 12, 31),
+                    }]
+                }]
+
+            }]).build()
+        period = project.get_period(period_start=date(2010, 1, 1))
+
+        # When
+        self.c.login(username=username, password=password)
+        url = '/rest/v1/indicator_period_data_framework/?format=json'
+        data = {
+            'period': period.id,
+            'value': 10,
+            'disaggregations': [
+                {'dimension_value': project.get_disaggregation('Gender', 'Male').id, 'value': 4},
+                {'dimension_value': project.get_disaggregation('Gender', 'Female').id, 'value': 7},
+            ]
+        }
+        response = self.c.post(url, data=json.dumps(data), content_type='application/json')
+
+        # Then
+        self.assertEqual(400, response.status_code)
+
+    def test_unit_type_indicator_edit_update_invalid_case(self):
+        # Given
+        username, password = 'test@akvo.org', 'password'
+        org, user = create_org_user(username, password)
+        project = ProjectFixtureBuilder()\
+            .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
+            .with_disaggregations({
+                'Gender': ['Male', 'Female'],
+            })\
+            .with_results([{
+                'title': 'Result  #1',
+                'indicators': [{
+                    'title': 'Indicator #1',
+                    'periods': [{
+                        'period_start': date(2010, 1, 1),
+                        'period_end': date(2010, 12, 31),
+                    }]
+                }]
+
+            }]).build()
+        period = project.get_period(period_start=date(2010, 1, 1))
+        update = period.add_update(user, value=10, disaggregations={
+            'Gender': {
+                'Male': {'value': 3},
+                'Female': {'value': 7},
+            }
+        })
+
+        # When
+        self.c.login(username=username, password=password)
+        url = '/rest/v1/indicator_period_data_framework/{}/?format=json'
+        data = {
+            'disaggregations': [
+                {'dimension_value': project.get_disaggregation('Gender', 'Male').id, 'value': 4},
+            ]
+        }
+        response = self.c.patch(url.format(update.id),
+                                data=json.dumps(data),
+                                content_type='application/json')
+
+        # Then
+        self.assertEqual(400, response.status_code)
+
+    def test_percentage_type_indicator_new_update_invalid_denominator_case(self):
+        # Given
+        username, password = 'test@akvo.org', 'password'
+        org, user = create_org_user(username, password)
+        project = ProjectFixtureBuilder()\
+            .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
+            .with_disaggregations({
+                'Gender': ['Male', 'Female'],
+            })\
+            .with_results([{
+                'title': 'Result  #1',
+                'indicators': [{
+                    'title': 'Indicator #1',
+                    'type': QUANTITATIVE,
+                    'measure': PERCENTAGE_MEASURE,
+                    'periods': [{
+                        'period_start': date(2010, 1, 1),
+                        'period_end': date(2010, 12, 31),
+                    }]
+                }]
+
+            }]).build()
+        period = project.get_period(period_start=date(2010, 1, 1))
+
+        # When
+        self.c.login(username=username, password=password)
+        url = '/rest/v1/indicator_period_data_framework/?format=json'
+        data = {
+            'period': period.id,
+            'denominator': 10,
+            'disaggregations': [
+                {'dimension_value': project.get_disaggregation('Gender', 'Male').id, 'denominator': 11},
+            ]
+        }
+        response = self.c.post(url, data=json.dumps(data), content_type='application/json')
+
+        # Then
+        self.assertEqual(400, response.status_code)
 
-    def create_org_user(self, username='test@akvo.org', password='password', org='Acme Org'):
-        user = self.create_user(username, password)
-        org = self.create_organisation(org)
-        self.make_org_project_editor(user, org)
-        return org, user
+    def test_percentage_type_indicator_edit_update_invalid_denominator_case(self):
+        # Given
+        username, password = 'test@akvo.org', 'password'
+        org, user = create_org_user(username, password)
+        project = ProjectFixtureBuilder()\
+            .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
+            .with_disaggregations({
+                'Gender': ['Male', 'Female'],
+            })\
+            .with_results([{
+                'title': 'Result  #1',
+                'indicators': [{
+                    'title': 'Indicator #1',
+                    'type': QUANTITATIVE,
+                    'measure': PERCENTAGE_MEASURE,
+                    'periods': [{
+                        'period_start': date(2010, 1, 1),
+                        'period_end': date(2010, 12, 31),
+                    }]
+                }]
+
+            }]).build()
+        period = project.get_period(period_start=date(2010, 1, 1))
+        update = period.add_update(user, denominator=10, disaggregations={
+            'Gender': {
+                'Male': {'denominator': 10},
+            }
+        })
+
+        # When
+        self.c.login(username=username, password=password)
+        url = '/rest/v1/indicator_period_data_framework/{}/?format=json'
+        data = {
+            'disaggregations': [
+                {'dimension_value': project.get_disaggregation('Gender', 'Male').id, 'denominator': 11},
+            ]
+        }
+        response = self.c.patch(url.format(update.id),
+                                data=json.dumps(data),
+                                content_type='application/json')
+
+        # Then
+        self.assertEqual(400, response.status_code)
+
+    def test_percentage_type_indicator_new_update_invalid_numerator_case(self):
+        # Given
+        username, password = 'test@akvo.org', 'password'
+        org, user = create_org_user(username, password)
+        project = ProjectFixtureBuilder()\
+            .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
+            .with_disaggregations({
+                'Gender': ['Male', 'Female'],
+            })\
+            .with_results([{
+                'title': 'Result  #1',
+                'indicators': [{
+                    'title': 'Indicator #1',
+                    'type': QUANTITATIVE,
+                    'measure': PERCENTAGE_MEASURE,
+                    'periods': [{
+                        'period_start': date(2010, 1, 1),
+                        'period_end': date(2010, 12, 31),
+                    }]
+                }]
+
+            }]).build()
+        period = project.get_period(period_start=date(2010, 1, 1))
+
+        # When
+        self.c.login(username=username, password=password)
+        url = '/rest/v1/indicator_period_data_framework/?format=json'
+        data = {
+            'period': period.id,
+            'numerator': 10,
+            'disaggregations': [
+                {'dimension_value': project.get_disaggregation('Gender', 'Male').id, 'numerator': 11},
+            ]
+        }
+        response = self.c.post(url, data=json.dumps(data), content_type='application/json')
+
+        # Then
+        self.assertEqual(400, response.status_code)
+
+    def test_percentage_type_indicator_edit_update_invalid_numerator_case(self):
+        # Given
+        username, password = 'test@akvo.org', 'password'
+        org, user = create_org_user(username, password)
+        project = ProjectFixtureBuilder()\
+            .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
+            .with_disaggregations({
+                'Gender': ['Male', 'Female'],
+            })\
+            .with_results([{
+                'title': 'Result  #1',
+                'indicators': [{
+                    'title': 'Indicator #1',
+                    'type': QUANTITATIVE,
+                    'measure': PERCENTAGE_MEASURE,
+                    'periods': [{
+                        'period_start': date(2010, 1, 1),
+                        'period_end': date(2010, 12, 31),
+                    }]
+                }]
+
+            }]).build()
+        period = project.get_period(period_start=date(2010, 1, 1))
+        update = period.add_update(user, denominator=10, disaggregations={
+            'Gender': {
+                'Male': {'numerator': 10},
+            }
+        })
+
+        # When
+        self.c.login(username=username, password=password)
+        url = '/rest/v1/indicator_period_data_framework/{}/?format=json'
+        data = {
+            'disaggregations': [
+                {'dimension_value': project.get_disaggregation('Gender', 'Male').id, 'numerator': 11},
+            ]
+        }
+        response = self.c.patch(url.format(update.id),
+                                data=json.dumps(data),
+                                content_type='application/json')
+
+        # Then
+        self.assertEqual(400, response.status_code)
+
+
+class IndicatorPeriodDataAttachmentsTestCase(BaseTestCase):
 
     def test_create_update_with_multiple_photo_and_file(self):
         # Given
         username, password = 'test@akvo.org', 'password'
-        org, user = self.create_org_user(username, password)
+        org, user = create_org_user(username, password)
         project = ProjectFixtureBuilder()\
             .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
             .with_results([{
@@ -544,7 +795,7 @@ def test_create_update_with_multiple_photo_and_file(self):
     def test_add_new_files_to_an_update(self):
         # Given
         username, password = 'test@akvo.org', 'password'
-        org, user = self.create_org_user(username, password)
+        org, user = create_org_user(username, password)
         project = ProjectFixtureBuilder()\
             .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
             .with_results([{
@@ -577,7 +828,7 @@ def test_add_new_files_to_an_update(self):
 
     def test_add_new_file_only_for_update_creator(self):
         # Given
-        org, creator = self.create_org_user()
+        org, creator = create_org_user()
         project = ProjectFixtureBuilder()\
             .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
             .with_results([{
@@ -611,7 +862,7 @@ def test_add_new_file_only_for_update_creator(self):
     def test_add_new_photos_to_an_update(self):
         # Given
         username, password = 'test@akvo.org', 'password'
-        org, user = self.create_org_user(username, password)
+        org, user = create_org_user(username, password)
         project = ProjectFixtureBuilder()\
             .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
             .with_results([{
@@ -647,7 +898,7 @@ def test_add_new_photos_to_an_update(self):
 
     def test_add_new_photo_only_for_update_creator(self):
         # Given
-        org, creator = self.create_org_user()
+        org, creator = create_org_user()
         project = ProjectFixtureBuilder()\
             .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
             .with_results([{
@@ -684,7 +935,7 @@ def test_add_new_photo_only_for_update_creator(self):
     def test_remove_a_file_from_an_update(self):
         # Given
         username, password = 'test@akvo.org', 'password'
-        org, user = self.create_org_user(username, password)
+        org, user = create_org_user(username, password)
         project = ProjectFixtureBuilder()\
             .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
             .with_results([{
@@ -716,7 +967,7 @@ def test_remove_a_file_from_an_update(self):
     def test_remove_a_photo_from_an_update(self):
         # Given
         username, password = 'test@akvo.org', 'password'
-        org, user = self.create_org_user(username, password)
+        org, user = create_org_user(username, password)
         project = ProjectFixtureBuilder()\
             .with_partner(org, Partnership.IATI_REPORTING_ORGANISATION)\
             .with_results([{