Repository: kartoza/prj.app

Issue:
worksheet links can become stale or unavailable and should be manually fixed
# Problem

All lessons have further reading sections where we put links. When the links become unavailable the reading section becomes stale.

# Solution

I suggest a little tool or script that basically generates all invalid links in a PDF/text file like

```
The following links are unavailable
https://changelog.qgis.org/en/qgis/lesson/symbology-22/detail/81/?q=3.1 has invalid links or unavailable links:
* http://foo.bar.txt
```
Or just saying
```
The following sections have invalid links
https://changelog.qgis.org/en/qgis/lesson/symbology-22/detail/81/?q=3.1
https://changelog.qgis.org/en/qgis/lesson/symbology-22/detail/167/?q=3.3
```

This tool can be run per need and should ideally be visible to the lessons manager.

Use case:
Before each training session, we run the tool and manually fix the links in the respective sections



Discussion:

Resolving patch:
diff --git a/django_project/lesson/urls.py b/django_project/lesson/urls.py
--- a/django_project/lesson/urls.py
+++ b/django_project/lesson/urls.py
@@ -22,6 +22,8 @@
     FurtherReadingCreateView,
     FurtherReadingDeleteView,
     FurtherReadingUpdateView,
+    get_invalid_FurtherReading_links,
+    print_invalid_FurterReading_links
 )
 from lesson.views.section import (
     SectionCreateView,
@@ -218,6 +220,15 @@
               'delete/(?P<pk>[\w-]+)/$',
         view=AnswerDeleteView.as_view(),
         name='answer-delete'),
+
+    # Json invalid Further reading Link
+    url(regex='(?P<project_slug>[\w-]+)/lessons/invalid_further_reading/$',
+        view=get_invalid_FurtherReading_links,
+        name='invalid-further-reading-links'),
+    url(regex='(?P<project_slug>[\w-]+)/lessons/'
+              'print_invalid_further_reading/$',
+        view=print_invalid_FurterReading_links,
+        name='print_invalid-further-reading-links')
 ]
 
 # The original urlpatterns below will redirect to urlpatterns above
diff --git a/django_project/lesson/utilities.py b/django_project/lesson/utilities.py
--- a/django_project/lesson/utilities.py
+++ b/django_project/lesson/utilities.py
@@ -2,9 +2,14 @@
 """Tools for the lesson app."""
 
 import json
+import re
+import requests
+
 from unidecode import unidecode
+from urllib.parse import urlparse
 
 from django.http import Http404, HttpResponse
+from django.urls import reverse
 from django.utils.text import slugify
 
 from core.settings.contrib import STOP_WORDS
@@ -60,3 +65,83 @@ def re_order_features(request, features):
             feature.save()
 
     return HttpResponse('')
+
+
+class GetInvalidFurtherReadingLink(object):
+    def __init__(self, project):
+        self.project = project
+
+    def get_all_invalid_url(self):
+        from lesson.models.worksheet import Worksheet
+        worksheets = Worksheet.objects.all().filter(
+            section__project=self.project,
+            furtherreading__isnull=False
+        )
+        result = []
+        for worksheet in worksheets:
+            worksheet_url = self.get_worksheet_url(
+                worksheet_pk=worksheet.id,
+                section_slug=worksheet.section.slug,
+                project_slug=worksheet.section.project.slug
+            )
+            further_reading = worksheet.furtherreading_set.all()
+            for obj in further_reading:
+                urls = self.get_url_list(obj.text)
+                for url in urls:
+                    ctx = {
+                        'worksheet_url': worksheet_url,
+                        'worksheet': worksheet.module,
+                        'invalid_url': url
+                    }
+                    if ctx in result:
+                        continue
+                    invalid_url = self.check_if_url_invalid(url)
+                    if invalid_url:
+                        result.append(
+                            ctx
+                        )
+        return result
+
+    def get_url_list(self, text):
+        # https://www.geeksforgeeks.org/python-check-url-string/
+        regex = (
+            r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)"
+            r"(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+"
+            r"(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)"
+            r"|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))")
+        url = re.findall(regex, text)
+        result = []
+        for x in url:
+            _ = x[0].replace('&nbsp', '')
+            _ = _.replace('&amp', '')
+            result.append(_)
+        return result
+
+    def check_if_url_invalid(self, url):
+        """Return invalid url"""
+
+        try:
+            parsed_url = urlparse(url)  # e.g https://plugins.qgis.org/
+            if not (all([parsed_url.scheme,  # e.g http
+                         parsed_url.netloc])):  # e.g www.qgis.org
+                return url
+        except Exception:
+            return url
+
+        # Check if url is exist
+        try:
+            req = requests.head(url)
+        except requests.exceptions.SSLError:
+            req = requests.head(url, verify=False)
+        except Exception:
+            return url
+        if req.status_code >= 400:
+            return url
+        return None
+
+    def get_worksheet_url(self, worksheet_pk, section_slug, project_slug):
+        return reverse('worksheet-detail', kwargs={
+            'pk': worksheet_pk,
+            'section_slug': section_slug,
+            'project_slug': project_slug
+        })
diff --git a/django_project/lesson/views/further_reading.py b/django_project/lesson/views/further_reading.py
--- a/django_project/lesson/views/further_reading.py
+++ b/django_project/lesson/views/further_reading.py
@@ -1,6 +1,9 @@
 # coding=utf-8
 """Further reading views."""
 
+import json
+
+from django.http import JsonResponse, HttpResponse
 from django.urls import reverse
 from django.views.generic import (
     CreateView,
@@ -12,9 +15,11 @@
 
 from braces.views import LoginRequiredMixin
 
+from base.models.project import Project
 from lesson.models.further_reading import FurtherReading
 from lesson.models.worksheet import Worksheet
 from lesson.forms.further_reading import FurtherReadingForm
+from lesson.utilities import GetInvalidFurtherReadingLink
 
 
 class FurtherReadingMixin(object):
@@ -120,3 +125,41 @@ def get_success_url(self):
             'section_slug': self.object.worksheet.section.slug,
             'project_slug': self.object.worksheet.section.project.slug
         })
+
+
+def get_invalid_FurtherReading_links(request, **kwargs):
+
+    project_slug = kwargs.get('project_slug', None)
+    if not project_slug:
+        return JsonResponse({'data': None})
+
+    project = Project.objects.get(slug=project_slug)
+    if not project:
+        return JsonResponse({'data': None})
+
+    invalid_links_list = GetInvalidFurtherReadingLink(
+        project).get_all_invalid_url()
+
+    return JsonResponse({
+        'data': invalid_links_list,
+        'project_slug': project_slug,
+        'project_name': project.name
+    })
+
+
+def print_invalid_FurterReading_links(request, **kwargs):
+    project_slug = kwargs.get('project_slug', None)
+    data = json.loads(request.GET.get('data'))
+
+    from changes.utils.render_to_pdf import render_to_pdf
+    pdf = render_to_pdf(
+        'further_reading/print_invalid_links.html', data)
+
+    response = HttpResponse(pdf, content_type='application/pdf')
+    filename = f'Invalid_FurtherReading_{project_slug}.pdf'
+    content = "inline; filename='%s'" % (filename)
+    download = request.GET.get("download")
+    if download:
+        content = "attachment; filename='%s'" % (filename)
+    response['Content-Disposition'] = content
+    return response


Test patch:
diff --git a/django_project/lesson/tests/model_factories.py b/django_project/lesson/tests/model_factories.py
--- a/django_project/lesson/tests/model_factories.py
+++ b/django_project/lesson/tests/model_factories.py
@@ -6,6 +6,7 @@
 from base.tests.model_factories import ProjectF
 
 from lesson.models import (
+    FurtherReading,
     License,
     Section,
     Specification,
@@ -76,3 +77,14 @@ class Meta:
 
     name = 'Test License'
     file = factory.django.FileField(byte=b'test license content')
+
+
+class FurtherReadingF(factory.django.DjangoModelFactory):
+    """Further Reading model factory."""
+
+    class Meta:
+        model = FurtherReading
+
+    text = ('Test Further Reading. <a href="https://github.com/kartoza/'
+            'prj.app">Projecta Repository</a>')
+    worksheet = factory.SubFactory(WorksheetF)
diff --git a/django_project/lesson/tests/test_utilities.py b/django_project/lesson/tests/test_utilities.py
new file mode 100644
--- /dev/null
+++ b/django_project/lesson/tests/test_utilities.py
@@ -0,0 +1,95 @@
+# coding=utf-8
+"""Test for lesson utilities."""
+
+from django.test import TestCase
+
+from lesson.tests.model_factories import (FurtherReadingF,
+                                          SectionF,
+                                          WorksheetF)
+from base.tests.model_factories import ProjectF
+
+from lesson.utilities import GetInvalidFurtherReadingLink
+
+
+def side_effect_build_absolute_uri(value):
+    """Mocking build_absolute_uri side_effect.
+
+    Return the value which is the relative path instead of absolute path
+    """
+    return value
+
+
+class InvalidFurtherReadingURL(TestCase):
+
+    def setUp(self):
+        # Create project
+        self.test_project = ProjectF.create(
+            name = 'Test project'
+        )
+
+        # Create section
+        self.test_section = SectionF.create(
+            project=self.test_project,
+            name='Test section'
+        )
+        self.test_worksheet = WorksheetF.create(
+            section=self.test_section,
+            published=True,
+            module='Test Invalid Link'
+        )
+        self.kwargs_project = {'project_slug': self.test_section.project.slug}
+        self.kwargs_section = {'slug': self.test_section.slug}
+        self.kwargs_section_full = {
+            'project_slug': self.test_section.project.slug,
+            'slug': self.test_section.slug
+        }
+        self.kwargs_worksheet_full = {
+            'project_slug': self.test_section.project.slug,
+            'section_slug': self.test_section.slug,
+            'pk': self.test_worksheet.pk
+        }
+        self.test_further_reading = FurtherReadingF.create(
+            worksheet=self.test_worksheet)
+
+    def test_get_further_reading_link_list(self):
+        self.test_further_reading.text = (
+            'Test for link: https://changelog.kartoza.com/en/ '
+            'and '
+            'https://changelog.qgis.org/en/qgis/lessons/#introduction-qgis-1')
+        self.test_further_reading.save()
+        obj = GetInvalidFurtherReadingLink(self.test_project)
+        result = obj.get_url_list(self.test_further_reading.text)
+        self.assertEqual(
+            result,
+            ['https://changelog.kartoza.com/en/',
+             'https://changelog.qgis.org/en/qgis/lessons/#introduction-qgis-1']
+        )
+
+    def test_check_if_url_invalid_return_none(self):
+        url = 'http://www.example.com'
+        obj = GetInvalidFurtherReadingLink(self.test_project)
+        result = obj.check_if_url_invalid(url)
+        self.assertFalse(result)
+
+    def test_check_if_url_invalid_return_not_none(self):
+        obj = GetInvalidFurtherReadingLink(self.test_project)
+        url = 'http://www .example.com'
+        result = obj.check_if_url_invalid(url)
+        self.assertTrue(result)
+        self.assertEqual(result, url)
+        url = 'http://www.example.com/this-page-is-not-exist/'
+        result = obj.check_if_url_invalid(url)
+        self.assertTrue(result)
+        self.assertEqual(result, url)
+
+    def test_get_all_invalid_url(self):
+        self.test_further_reading.text = (
+            'Test for invalid link: '
+            'http://www.example.com/this-page-is-not-exist/')
+        self.test_further_reading.save()
+        obj = GetInvalidFurtherReadingLink(self.test_project)
+        result = obj.get_all_invalid_url()
+        self.assertEqual(result[0]['worksheet'], 'Test Invalid Link')
+        self.assertEqual(
+            result[0]['invalid_url'],
+            'http://www.example.com/this-page-is-not-exist/')