Repository: DjangoGirls/djangogirls

Issue:
API for pulling stories from blog to website
Currently, our blog posts are published on Tumblr and have to be manually added to the Story model for them to appear on the website. It would be great to have an API do that for us so that we don't have outdated posts listed on the website because someone forgot to update the model after publishing a new post.

Tumblr has an [API Documentation](https://www.tumblr.com/docs/en/api/v2) that could be explored and an API developed to make this work.


Discussion:
I can probably take this one - would you mind registering an application under the django girls Tumblr account (https://www.tumblr.com/oauth/apps) and sharing keys (or sandbox keys if they exist) with me through Slack?
Thanks @ramonsaraiva for picking this up.

Resolving patch:
diff --git a/core/tumblr_client.py b/core/tumblr_client.py
new file mode 100644
--- /dev/null
+++ b/core/tumblr_client.py
@@ -0,0 +1,91 @@
+import logging
+from dataclasses import dataclass
+import datetime
+from typing import (
+    Any,
+    Iterator,
+    Optional,
+)
+
+import requests
+from requests.exceptions import RequestException
+
+from django.conf import settings
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class RemoteStory:
+    url: str
+    content_parts: list[dict[str, Any]]
+    created: datetime.datetime
+
+    @property
+    def text_parts(self) -> list[str]:
+        return [
+            content_part["text"]
+            for content_part in self.content_parts
+            if content_part["type"] == "text"
+        ]
+
+    @property
+    def image_parts(self) -> list[dict]:
+        return [
+            content_part["media"]
+            for content_part in self.content_parts
+            if content_part["type"] == "image"
+        ]
+
+    @property
+    def banner_url(self) -> Optional[str]:
+        if not self.image_parts:
+            return None
+        # Each image part has multiple image sizes, we're picking the first
+        return self.image_parts[0][0]["url"]
+
+    @property
+    def title(self) -> str:
+        return self.text_parts[0]
+
+    @property
+    def content(self) -> str:
+        return " ".join(self.text_parts[1:])
+
+    @property
+    def is_story(self) -> bool:
+        return "Your Django Story" in self.title
+
+
+def parse_posts(posts_data: list[dict[str, Any]]) -> Iterator[RemoteStory]:
+    for post_data in posts_data:
+        yield RemoteStory(
+            url=post_data["post_url"],
+            content_parts=post_data["content"],
+            created=datetime.datetime.utcfromtimestamp(post_data["timestamp"]),
+        )
+
+
+def request_latest_stories() -> Iterator[RemoteStory]:
+    # TODO: probably write a wrapper for tumblr requests if
+    # more endpoints are used in the future
+    blog_base_url = (
+        f"{settings.TUMBLR_API_BASE_URL}/blog/{settings.TUMBLR_BLOG_HOSTNAME}"
+    )
+    posts_endpoint = f"{blog_base_url}/posts/"
+    try:
+        # `npf` means Neue Post Format - so we don't get different response
+        # outputs for legacy versus `NPF` posts
+        response = requests.get(
+            posts_endpoint,
+            params={"api_key": settings.TUMBLR_API_KEY, "npf": True},
+            timeout=5,
+        )
+        response.raise_for_status()
+    except RequestException:
+        logger.exception("Exception when requesting tumblr posts")
+        return iter(())
+
+    posts_data = response.json()["response"]["posts"]
+    return parse_posts(posts_data)
diff --git a/djangogirls/settings.py b/djangogirls/settings.py
--- a/djangogirls/settings.py
+++ b/djangogirls/settings.py
@@ -274,3 +274,7 @@ def gettext(s):
 RECAPTCHA_TESTING = os.environ.get("RECAPTCHA_TESTING") == 'True'
 if RECAPTCHA_TESTING:
     SILENCED_SYSTEM_CHECKS = ['captcha.recaptcha_test_key_error']
+
+TUMBLR_API_KEY = os.environ.get("TUMBLR_API_KEY")
+TUMBLR_API_BASE_URL = os.environ.get("TUMBLR_API_BASE_URL", "https://api.tumblr.com/v2")
+TUMBLR_BLOG_HOSTNAME = os.environ.get("TUMBLR_BLOG_HOSTNAME", "blog.djangogirls.org")
diff --git a/story/management/commands/fetch_tumblr_stories.py b/story/management/commands/fetch_tumblr_stories.py
new file mode 100644
--- /dev/null
+++ b/story/management/commands/fetch_tumblr_stories.py
@@ -0,0 +1,51 @@
+from typing import Tuple
+from urllib.parse import urlparse
+from urllib.request import urlretrieve
+
+from django.core.files import File
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from core.tumblr_client import request_latest_stories
+from story.models import Story
+
+
+def download_image(url: str) -> Tuple[str, File]:
+    image_name = urlparse(url).path.split("/")[-1]
+    content = urlretrieve(url)
+    return image_name, File(open(content[0], "rb"))
+
+
+class Command(BaseCommand):
+    help = "Fetch Django Girls stories from Tumblr blog"
+
+    def handle(self, *args, **options):
+        latest_stories = list(request_latest_stories())
+        remote_urls = set(remote_story.url for remote_story in latest_stories)
+        in_db_urls = set(
+            Story.objects.filter(post_url__in=remote_urls).values_list(
+                "post_url", flat=True
+            )
+        )
+        to_create_urls = remote_urls.difference(in_db_urls)
+        missing_stories = [
+            remote_story
+            for remote_story in latest_stories
+            if remote_story.url in to_create_urls
+        ]
+        created = 0
+        for missing_story in missing_stories:
+            self.stdout.write(f"Fetching {missing_story.title}")
+            with transaction.atomic():
+                story = Story.objects.create(
+                    post_url=missing_story.url,
+                    name=missing_story.title,
+                    content=missing_story.content,
+                    is_story=missing_story.is_story,
+                )
+                story.created = missing_story.created
+                story.save()
+                if missing_story.is_story and missing_story.banner_url:
+                    story.image.save(*download_image(missing_story.banner_url), save=True)
+            created += 1
+        self.stdout.write(f"{created} stories loaded and created from Tumblr blog")


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,8 +1,11 @@
+from datetime import datetime
+
 import pytest
 
 from django.contrib.auth.models import Permission, Group
 
 from core.models import Event, User
+from core.tumblr_client import RemoteStory
 from pictures.models import StockPicture
 from sponsor.models import Donor
 
@@ -235,3 +238,19 @@ def hidden_donors():
         Donor(name="Tanaka", amount=100, visible=False)
     ])
     return hidden_donors
+
+
+@pytest.fixture()
+def remote_story():
+    return RemoteStory(
+        url="tumblr.com/stories/1",
+        content_parts=[
+            {"text": "This is a story text", "type": "text"},
+            {"text": "This is another story text", "type": "text"},
+            {
+                "media": [{"url": "/media-small"}, {"url": "/media-large"}],
+                "type": "image",
+            },
+        ],
+        created=datetime.utcnow(),
+    )
diff --git a/tests/core/test_tumblr_client.py b/tests/core/test_tumblr_client.py
new file mode 100644
--- /dev/null
+++ b/tests/core/test_tumblr_client.py
@@ -0,0 +1,91 @@
+from typing import Any
+from unittest import mock
+
+from requests.exceptions import RequestException
+
+from core import tumblr_client
+
+
+def test_remote_story_text_parts(remote_story: tumblr_client.RemoteStory):
+    assert len(remote_story.text_parts) == 2
+    assert remote_story.text_parts[0] == "This is a story text"
+    assert remote_story.text_parts[1] == "This is another story text"
+
+
+def test_remote_story_image_parts(remote_story: tumblr_client.RemoteStory):
+    assert len(remote_story.image_parts) == 1
+    assert remote_story.image_parts[0][0]["url"] == "/media-small"
+    assert remote_story.image_parts[0][1]["url"] == "/media-large"
+
+
+def test_remote_story_banner_url_no_image_parts(
+    remote_story: tumblr_client.RemoteStory,
+):
+    remote_story.content_parts = []
+    assert remote_story.banner_url is None
+
+
+def test_remote_story_banner_url(
+    remote_story: tumblr_client.RemoteStory,
+):
+    assert remote_story.banner_url == "/media-small"
+
+
+def test_remote_story_title(
+    remote_story: tumblr_client.RemoteStory,
+):
+    assert remote_story.title == "This is a story text"
+
+
+def test_remote_story_content(
+    remote_story: tumblr_client.RemoteStory,
+):
+    assert remote_story.content == "This is another story text"
+
+
+def test_remote_story_is_story_not_story(
+    remote_story: tumblr_client.RemoteStory,
+):
+    assert not remote_story.is_story
+
+
+def test_remote_story_is_story_story(
+    remote_story: tumblr_client.RemoteStory,
+):
+    remote_story.content_parts[0] = {"type": "text", "text": "Your Django Story"}
+    assert remote_story.is_story
+
+
+def test_parse_posts_no_post_data():
+    assert len(list(tumblr_client.parse_posts([]))) == 0
+
+
+def test_parse_posts():
+    posts_data: list[dict[str, Any]] = [
+        {
+            "post_url": "tumblr.com/stories/1",
+            "content": [
+                {"type": "text", "text": "Hello"},
+                {"type": "text", "text": "there"},
+            ],
+            "timestamp": 1663437643,
+        }
+    ]
+    remote_story = next(tumblr_client.parse_posts(posts_data))
+    assert isinstance(remote_story, tumblr_client.RemoteStory)
+
+
+@mock.patch("requests.get")
+@mock.patch("core.tumblr_client.logger.exception")
+def test_request_latest_stories_exception(p_logger_exception, p_get):
+    p_get.side_effect = RequestException()
+    assert len(list(tumblr_client.request_latest_stories())) == 0
+    p_logger_exception.assert_called_once_with("Exception when requesting tumblr posts")
+
+
+@mock.patch("requests.get")
+def test_request_latest_stories(p_get):
+    response = mock.Mock()
+    response.json = lambda: {"response": {"posts": []}}
+    p_get.return_value = response
+    assert len(list(tumblr_client.request_latest_stories())) == 0
diff --git a/tests/story/test_commands.py b/tests/story/test_commands.py
new file mode 100644
--- /dev/null
+++ b/tests/story/test_commands.py
@@ -0,0 +1,59 @@
+from unittest import mock
+
+from django.core.files import File
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+from story.management.commands import fetch_tumblr_stories
+from story.models import Story
+
+
+@mock.patch("story.management.commands.fetch_tumblr_stories.urlretrieve")
+def test_fetch_tumblr_stories_download_image(p_urlretrieve):
+    with mock.patch(
+        "story.management.commands.fetch_tumblr_stories.open",
+        mock.mock_open(read_data=""),
+    ):
+        image_name, image_file = fetch_tumblr_stories.download_image(
+            "tumblr.com/image.jpg"
+        )
+    p_urlretrieve.assert_called_once_with("tumblr.com/image.jpg")
+    assert image_name == "image.jpg"
+    assert isinstance(image_file, File)
+
+
+@mock.patch("story.management.commands.fetch_tumblr_stories.request_latest_stories")
+@mock.patch("story.management.commands.fetch_tumblr_stories.download_image")
+def test_fetch_tumblr_stories_command_existing_stories(
+    p_download_image, p_request_latest_stories, remote_story
+):
+    Story.objects.create(
+        post_url=remote_story.url,
+        name=remote_story.title,
+        content=remote_story.content,
+        is_story=remote_story.is_story,
+    )
+    p_request_latest_stories.return_value = [remote_story]
+    temp_file = SimpleUploadedFile("image.jpg", "")
+    p_download_image.return_value = (temp_file.name, temp_file)
+    command = fetch_tumblr_stories.Command()
+    command.handle()
+    assert Story.objects.count() == 1
+
+
+@mock.patch("story.management.commands.fetch_tumblr_stories.request_latest_stories")
+@mock.patch("story.management.commands.fetch_tumblr_stories.download_image")
+def test_fetch_tumblr_stories_command_new_stories(
+    p_download_image, p_request_latest_stories, remote_story
+):
+    Story.objects.create(
+        post_url="tumblr.com/stories/99999",
+        name=remote_story.title,
+        content=remote_story.content,
+        is_story=remote_story.is_story,
+    )
+    p_request_latest_stories.return_value = [remote_story]
+    temp_file = SimpleUploadedFile("image.jpg", "")
+    p_download_image.return_value = (temp_file.name, temp_file)
+    command = fetch_tumblr_stories.Command()
+    command.handle()
+    assert Story.objects.count() == 2