Repository: LibraryOfCongress/concordia

Issue:
Refactor exporter - look at past tickets and start prelim tests



Discussion:
Do we want to have streaming bag creation as a goal for this? I'm thinking that might be more hassle than its worth we just use another S3 bucket to assemble images but I think that also raises the question of whether we should bother with media at all vs. just the metadata.
For now, we just need the transcription data. No images yet.
That'll keep file sizes manageable — we probably just just be storing IDs & URLs anyway. One thing I remember from the last time I looked at the code was that it didn't seem well-defined how we handled things which weren't complete. Do we have a policy decision from @colefemeno about whether that means we need to exclude uncompleted items, expose the status, etc.?

Resolving patch:
diff --git a/exporter/views.py b/exporter/views.py
--- a/exporter/views.py
+++ b/exporter/views.py
@@ -1,4 +1,5 @@
 import os
+import re
 import shutil
 
 import bagit
@@ -10,29 +11,44 @@
 from django.views.generic import TemplateView
 from tabular_export.core import export_to_csv_response, flatten_queryset
 
-from concordia.models import Asset, Campaign, Transcription
-from concordia.storage import ASSET_STORAGE
+from concordia.models import Asset, Transcription
+
+
+def get_latest_transcription_data(campaign_slug):
+    latest_trans_subquery = (
+        Transcription.objects.filter(asset=OuterRef("pk"))
+        .order_by("-pk")
+        .values("text")
+    )
+    assets = Asset.objects.annotate(
+        latest_transcription=Subquery(latest_trans_subquery[:1])
+    )
+    assets = assets.filter(item__project__campaign__slug=campaign_slug)
+    return assets
+
+
+def get_original_asset_id(download_url):
+    """
+    Extract the bit from the download url
+    that identifies this image uniquely on loc.gov
+    """
+    if download_url.startswith("http://tile.loc.gov/"):
+        pattern = r"/service:([A-Za-z0-9:]*)/"
+        asset_id = re.search(pattern, download_url)
+        assert asset_id
+        return asset_id.group(1).replace(":", "-")
+    else:
+        return download_url
 
 
 class ExportCampaignToCSV(TemplateView):
     """
     Exports the most recent transcription for each asset in a campaign
-
     """
 
     @method_decorator(login_required)
     def get(self, request, *args, **kwargs):
-        latest_trans_subquery = (
-            Transcription.objects.filter(asset=OuterRef("pk"))
-            .order_by("-pk")
-            .values("text")
-        )
-        assets = Asset.objects.annotate(
-            latest_transcription=Subquery(latest_trans_subquery[:1])
-        )
-        assets = assets.filter(
-            item__project__campaign__slug=self.kwargs["campaign_slug"]
-        )
+        assets = get_latest_transcription_data(self.kwargs["campaign_slug"])
 
         headers, data = flatten_queryset(
             assets,
@@ -64,68 +80,35 @@ def get(self, request, *args, **kwargs):
         )
 
 
+# FIXME: we should be able to export at the project and item level, too
 class ExportCampaignToBagit(TemplateView):
     """
-    Creates temp directory structure for source data.  Copies source image
-    file from S3 or local storage into temp directory, builds export.csv
-    with meta, transcription, and tag data.  Executes bagit.py to turn temp
-    directory into bagit strucutre.  Builds and exports bagit structure as
-    zip.  Removes all temporary directories and files.
-
+    Creates temp directory structure for source data.
+    Executes bagit.py to turn temp directory into bagit strucutre.
+    Builds and exports bagit structure as zip.
+    Removes all temporary directories and files.
     """
 
-    include_images = True
-    template_name = "transcriptions/campaign.html"
-
     @method_decorator(login_required)
     def get(self, request, *args, **kwargs):
-        campaign = Campaign.objects.get(slug=self.kwargs["campaign_slug"])
-        asset_list = Asset.objects.filter(item__project__campaign=campaign).order_by(
-            "title", "sequence"
-        )
+        campaign_slug = self.kwargs["campaign_slug"]
+        assets = get_latest_transcription_data(campaign_slug)
 
-        # FIXME: this code should be working in a separate path than the media root!
-        # FIXME: we should be able to export at the project and item level, too
-        export_base_dir = os.path.join(settings.MEDIA_ROOT, "exporter", campaign.slug)
-
-        for asset in asset_list:
-            src = os.path.join(
-                settings.MEDIA_ROOT,
-                asset.item.project.campaign.slug,
-                asset.item.project.slug,
-                asset.item.item_id,
-                asset.slug,
-                asset.media_url,
-            )
+        export_base_dir = os.path.join(settings.SITE_ROOT_DIR, "tmp", campaign_slug)
+
+        for asset in assets:
             dest_folder = os.path.join(
-                export_base_dir, asset.item.project.slug, asset.item.item_id, asset.slug
+                export_base_dir, asset.item.project.slug, asset.item.item_id
             )
             os.makedirs(dest_folder, exist_ok=True)
-            dest = os.path.join(dest_folder, asset.media_url)
-
-            if self.include_images:
-                with open(dest, mode="wb") as dest_file:
-                    with ASSET_STORAGE.open(src, mode="rb") as src_file:
-                        for chunk in src_file.chunks(1048576):
-                            dest_file.write(chunk)
-
-            # Get transcription data
-            # FIXME: if we're not including all transcriptions,
-            # we should pick the completed or latest versions!
-
-            try:
-                transcription = Transcription.objects.get(
-                    asset=asset, user=self.request.user
-                ).text
-            except Transcription.DoesNotExist:
-                transcription = ""
 
             # Build transcription output text file
-            tran_output_path = os.path.join(
-                dest_folder, "%s.txt" % os.path.basename(asset.media_url)
+            text_output_path = os.path.join(
+                dest_folder,
+                "%s.txt" % os.path.basename(get_original_asset_id(asset.download_url)),
             )
-            with open(tran_output_path, "w") as f:
-                f.write(transcription)
+            with open(text_output_path, "w") as f:
+                f.write(asset.latest_transcription or "")
 
         # Turn Structure into bagit format
         bagit.make_bag(export_base_dir, {"Contact-Name": request.user.username})
@@ -137,7 +120,9 @@ def get(self, request, *args, **kwargs):
         # Download zip
         with open("%s.zip" % export_base_dir, "rb") as zip_file:
             response = HttpResponse(zip_file, content_type="application/zip")
-        response["Content-Disposition"] = "attachment; filename=%s.zip" % campaign.slug
+        response["Content-Disposition"] = "attachment; filename=%s.zip" % campaign_slug
+
+        # Upload zip to S3 bucket
 
         # Clean up temp folders & zipfile once exported
         shutil.rmtree(export_base_dir)


Test patch:
diff --git a/exporter/tests/test_view.py b/exporter/tests/test_view.py
--- a/exporter/tests/test_view.py
+++ b/exporter/tests/test_view.py
@@ -1,10 +1,6 @@
 import io
-import os
 import zipfile
 
-from django.conf import settings
-from django.core.files.base import ContentFile
-from django.core.files.storage import default_storage
 from django.test import TestCase
 from django.urls import reverse
 
@@ -16,6 +12,11 @@
     create_project,
 )
 
+DOWNLOAD_URL = (
+    "http://tile.loc.gov/image-services/iiif/"
+    "service:mss:mal:003:0036300:002/full/pct:25/0/default.jpg"
+)
+
 
 class ViewTest_Exporter(TestCase):
     """
@@ -24,45 +25,13 @@ class ViewTest_Exporter(TestCase):
     Make sure the postgresql db is available. Run docker-compose up db
     """
 
-    def login_user(self):
-        """
-        Create a user and log the user in
-        """
-
-        self.user = User.objects.create(username="tester", email="tester@example.com")
-        self.user.set_password("top_secret")
-        self.user.save()
+    def setUp(self):
+        user = User.objects.create(username="tester", email="tester@example.com")
+        user.set_password("top_secret")
+        user.save()
 
         self.assertTrue(self.client.login(username="tester", password="top_secret"))
 
-    def test_csv_export(self):
-        """
-        Test GET route /campaigns/export/<slug-value>/ (campaign)
-        """
-        self.login_user()
-
-        asset = create_asset()
-
-        response = self.client.get(
-            reverse(
-                "transcriptions:export-csv", args=(asset.item.project.campaign.slug,)
-            )
-        )
-
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(
-            response.content.decode("utf-8"),
-            "Campaign,Title,Description,MediaUrl,Transcription,Tags\r\n"
-            "Test Campaign,Test Asset,,1.jpg,,\r\n",
-        )
-
-    def test_bagit_export(self):
-        """
-        Test the http GET on route /campaigns/exportBagit/<campaignname>/
-        """
-
-        self.login_user()
-
         campaign = create_campaign(published=True)
         project = create_project(campaign=campaign, published=True)
         item = create_item(project=project, published=True)
@@ -71,33 +40,56 @@ def test_bagit_export(self):
             item=item,
             title="TestAsset",
             description="Asset Description",
-            media_url="1.jpg",
+            download_url=DOWNLOAD_URL,
             media_type=MediaType.IMAGE,
             sequence=1,
         )
 
         # add a Transcription object
-        transcription1 = Transcription(asset=asset, user=self.user, text="Sample")
+        transcription1 = Transcription(asset=asset, user=user, text="Sample")
         transcription1.full_clean()
         transcription1.save()
 
-        item_dir = os.path.join(
-            settings.MEDIA_ROOT, campaign.slug, project.slug, item.item_id, asset.slug
+    def test_csv_export(self):
+        """
+        Test GET route /campaigns/exportCSV/<slug-value>/ (campaign)
+        """
+
+        campaign_slug = "test-campaign"
+
+        response = self.client.get(
+            reverse("transcriptions:export-csv", args=(campaign_slug,))
         )
 
-        asset_file = ContentFile(b"Not a real JPEG")
-        default_storage.save(
-            os.path.join(item_dir, f"{asset.sequence}.jpg"), asset_file
+        expected_response_content = (
+            "b'Campaign,Project,Item,ItemId,Asset,"
+            "AssetStatus,DownloadUrl,Transcription\\r\\n'"
+            "b'Test Campaign,Test Project,Test Item,"
+            "testitem0123456789,TestAsset,edit,"
+            "http://tile.loc.gov/image-services/"
+            "iiif/service:mss:mal:003:0036300:002/full"
+            "/pct:25/0/default.jpg,Sample\\r\\n'"
         )
 
+        self.assertEqual(response.status_code, 200)
+        response_content = "".join(map(str, response.streaming_content))
+        self.assertEqual(response_content, expected_response_content)
+
+    def test_bagit_export(self):
+        """
+        Test the http GET on route /campaigns/exportBagit/<campaignname>/
+        """
+
+        campaign_slug = "test-campaign"
+
         response = self.client.get(
-            reverse("transcriptions:export-bagit", args=(campaign.slug,))
+            reverse("transcriptions:export-bagit", args=(campaign_slug,))
         )
 
         self.assertEqual(response.status_code, 200)
         self.assertEquals(
             response.get("Content-Disposition"),
-            "attachment; filename=%s.zip" % campaign.slug,
+            "attachment; filename=%s.zip" % campaign_slug,
         )
 
         f = io.BytesIO(response.content)
@@ -106,6 +98,6 @@ def test_bagit_export(self):
         self.assertIn("bagit.txt", zipped_file.namelist())
         self.assertIn("bag-info.txt", zipped_file.namelist())
         self.assertIn(
-            "data/test-project/testitem0123456789/testasset/1.jpg",
+            "data/test-project/testitem0123456789/mss-mal-003-0036300-002.txt",
             zipped_file.namelist(),
         )