Repository: LibraryOfCongress/concordia

Issue:
Update Exporter config to include original image file name
To make ETL simple, the export of the transcription data needs to match the original format of the image name and directory name.


Discussion:

Resolving patch:
diff --git a/concordia/settings_dev.py b/concordia/settings_dev.py
--- a/concordia/settings_dev.py
+++ b/concordia/settings_dev.py
@@ -19,14 +19,6 @@
 CELERY_BROKER_URL = "pyamqp://guest@localhost"
 CELERY_RESULT_BACKEND = "rpc://"
 
-S3_BUCKET_NAME = "concordia-staticpages"
-
-DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-AWS_STORAGE_BUCKET_NAME = S3_BUCKET_NAME
-AWS_DEFAULT_ACL = None  # Don't set an ACL on the files, inherit the bucket ACLs
-
-MEDIA_URL = "https://%s.s3.amazonaws.com/" % S3_BUCKET_NAME
-
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 EMAIL_FILE_PATH = "/tmp/concordia-messages"  # change this to a proper location
 DEFAULT_FROM_EMAIL = os.environ.get("DEFAULT_FROM_EMAIL", "")
diff --git a/concordia/settings_prod.py b/concordia/settings_prod.py
--- a/concordia/settings_prod.py
+++ b/concordia/settings_prod.py
@@ -50,6 +50,7 @@
 CELERY_RESULT_BACKEND = "rpc://"
 
 S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME")
+EXPORT_S3_BUCKET_NAME = os.getenv("EXPORT_S3_BUCKET_NAME")
 
 DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 AWS_STORAGE_BUCKET_NAME = S3_BUCKET_NAME
diff --git a/exporter/views.py b/exporter/views.py
--- a/exporter/views.py
+++ b/exporter/views.py
@@ -1,12 +1,15 @@
 import os
 import re
 import shutil
+import tempfile
+from datetime import datetime
 
 import bagit
+import boto3
 from django.conf import settings
-from django.contrib.auth.decorators import login_required
-from django.db.models import Subquery, OuterRef
-from django.http import HttpResponse
+from django.contrib.admin.views.decorators import staff_member_required
+from django.db.models import OuterRef, Subquery
+from django.http import HttpResponse, HttpResponseRedirect
 from django.utils.decorators import method_decorator
 from django.views.generic import TemplateView
 from tabular_export.core import export_to_csv_response, flatten_queryset
@@ -14,16 +17,14 @@
 from concordia.models import Asset, Transcription
 
 
-def get_latest_transcription_data(campaign_slug):
+def get_latest_transcription_data(asset_qs):
     latest_trans_subquery = (
         Transcription.objects.filter(asset=OuterRef("pk"))
         .order_by("-pk")
         .values("text")
     )
-    assets = Asset.objects.annotate(
-        latest_transcription=Subquery(latest_trans_subquery[:1])
-    )
-    assets = assets.filter(item__project__campaign__slug=campaign_slug)
+
+    assets = asset_qs.annotate(latest_transcription=Subquery(latest_trans_subquery[:1]))
     return assets
 
 
@@ -33,10 +34,10 @@ def get_original_asset_id(download_url):
     that identifies this image uniquely on loc.gov
     """
     if download_url.startswith("http://tile.loc.gov/"):
-        pattern = r"/service:([A-Za-z0-9:]*)/"
+        pattern = r"/service:([A-Za-z0-9:\-]*)/"
         asset_id = re.search(pattern, download_url)
         assert asset_id
-        return asset_id.group(1).replace(":", "-")
+        return asset_id.group(1)
     else:
         return download_url
 
@@ -46,9 +47,12 @@ class ExportCampaignToCSV(TemplateView):
     Exports the most recent transcription for each asset in a campaign
     """
 
-    @method_decorator(login_required)
+    @method_decorator(staff_member_required)
     def get(self, request, *args, **kwargs):
-        assets = get_latest_transcription_data(self.kwargs["campaign_slug"])
+        asset_qs = Asset.objects.filter(
+            item__project__campaign__slug=self.kwargs["campaign_slug"]
+        )
+        assets = get_latest_transcription_data(asset_qs)
 
         headers, data = flatten_queryset(
             assets,
@@ -86,15 +90,25 @@ class ExportCampaignToBagit(TemplateView):
     Creates temp directory structure for source data.
     Executes bagit.py to turn temp directory into bagit strucutre.
     Builds and exports bagit structure as zip.
+    Uploads zip to S3 if configured.
     Removes all temporary directories and files.
     """
 
-    @method_decorator(login_required)
+    @method_decorator(staff_member_required)
     def get(self, request, *args, **kwargs):
         campaign_slug = self.kwargs["campaign_slug"]
-        assets = get_latest_transcription_data(campaign_slug)
+        asset_qs = Asset.objects.filter(
+            item__project__campaign__slug=campaign_slug,
+            transcription_status="completed",
+        )
+
+        assets = get_latest_transcription_data(asset_qs)
+
+        with tempfile.TemporaryDirectory(prefix=campaign_slug) as export_base_dir:
+            return self.do_bagit_export(assets, export_base_dir, campaign_slug)
 
-        export_base_dir = os.path.join(settings.SITE_ROOT_DIR, "tmp", campaign_slug)
+    def do_bagit_export(self, assets, export_base_dir, campaign_slug):
+        os.makedirs(export_base_dir, exist_ok=True)
 
         for asset in assets:
             dest_folder = os.path.join(
@@ -111,21 +125,33 @@ def get(self, request, *args, **kwargs):
                 f.write(asset.latest_transcription or "")
 
         # Turn Structure into bagit format
-        bagit.make_bag(export_base_dir, {"Contact-Name": request.user.username})
+        bagit.make_bag(export_base_dir)
 
         # Build .zip file of bagit formatted Campaign Folder
         archive_name = export_base_dir
         shutil.make_archive(archive_name, "zip", export_base_dir)
 
-        # Download zip
-        with open("%s.zip" % export_base_dir, "rb") as zip_file:
-            response = HttpResponse(zip_file, content_type="application/zip")
-        response["Content-Disposition"] = "attachment; filename=%s.zip" % campaign_slug
-
         # Upload zip to S3 bucket
+        s3_bucket = getattr(settings, "EXPORT_S3_BUCKET_NAME", None)
+        export_filename = "%s-%s.zip" % (
+            campaign_slug,
+            datetime.today().isoformat(timespec="minutes"),
+        )
 
-        # Clean up temp folders & zipfile once exported
-        shutil.rmtree(export_base_dir)
-        os.remove("%s.zip" % export_base_dir)
+        if s3_bucket:
+            s3 = boto3.resource("s3")
+            s3.Bucket(s3_bucket).upload_file(
+                "%s.zip" % export_base_dir, "%s" % export_filename
+            )
 
-        return response
+            return HttpResponseRedirect(
+                "https://%s.s3.amazonaws.com/%s" % (s3_bucket, export_filename)
+            )
+        else:
+            # Download zip from local storage
+            with open("%s.zip" % export_base_dir, "rb") as zip_file:
+                response = HttpResponse(zip_file, content_type="application/zip")
+            response["Content-Disposition"] = (
+                "attachment; filename=%s" % export_filename
+            )
+            return response


Test patch:
diff --git a/exporter/tests/test_view.py b/exporter/tests/test_view.py
--- a/exporter/tests/test_view.py
+++ b/exporter/tests/test_view.py
@@ -1,5 +1,6 @@
 import io
 import zipfile
+from datetime import datetime
 
 from django.test import TestCase
 from django.urls import reverse
@@ -26,7 +27,9 @@ class ViewTest_Exporter(TestCase):
     """
 
     def setUp(self):
-        user = User.objects.create(username="tester", email="tester@example.com")
+        user = User.objects.create(
+            username="tester", email="tester@example.com", is_staff=True
+        )
         user.set_password("top_secret")
         user.save()
 
@@ -46,7 +49,13 @@ def setUp(self):
         )
 
         # add a Transcription object
-        transcription1 = Transcription(asset=asset, user=user, text="Sample")
+        transcription1 = Transcription(
+            asset=asset,
+            user=user,
+            text="Sample",
+            submitted=datetime.now(),
+            accepted=datetime.now(),
+        )
         transcription1.full_clean()
         transcription1.save()
 
@@ -65,7 +74,7 @@ def test_csv_export(self):
             "b'Campaign,Project,Item,ItemId,Asset,"
             "AssetStatus,DownloadUrl,Transcription\\r\\n'"
             "b'Test Campaign,Test Project,Test Item,"
-            "testitem0123456789,TestAsset,edit,"
+            "testitem0123456789,TestAsset,completed,"
             "http://tile.loc.gov/image-services/"
             "iiif/service:mss:mal:003:0036300:002/full"
             "/pct:25/0/default.jpg,Sample\\r\\n'"
@@ -87,9 +96,14 @@ def test_bagit_export(self):
         )
 
         self.assertEqual(response.status_code, 200)
+
+        export_filename = "%s-%s.zip" % (
+            campaign_slug,
+            datetime.today().isoformat(timespec="minutes"),
+        )
         self.assertEquals(
             response.get("Content-Disposition"),
-            "attachment; filename=%s.zip" % campaign_slug,
+            "attachment; filename=%s" % export_filename,
         )
 
         f = io.BytesIO(response.content)
@@ -98,6 +112,6 @@ def test_bagit_export(self):
         self.assertIn("bagit.txt", zipped_file.namelist())
         self.assertIn("bag-info.txt", zipped_file.namelist())
         self.assertIn(
-            "data/test-project/testitem0123456789/mss-mal-003-0036300-002.txt",
+            "data/test-project/testitem0123456789/mss:mal:003:0036300:002.txt",
             zipped_file.namelist(),
         )