Repository: hydroshare/hydroshare

Issue:
Create bag ingestion management command
Metadata xml file ingestion with https://github.com/hydroshare/hydroshare/blob/develop/hs_core/hs_rdf.py

Implement a management command to take a zipped bag archive of a resource, unpack it, add the files to a resource and ingest the metadata files.


Discussion:

Resolving patch:
diff --git a/hs_core/management/commands/ingest_bag.py b/hs_core/management/commands/ingest_bag.py
new file mode 100644
--- /dev/null
+++ b/hs_core/management/commands/ingest_bag.py
@@ -0,0 +1,104 @@
+"""
+This command ingests a bag into a resource.  You can provide a resource_id to ingest the bag or the command will create
+a new resource to ingest the bag into.
+"""
+import os
+
+from django.core.exceptions import ValidationError
+from django.core.files.uploadedfile import UploadedFile
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+
+from hs_core.hydroshare import delete_resource_file, add_file_to_resource, create_resource
+from hs_composite_resource.models import CompositeResource
+from zipfile import ZipFile
+from pathlib import Path
+
+from hs_core.views.utils import ingest_bag
+
+
+def extract_resource_id(bag_file_path):
+    """Extracts the resource id by reading the root directory name inside the zip"""
+    with ZipFile(bag_file_path, 'r') as zip_file:
+        resource_id = Path(zip_file.namelist()[0]).parts[0]
+    return resource_id
+
+
+def is_hydrohsare_bagit(bag_file_path):
+    """Checks for a hydroshare bagit file structure in a zip"""
+    resource_id = extract_resource_id(bag_file_path)
+    bagit_files = [
+        os.path.join(resource_id, "bagit.txt"),
+        os.path.join(resource_id, "manifest-md5.txt"),
+        os.path.join(resource_id, "readme.txt"),
+        os.path.join(resource_id, "tagmanifest-md5.txt"),
+        os.path.join(resource_id, "data", "resourcemetadata.xml"),
+        os.path.join(resource_id, "data", "resourcemap.xml")
+    ]
+    with ZipFile(bag_file_path, 'r') as zip_file:
+        for bagit_file in bagit_files:
+            if bagit_file not in zip_file.namelist():
+                return False
+    return True
+
+
+class Command(BaseCommand):
+    help = "Ingest a zipped bag archive into a resource"
+
+    def add_arguments(self, parser):
+
+        # a list of resource id's: none does nothing.
+        parser.add_argument('bag_path', type=str)
+
+        parser.add_argument(
+            '-r',
+            '--resource_id',
+            type=str,
+            help="Existing resource to ingest into. If omitted, the bag's resource_id will be used")
+
+        parser.add_argument(
+            '-o',
+            '--overwrite',
+            action='store_true',
+            help="Overwrites resource if it exists on HydroShare"
+        )
+
+        parser.add_argument(
+            '-u',
+            '--user_id',
+            type=int,
+            help="The id of the user to make the owner of the resource, defaults to Hydroshare Administrator (1)",
+            default=1
+        )
+
+    def handle(self, *args, **options):
+        bag_path = options['bag_path']
+
+        resource_id = options['resource_id']
+
+        if not is_hydrohsare_bagit(bag_path):
+            raise ValidationError(f"{bag_path} is not a valid HydroShare bagit zip")
+
+        if not resource_id:
+            resource_id = extract_resource_id(bag_path)
+
+        resource_exists = CompositeResource.objects.filter(short_id=resource_id).exists()
+        user = User.objects.get(id=options['user_id'])
+        if resource_exists:
+            if not options['overwrite']:
+                raise ValidationError(f"Resource {resource_id} exists, include the overwrite command or provide "
+                                      f"another resource_id.")
+
+            res = CompositeResource.objects.get(short_id=resource_id)
+
+            # delete all files and logical files
+            for file in res.files.all():
+                delete_resource_file(resource_id, file.id, user)
+
+        else:
+            res = create_resource("CompositeResource", user, "Title will be overwritten", short_id=resource_id)
+
+        bag_file_name = os.path.basename(bag_path)
+        bag_file = UploadedFile(file=open(bag_path, mode="rb"), name=bag_file_name)
+        bag_res_file = add_file_to_resource(res, bag_file)
+        ingest_bag(res, bag_res_file, user)
diff --git a/hs_core/views/utils.py b/hs_core/views/utils.py
--- a/hs_core/views/utils.py
+++ b/hs_core/views/utils.py
@@ -972,7 +972,6 @@ def unzip_file(user, res_id, zip_with_rel_path, bool_remove_original,
     if not resource.supports_unzip(zip_with_rel_path):
         raise ValidationError("Unzipping of this file is not supported.")
 
-    working_dir = os.path.dirname(zip_with_full_path)
     unzip_path = None
     try:
         if overwrite:
@@ -981,19 +980,13 @@ def unzip_file(user, res_id, zip_with_rel_path, bool_remove_original,
             # list all files to be moved into the resource
             unzipped_files = listfiles_recursively(istorage, unzip_path)
             unzipped_foldername = os.path.basename(unzip_path)
-            destination_folders = []
-            # list all folders to be written into the resource
-            for folder in listfolders(istorage, unzip_path):
-                destination_folder = os.path.join(working_dir, folder)
-                destination_folders.append(destination_folder)
 
-            irods_files = []
+            res_files = []
             for unzipped_file in unzipped_files:
-                irods_files.append(IrodsFile(unzipped_file, istorage))
-            res_files = irods_files
+                res_files.append(IrodsFile(unzipped_file, istorage))
             meta_files = []
             if ingest_metadata:
-                res_files, meta_files, map_files = identify_metadata_files(irods_files)
+                res_files, meta_files, map_files = identify_metadata_files(res_files)
             # walk through each unzipped file, delete aggregations if they exist
             for file in res_files:
                 destination_file = _get_destination_filename(file.name, unzipped_foldername)
@@ -1050,6 +1043,63 @@ def unzip_file(user, res_id, zip_with_rel_path, bool_remove_original,
     hydroshare.utils.resource_modified(resource, user, overwrite_bag=False)
 
 
+def ingest_bag(resource, bag_file, user):
+    """
+    Ingests a zipped bagit archive of a hydroshare reosource that has been uploaded to the resource
+    :param resource: The CompositeResource to ingest the bag into
+    :param bag_file: The ResourceFile of the zipped bag in the resource
+    :param user: The HydroShare user object to do the action as
+    """
+    from hs_file_types.utils import identify_metadata_files, ingest_metadata_files
+
+    istorage = resource.get_irods_storage()
+    zip_with_full_path = os.path.join(resource.file_path, bag_file.short_path)
+
+    # unzip to a temporary folder=
+    unzip_path = istorage.unzip(zip_with_full_path, unzipped_folder=uuid4().hex)
+    delete_resource_file(resource.short_id, bag_file.id, user)
+
+    # list all files to be moved into the resource
+    unzipped_files = listfiles_recursively(istorage, unzip_path)
+
+    res_files = []
+    for unzipped_file in unzipped_files:
+        res_files.append(IrodsFile(unzipped_file, istorage))
+    res_files, meta_files, map_files = identify_metadata_files(res_files)
+
+    # filter res_files to only files in the data/contents directory
+    data_contents_dir = os.path.join("data", "contents")
+    res_files = [res_file for res_file in res_files if res_file.name.count(data_contents_dir) > 1]
+
+    # now move each file to the destination
+    def destination_filename(resource, file):
+        """Parses the temporary filename to the destination filename"""
+        dc_dir = os.path.join("data", "contents")
+        relative_path = dc_dir.join(file.split(dc_dir, 2)[2:])
+        return os.path.join(resource.file_path, relative_path.strip("/"))
+
+    added_resource_files = []
+    for file in res_files:
+        destination_file = destination_filename(resource, file.name)
+        istorage.moveFile(file.name, destination_file)
+
+        irods_path = resource.get_irods_path(destination_file)
+        res_file = link_irods_file_to_django(resource, irods_path)
+        added_resource_files.append(res_file)
+
+    check_aggregations(resource, added_resource_files)
+
+    ingest_metadata_files(resource, meta_files, map_files)
+
+    istorage.delete(unzip_path)
+
+    # In addition to the Date metadataelement with type created, the resource django model created field is used and
+    # needs to be updated.
+    created = [d for d in resource.metadata.dates.all() if d.type == 'created'][0]
+    resource.created = created.start_date
+    resource.save()
+
+
 def _get_destination_filename(file, unzipped_foldername):
     """
     Returns the destination file path by removing the temp unzipped_foldername from the file path.


Test patch:
diff --git a/hs_core/tests/api/native/test_bag_ingestion_command.py b/hs_core/tests/api/native/test_bag_ingestion_command.py
new file mode 100755
--- /dev/null
+++ b/hs_core/tests/api/native/test_bag_ingestion_command.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+from unittest import TestCase
+
+from django.contrib.auth.models import Group
+from django.core.exceptions import ValidationError
+
+from hs_composite_resource.models import CompositeResource
+from hs_core.testing import MockIRODSTestCaseMixin
+from hs_core import hydroshare
+from django.core.management import call_command
+
+
+class TestIngestBag(MockIRODSTestCaseMixin, TestCase):
+
+    def setUp(self):
+        super(TestIngestBag, self).setUp()
+
+        self.hs_group, _ = Group.objects.get_or_create(name='Hydroshare Author')
+        # create a user
+        self.user = hydroshare.create_account(
+            'test_bag_ingestion@email.com',
+            username='bag_ingestion_test',
+            first_name='some_first_name',
+            last_name='some_last_name',
+            superuser=False,
+            groups=[self.hs_group],
+        )
+
+        # delete all resources in case a test isn't cleaning up after itself
+        CompositeResource.objects.all().delete()
+
+    def tearDown(self):
+        super(TestIngestBag, self).tearDown()
+        self.user.delete()
+        self.hs_group.delete()
+        CompositeResource.objects.all().delete()
+
+    def test_bag_ingestion_command(self):
+        assert CompositeResource.objects.all().count() == 0
+        call_command("ingest_bag", 'hs_core/tests/data/94c34d62d36a4faf97fbccfa18ddd6b9.zip', user_id=str(self.user.id))
+        assert CompositeResource.objects.all().count() == 1
+        res = CompositeResource.objects.all().first()
+        assert res.metadata.title.value == "COVID-19 Data Plotting"
+        assert res.files.all().count() == 1
+        readme = res.files.all().first()
+        assert readme.file_name == "README.md"
+
+    def test_bag_ingestion_command_overwrite(self):
+        assert CompositeResource.objects.all().count() == 0
+        res = hydroshare.create_resource("CompositeResource", self.user, "To overwrite")
+        call_command("ingest_bag", 'hs_core/tests/data/94c34d62d36a4faf97fbccfa18ddd6b9.zip', user_id=str(self.user.id),
+                     overwrite=True, resource_id=res.short_id)
+        assert CompositeResource.objects.all().count() == 1
+        res = CompositeResource.objects.all().first()
+        assert res.metadata.title.value == "COVID-19 Data Plotting"
+        assert res.files.all().count() == 1
+        readme = res.files.all().first()
+        assert readme.file_name == "README.md"
+
+    def test_bag_ingestion_command_overwrite_catch(self):
+        res = hydroshare.create_resource("CompositeResource", self.user, "To overwrite")
+        try:
+            call_command("ingest_bag", 'hs_core/tests/data/94c34d62d36a4faf97fbccfa18ddd6b9.zip',
+                         user_id=str(self.user.id), resource_id=res.short_id)
+            assert False, "should have thrown error"
+        except ValidationError as e:
+            assert f"Resource {res.short_id} exists, include the overwrite command or provide another resource_id." \
+                   in str(e)
diff --git a/hs_core/tests/data/94c34d62d36a4faf97fbccfa18ddd6b9.zip b/hs_core/tests/data/94c34d62d36a4faf97fbccfa18ddd6b9.zip
new file mode 100644
Binary files /dev/null and b/hs_core/tests/data/94c34d62d36a4faf97fbccfa18ddd6b9.zip differ