Repository: hydroshare/hydroshare

Issue:
Metrics should be standardized
**Describe the feature you'd like and what it will do**
The [MakeDataCount](https://makedatacount.org/data-usage/) effort to standardize metrics collection and reporting is worthwhile for us to aspire to. We probably don’t want to actually get audited and become a member, but we can still implement the [COUTERr5](https://www.projectcounter.org/wp-content/uploads/2017/07/Tech_Notes_20170710.pdf) standards.

**Additional context**
HS v 2.8

I found this set of notes helpful: https://www.projectcounter.org/wp-content/uploads/2017/07/Tech_Notes_20170710.pdf

[Datacite has a simplified list](https://support.datacite.org/docs/processing) of the basic things we will want to track in order to be compliant with the COUNTER Code of Practice 

Set of minimal fields:

> Date and time
> Request IP address
> Session cookie ID: an ID kept in a session cookie which only lives as long as the browser/session is open
> User cookie ID: an ID that identifies a user session that may persist past the closing of the browser window to future visits
> Username or user ID: identifies a user of the system because they have logged in or identified themselves
> The requested URL
> The DOI name. The DOI Identifier which uniquely identifies this dataset
> The size of the dataset. Size (only needed for requests which download a dataset and not for investigations which display landing pages or metadata about a dataset)
> The user-agent string being sent by the client

It also requires the following [metadata](https://support.datacite.org/docs/processing#enriching-with-doi-metadata) about the resource:

> title
> publisher
> publisher Id
> creators
> publication date
> version of dataset
> URL
> publication year

If we ever want to, it looks like we could use a tool like [counter-processor](https://github.com/CDLUC3/counter-processor) to actually process our logs into a more compliant (submittable) format.



Discussion:
related to https://github.com/hydroshare/hydroshare/issues/4733

Resolving patch:
diff --git a/django_irods/views.py b/django_irods/views.py
--- a/django_irods/views.py
+++ b/django_irods/views.py
@@ -18,7 +18,7 @@
     get_task_notification
 )
 
-from hs_core.signals import pre_download_file, pre_check_bag_flag
+from hs_core.signals import pre_download_file, pre_download_resource, pre_check_bag_flag
 from hs_core.tasks import create_bag_by_irods, create_temp_zip, delete_zip
 from hs_core.views.utils import authorize, ACTION_TO_AUTHORIZE
 from drf_yasg.utils import swagger_auto_schema
@@ -282,14 +282,6 @@ def download(request, path, use_async=True, use_reverse_proxy=True,
                 res.setAVU("bag_modified", True)  # ensure bag_modified is set when irods_output_path does not exist
                 create_bag_by_irods(res_id, create_zip=False)
 
-        # send signal for pre download file
-        # TODO: does not contain subdirectory information: duplicate refreshes possible
-        download_file_name = split_path_strs[-1]  # end of path
-        # this logs the download request in the tracking system
-        pre_download_file.send(sender=resource_cls, resource=res,
-                               download_file_name=download_file_name,
-                               request=request)
-
     # If we get this far,
     # * path and irods_path point to true input
     # * output_path and irods_output_path point to true output.
@@ -304,6 +296,17 @@ def download(request, path, use_async=True, use_reverse_proxy=True,
     # TODO: standardize this to make it less brittle
     stdout = session.run("ils", None, "-l", irods_output_path)[0].split()
     flen = int(stdout[3])
+    # this logs the download request in the tracking system
+    if is_bag_download:
+        pre_download_resource.send(sender=resource_cls, resource=res, request=request)
+    else:
+        download_file_name = split_path_strs[-1]
+        # send signal for pre download file
+        # TODO: does not contain subdirectory information: duplicate refreshes possible
+        pre_download_file.send(sender=resource_cls, resource=res,
+                               download_file_name=download_file_name,
+                               file_size=flen,
+                               request=request)
 
     # Allow reverse proxy if request was forwarded by nginx (HTTP_X_DJANGO_REVERSE_PROXY='true')
     # and reverse proxy is possible according to configuration (SENDFILE_ON=True)
diff --git a/hs_core/signals.py b/hs_core/signals.py
--- a/hs_core/signals.py
+++ b/hs_core/signals.py
@@ -14,7 +14,9 @@
                                                                      'request'])
 post_metadata_element_update = django.dispatch.Signal(providing_args=['element_name', 'element_id'])
 
-pre_download_file = django.dispatch.Signal(providing_args=['sender', 'request', 'resource', 'download_file_name'])
+pre_download_file = django.dispatch.Signal(providing_args=['sender', 'request', 'resource',
+                                                           'download_file_name', 'file_size'])
+pre_download_resource = django.dispatch.Signal(providing_args=['sender', 'request', 'resource'])
 
 pre_check_bag_flag = django.dispatch.Signal(providing_args=['resource'])
 
diff --git a/hs_tracking/signals.py b/hs_tracking/signals.py
--- a/hs_tracking/signals.py
+++ b/hs_tracking/signals.py
@@ -1,7 +1,10 @@
+import json
 from django.contrib.auth.signals import user_logged_in, user_logged_out
 from django.dispatch import receiver
 
-from hs_core.signals import pre_download_file, pre_delete_resource, post_create_resource
+from hs_core.signals import pre_download_file, pre_download_resource, pre_delete_resource, post_create_resource
+from hs_core.enums import RelationTypes
+from hs_core.hydroshare.utils import current_site_url
 
 from .models import Session
 from .models import Variable
@@ -34,6 +37,40 @@ def capture_logout(sender, **kwargs):
     session.record('logout', value=msg)
 
 
+def get_common_download_fields(**kwargs):
+    # get standard fields
+    user = None
+    if hasattr(kwargs, 'user'):
+        user = kwargs['user']
+    session = Session.objects.for_request(kwargs['request'], user)
+    fields = get_std_log_fields(kwargs['request'], session)
+
+    # add specific fields
+    fields['resource_size_bytes'] = kwargs['resource'].size
+    fields['resource_type'] = kwargs['resource'].resource_type
+    fields['resource_guid'] = kwargs['resource'].short_id
+
+    # MakeDataCount fields
+    fields['tracking_visitor_id'] = session.visitor.id
+    fields['doi'] = kwargs['resource'].doi
+    fields['user_agent'] = kwargs['request'].headers.get('user-agent')
+
+    # MakeDataCount also requires the following metadata about the resource:
+    res_metadata = {
+        'title': kwargs['resource'].title,
+        'creators': " ;".join(str(name) for name in kwargs['resource'].metadata.creators.only('name').all()),
+        'is_replaced_by': kwargs['resource'].get_relation_version_res_url(RelationTypes.isReplacedBy) or None,
+        'is_version_of': kwargs['resource'].get_relation_version_res_url(RelationTypes.isVersionOf) or None,
+        'resource_url': current_site_url() + kwargs['resource'].absolute_url
+    }
+    if kwargs['resource'].raccess.published:
+        res_metadata['publisher'] = str(kwargs['resource'].metadata.publisher),
+        res_metadata['publish_date'] = kwargs['resource'].metadata.dates.all().filter(type='published').first()\
+            .start_date.strftime("%m/%d/%Y %H:%M:%S.%f")
+    fields['resource_metadata'] = json.dumps(res_metadata)
+    return fields, session
+
+
 @receiver(pre_download_file)
 def capture_download(**kwargs):
 
@@ -46,18 +83,9 @@ def capture_download(**kwargs):
     if not is_human:
         return
 
-    # get standard fields
-    user = None
-    if hasattr(kwargs, 'user'):
-        user = kwargs['user']
-    session = Session.objects.for_request(kwargs['request'], user)
-    fields = get_std_log_fields(kwargs['request'], session)
-
-    # add specific fields
+    fields, session = get_common_download_fields(**kwargs)
+    fields['file_size'] = kwargs['file_size']
     fields['filename'] = kwargs['download_file_name']
-    fields['resource_size_bytes'] = kwargs['resource'].size
-    fields['resource_type'] = kwargs['resource'].resource_type
-    fields['resource_guid'] = kwargs['resource'].short_id
 
     # format the 'download' kwargs
     msg = Variable.format_kwargs(**fields)
@@ -68,6 +96,29 @@ def capture_download(**kwargs):
                    resource_id=kwargs['resource'].short_id)
 
 
+@receiver(pre_download_resource)
+def capture_resource_download(**kwargs):
+
+    # exit early if the request is not passed in as a kwarg
+    if 'request' not in list(kwargs.keys()):
+        return
+
+    # exit early if not human (necessary b/c this action does not require log in)
+    is_human = getattr(kwargs['request'], 'is_human', False)
+    if not is_human:
+        return
+
+    fields, session = get_common_download_fields(**kwargs)
+
+    # format the 'download' kwargs
+    msg = Variable.format_kwargs(**fields)
+
+    # record the download action
+    session.record('resource_download', value=msg,
+                   resource=kwargs['resource'],
+                   resource_id=kwargs['resource'].short_id)
+
+
 @receiver(post_create_resource)
 def capture_resource_create(**kwargs):
 


Test patch: