Repository: rucio/rucio

Issue:
RSE_Expressions link broken in documentation
Motivation
----------
Following the [RSE Expressions](http://rucio.readthedocs.io/RSE_Expressions.html) on http://rucio.readthedocs.io/overview_Rucio_Storage_Element.html leads to a non-existing page.

Modification
------------
Either the link in https://github.com/rucio/rucio/blob/master/doc/source/overview_Rucio_Storage_Element.rst needs to be made lower-case or the file https://github.com/rucio/rucio/blob/master/doc/source/rse_expressions.rst needs to be renamed to `RSE_Expressions`.


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/replica_sorter.py b/lib/rucio/common/replica_sorter.py
--- a/lib/rucio/common/replica_sorter.py
+++ b/lib/rucio/common/replica_sorter.py
@@ -14,6 +14,9 @@
  This product includes GeoLite data created by MaxMind,
  available from <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
 '''
+
+from __future__ import print_function
+
 import gzip
 import os
 import random
@@ -56,10 +59,10 @@ def __get_geoip_db(directory, filename):
     if directory.endswith('/'):
         directory = directory[:-1]
     if not os.path.isfile('%s/%s' % (directory, filename)):
-        print '%s does not exist. Downloading it.' % (filename)
+        print('%s does not exist. Downloading it.' % (filename))
         __download_geoip_db(directory, filename)
     elif (time.time() - os.stat('%s/%s' % (directory, filename)).st_atime > 30 * 86400):
-        print '%s is too old. Re-downloading it.' % (filename)
+        print('%s is too old. Re-downloading it.' % (filename))
         __download_geoip_db(directory, filename)
     else:
         return
@@ -87,13 +90,13 @@ def __get_lat_long(se, gi, gi2):
     except socket.gaierror, e:
         try:
             # Host unknown. It might be IPv6. Trying with geoip2
-            print e
+            print(e)
             ip = socket.getaddrinfo(se, None)[0][4][0]
             response = gi2.city(ip)
             return response.location.latitude, response.location.longitude
         except socket.gaierror, e:
             # Host definitively unknown
-            print e
+            print(e)
             return None, None
 
 
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -18,12 +18,14 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2017
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2017
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
-# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2015
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2015-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2016-2018
 # - Frank Berghaus, <frank.berghaus@cern.ch>, 2017
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
 # - Tobias Wegner <twegner@cern.ch>, 2018
 
+from __future__ import print_function
+
 import base64
 import datetime
 import errno
@@ -267,7 +269,7 @@ def generate_http_error(status_code, exc_cls, exc_msg):
     try:
         return HTTPError(status, headers=headers, data=render_json(**data))
     except:
-        print {'Content-Type': 'application/octet-stream', 'ExceptionClass': exc_cls, 'ExceptionMessage': str(exc_msg).strip()}
+        print({'Content-Type': 'application/octet-stream', 'ExceptionClass': exc_cls, 'ExceptionMessage': str(exc_msg).strip()})
         raise
 
 
@@ -291,7 +293,7 @@ def generate_http_error_flask(status_code, exc_cls, exc_msg):
     try:
         return resp
     except:
-        print {'Content-Type': 'application/octet-stream', 'ExceptionClass': exc_cls, 'ExceptionMessage': str(exc_msg).strip()}
+        print({'Content-Type': 'application/octet-stream', 'ExceptionClass': exc_cls, 'ExceptionMessage': str(exc_msg).strip()})
         raise
 
 
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -13,7 +13,7 @@
   - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2015, 2017
   - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017
   - Ralph Vigne, <ralph.vigne@cern.ch>, 2013
-  - Thomas Beermann, <thomas.beermann@cern.ch>, 2014, 2016-2017
+  - Thomas Beermann, <thomas.beermann@cern.ch>, 2014, 2016-2018
   - Joaquin Bogado, <joaquin.bogado@cern.ch>, 2014-2015
   - Wen Guan, <wen.guan@cern.ch>, 2015
 '''
@@ -30,7 +30,7 @@
 from sqlalchemy.exc import DatabaseError, IntegrityError, CompileError
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import not_, func
-from sqlalchemy.sql.expression import bindparam, text, Insert, select, true
+from sqlalchemy.sql.expression import bindparam, case, text, Insert, select, true
 
 import rucio.core.rule
 import rucio.core.replica  # import add_replicas
@@ -1407,7 +1407,7 @@ def list_dids(scope, filters, type='collection', ignore_case=False, limit=None,
 @read_session
 def get_did_atime(scope, name, session=None):
     """
-    Get the accessed_at timestamp for a replica. Just for testing.
+    Get the accessed_at timestamp for a did. Just for testing.
     :param scope: the scope name.
     :param name: The data identifier name.
     :param session: Database session to use.
@@ -1417,6 +1417,19 @@ def get_did_atime(scope, name, session=None):
     return session.query(models.DataIdentifier.accessed_at).filter_by(scope=scope, name=name).one()[0]
 
 
+@read_session
+def get_did_access_cnt(scope, name, session=None):
+    """
+    Get the access_cnt for a did. Just for testing.
+    :param scope: the scope name.
+    :param name: The data identifier name.
+    :param session: Database session to use.
+
+    :returns: A datetime timestamp with the last access time.
+    """
+    return session.query(models.DataIdentifier.access_cnt).filter_by(scope=scope, name=name).one()[0]
+
+
 @stream_session
 def get_dataset_by_guid(guid, session=None):
     """
@@ -1441,7 +1454,7 @@ def get_dataset_by_guid(guid, session=None):
 @transactional_session
 def touch_dids(dids, session=None):
     """
-    Update the accessed_at timestamp of the given dids.
+    Update the accessed_at timestamp and the access_cnt of the given dids.
 
     :param replicas: the list of dids.
     :param session: The database session in use.
@@ -1450,10 +1463,15 @@ def touch_dids(dids, session=None):
     """
 
     now = datetime.utcnow()
+    none_value = None
     try:
         for did in dids:
-            session.query(models.DataIdentifier).filter_by(scope=did['scope'], name=did['name'], did_type=did['type']).\
-                update({'accessed_at': did.get('accessed_at') or now}, synchronize_session=False)
+            session.query(models.DataIdentifier).\
+                filter_by(scope=did['scope'], name=did['name'], did_type=did['type']).\
+                update({'accessed_at': did.get('accessed_at') or now,
+                        'access_cnt': case([(models.DataIdentifier.access_cnt == none_value, 1)],
+                                           else_=(models.DataIdentifier.access_cnt + 1))},
+                       synchronize_session=False)
     except DatabaseError:
         return False
 


Test patch:
diff --git a/lib/rucio/tests/test_did.py b/lib/rucio/tests/test_did.py
--- a/lib/rucio/tests/test_did.py
+++ b/lib/rucio/tests/test_did.py
@@ -7,7 +7,7 @@
 #
 # Authors:
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2013
-# - Thomas Beermann, <thomas.beermann@cern.ch>, 2013
+# - Thomas Beermann, <thomas.beermann@cern.ch>, 2013-2018
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2013-2015
 # - Yun-Pin Sun, <yun-pin.sun@cern.ch>, 2013
 # - Martin Barisits, <martin.barisits@cern.ch>, 2013-2015
@@ -32,7 +32,7 @@
 from rucio.common.utils import generate_uuid
 from rucio.core.account_limit import set_account_limit
 from rucio.core.did import (list_dids, add_did, delete_dids, get_did_atime, touch_dids, attach_dids,
-                            get_metadata, set_metadata, get_did)
+                            get_metadata, set_metadata, get_did, get_did_access_cnt)
 from rucio.core.rse import get_rse_id
 from rucio.core.replica import add_replica
 from rucio.db.sqla.constants import DIDType
@@ -58,7 +58,7 @@ def test_delete_dids(self):
             add_did(scope=tmp_scope, name=dsn['name'], type='DATASET', account='root')
         delete_dids(dids=dsns, account='root')
 
-    def test_touch_dids(self):
+    def test_touch_dids_atime(self):
         """ DATA IDENTIFIERS (CORE): Touch dids accessed_at timestamp"""
         tmp_scope = 'mock'
         tmp_dsn1 = 'dsn_%s' % generate_uuid()
@@ -76,6 +76,23 @@ def test_touch_dids(self):
         assert_equal(now, get_did_atime(scope=tmp_scope, name=tmp_dsn1))
         assert_equal(None, get_did_atime(scope=tmp_scope, name=tmp_dsn2))
 
+    def test_touch_dids_access_cnt(self):
+        """ DATA IDENTIFIERS (CORE): Increase dids access_cnt"""
+        tmp_scope = 'mock'
+        tmp_dsn1 = 'dsn_%s' % generate_uuid()
+        tmp_dsn2 = 'dsn_%s' % generate_uuid()
+
+        add_did(scope=tmp_scope, name=tmp_dsn1, type=DIDType.DATASET, account='root')
+        add_did(scope=tmp_scope, name=tmp_dsn2, type=DIDType.DATASET, account='root')
+
+        assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
+        assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))
+
+        for i in xrange(100):
+            touch_dids(dids=[{'scope': tmp_scope, 'name': tmp_dsn1, 'type': DIDType.DATASET}])
+        assert_equal(100, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn1))
+        assert_equal(None, get_did_access_cnt(scope=tmp_scope, name=tmp_dsn2))
+
     def test_update_dids(self):
         """ DATA IDENTIFIERS (CORE): Update file size and checksum"""
         tmp_scope = 'mock'