Repository: rucio/rucio

Issue:
Metalink redirector sort=geoip seems to remove all urls
Motivation
----------

```
$ curl -k -s https://rucio-lb-prod.cern.ch/redirect/data17_13TeV/data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data/metalink?schemes=root
<?xml version="1.0" encoding="UTF-8"?>
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
 <file name="data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data">
  <identity>data17_13TeV:data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data</identity>
  <hash type="adler32">386e681b</hash>
  <size>2621968192</size>
  <glfn name="/atlas/rucio/data17_13TeV:data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data"></glfn>
  <url location="BNL-OSG2_DATADISK" priority="1">root://dcdoor11.usatlas.bnl.gov:1094//pnfs/usatlas.bnl.gov/BNLT0D1/rucio/data17_13TeV/e2/5b/data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data</url>
 </file>
</metalink>
```

But when `sort=geoip` is added:

```
$ curl -k -s https://rucio-lb-prod.cern.ch/redirect/data17_13TeV/data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data/metalink?schemes=root\?sort=geoip
<?xml version="1.0" encoding="UTF-8"?>
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
 <file name="data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data">
  <identity>data17_13TeV:data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data</identity>
  <hash type="adler32">386e681b</hash>
  <size>2621968192</size>
  <glfn name="/atlas/rucio/data17_13TeV:data17_13TeV.00341534.physics_MinBias.daq.RAW._lb1254._SFO-2._0003.data"></glfn>
 </file>
</metalink>
```

Modification
------------
Something seems to fail in the geoip_sorter.


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/replica_sorter.py b/lib/rucio/common/replica_sorter.py
--- a/lib/rucio/common/replica_sorter.py
+++ b/lib/rucio/common/replica_sorter.py
@@ -13,19 +13,21 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2014
+# - Cedric Serfon, <cedric.serfon@cern.ch>, 2014-2019
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2017
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2017
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # This product includes GeoLite data created by MaxMind,
 # available from <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+#
+# PY3K COMPATIBLE
 
 from __future__ import print_function
 
-import gzip
 import os
 import random
 import socket
+import tarfile
 import time
 
 from math import asin, cos, radians, sin, sqrt
@@ -33,7 +35,6 @@
 from dogpile.cache import make_region
 
 import requests
-import pygeoip
 import geoip2.database
 
 from rucio.common import utils
@@ -47,95 +48,85 @@
 
 
 def __download_geoip_db(directory, filename):
-    path = 'http://geolite.maxmind.com/download/geoip/database/%s.gz' % (filename)
+    path = 'https://geolite.maxmind.com/download/geoip/database/%s.tar.gz' % (filename)
     try:
-        os.unlink('%s/%s.gz' % (directory, filename))
+        os.unlink('%s/%s.tar.gz' % (directory, filename))
     except OSError:
         pass
     result = requests.get(path, stream=True)
+    print(result)
     if result and result.status_code in [200, ]:
-        f = open('%s/%s.gz' % (directory, filename), 'wb')
+        file_object = open('%s/%s.tar.gz' % (directory, filename), 'wb')
         for chunk in result.iter_content(8192):
-            f.write(chunk)
-        f.close()
+            file_object.write(chunk)
+        file_object.close()
+        tarfile_name = '%s/%s.tar.gz' % (directory, filename)
+        with tarfile.open(name=tarfile_name, mode='r:gz') as tfile:
+            tfile.extractall(path=directory)
+            for entry in tfile:
+                if entry.name.find('%s.mmdb' % filename) > -1:
+                    print('Will move %s/%s to %s/%s' % (directory, entry.name, directory, entry.name.split('/')[-1]))
+                    os.rename('%s/%s' % (directory, entry.name), '%s/%s' % (directory, entry.name.split('/')[-1]))
+    else:
+        raise Exception('Cannot download geoip DB file. Status code %s' % result.status_code)
 
 
 def __get_geoip_db(directory, filename):
     if directory.endswith('/'):
         directory = directory[:-1]
-    if not os.path.isfile('%s/%s' % (directory, filename)):
+    if not os.path.isfile('%s/%s.mmdb' % (directory, filename)):
         print('%s does not exist. Downloading it.' % (filename))
         __download_geoip_db(directory, filename)
-    elif (time.time() - os.stat('%s/%s' % (directory, filename)).st_atime > 30 * 86400):
+    elif time.time() - os.stat('%s/%s.mmdb' % (directory, filename)).st_atime > 30 * 86400:
         print('%s is too old. Re-downloading it.' % (filename))
         __download_geoip_db(directory, filename)
-    else:
-        return
-    f = gzip.open('%s/%s.gz' % (directory, filename), 'rb')
-    file_content = f.read()
-    f.close()
-    g = open('%s/%s' % (directory, filename), 'wb')
-    g.write(file_content)
-    g.close()
-    os.unlink('%s/%s.gz' % (directory, filename))
     return
 
 
-def __get_lat_long(se, gi, gi2):
+def __get_lat_long(se, gi):
     """
     Get the latitude and longitude on one host using the GeoLite DB
     :param se  : A hostname or IP.
-    :param gi  : A GeoIP object (pygeoip API for IPv4).
-    :param gi2 : A Reader object (geoip2 API for IPv6).
+    :param gi : A Reader object (geoip2 API).
     """
     try:
-        ip = socket.gethostbyname(se)
-        d = gi.record_by_addr(ip)
-        return d['latitude'], d['longitude']
-    except socket.gaierror as e:
-        try:
-            # Host unknown. It might be IPv6. Trying with geoip2
-            print(e)
-            ip = socket.getaddrinfo(se, None)[0][4][0]
-            response = gi2.city(ip)
-            return response.location.latitude, response.location.longitude
-        except socket.gaierror as e:
-            # Host definitively unknown
-            print(e)
-            return None, None
+        ip = socket.getaddrinfo(se, None)[0][4][0]
+        response = gi.city(ip)
+        return response.location.latitude, response.location.longitude
+    except socket.gaierror as error:
+        # Host definitively unknown
+        print(error)
+    return None, None
 
 
 @REGION.cache_on_arguments(namespace='site_distance')
-def __get_distance(se1, se2):
+def __get_distance(se1, se2, ignore_error):
     """
     Get the distance between 2 host using the GeoLite DB
     :param se1 : A first hostname or IP.
     :param se2 : A second hostname or IP.
+    :ignore_error: Ignore exception when the GeoLite DB cannot be retrieved
     """
     directory = '/tmp'
-    filename = 'GeoLiteCity.dat'
-    __get_geoip_db(directory, filename)
+    ipv6_filename = 'GeoLite2-City'
+    try:
+        __get_geoip_db(directory, ipv6_filename)
 
-    directory = '/tmp'
-    ipv6_filename = 'GeoLite2-City.mmdb'
-    __get_geoip_db(directory, ipv6_filename)
-
-    gi = pygeoip.GeoIP('%s/%s' % (directory, filename))
-    gi2 = geoip2.database.Reader('%s/%s' % (directory, ipv6_filename))
-
-    lat1, long1 = __get_lat_long(se1, gi, gi2)
-    lat2, long2 = __get_lat_long(se2, gi, gi2)
-
-    if lat1 and lat2:
-        long1, lat1, long2, lat2 = map(radians, [long1, lat1, long2, lat2])
-        dlon = long2 - long1
-        dlat = lat2 - lat1
-        a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
-        c = 2 * asin(sqrt(a))
-        return 6378 * c
-    else:
-        # One host is on the Moon
-        return 360000
+        gi = geoip2.database.Reader('%s/%s' % (directory, '%s.mmdb' % ipv6_filename))
+
+        lat1, long1 = __get_lat_long(se1, gi)
+        lat2, long2 = __get_lat_long(se2, gi)
+
+        if lat1 and lat2:
+            long1, lat1, long2, lat2 = map(radians, [long1, lat1, long2, lat2])
+            dlon = long2 - long1
+            dlat = lat2 - lat1
+            return 6378 * 2 * asin(sqrt(sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2))
+    except Exception as error:
+        if not ignore_error:
+            raise error
+    # One host is on the Moon
+    return 360000
 
 
 def site_selector(replicas, site):
@@ -170,19 +161,20 @@ def sort_random(replicas):
     return list_replicas
 
 
-def sort_geoip(replicas, client_ip):
+def sort_geoip(replicas, client_ip, ignore_error=False):
     """
     Return a list of replicas sorted by geographical distance to the client IP.
     :param replicas : A dict with RSEs as values and replicas as keys (URIs).
     :param client_ip: The IP of the client.
+    :ignore_error: Ignore exception when the GeoLite DB cannot be retrieved
     """
 
     distances = {}
     for replica in replicas:
         se = replica.split('/')[2].split(':')[0]
-        distance = __get_distance(se, client_ip)
+        distance = __get_distance(se, client_ip, ignore_error)
         distances[replica] = distance
-    tmp = map(lambda x: x[0], sorted(list(distances.items()), key=lambda x: x[1]))
+    tmp = [x[0] for x in sorted(list(distances.items()), key=lambda x: x[1])]
 
     return tmp
 
diff --git a/lib/rucio/web/rest/flaskapi/v1/redirect.py b/lib/rucio/web/rest/flaskapi/v1/redirect.py
--- a/lib/rucio/web/rest/flaskapi/v1/redirect.py
+++ b/lib/rucio/web/rest/flaskapi/v1/redirect.py
@@ -15,7 +15,7 @@
 #
 # Authors:
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2014-2017
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
@@ -123,7 +123,7 @@ def get(self, scope, name):
 
                 # sort the actual replicas if necessary
                 if select == 'geoip':
-                    replicas = sort_geoip(dictreplica, client_location['ip'])
+                    replicas = sort_geoip(dictreplica, client_location['ip'], ignore_error=True)
                 elif select == 'closeness':
                     replicas = sort_closeness(dictreplica, client_location)
                 elif select == 'dynamic':
diff --git a/lib/rucio/web/rest/webpy/v1/redirect.py b/lib/rucio/web/rest/webpy/v1/redirect.py
--- a/lib/rucio/web/rest/webpy/v1/redirect.py
+++ b/lib/rucio/web/rest/webpy/v1/redirect.py
@@ -15,7 +15,7 @@
 #
 # Authors:
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2014-2017
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 #
@@ -134,7 +134,7 @@ def GET(self, scope, name):
 
                 # sort the actual replicas if necessary
                 if select == 'geoip':
-                    replicas = sort_geoip(dictreplica, client_location['ip'])
+                    replicas = sort_geoip(dictreplica, client_location['ip'], ignore_error=True)
                 elif select == 'closeness':
                     replicas = sort_closeness(dictreplica, client_location)
                 elif select == 'dynamic':


Test patch: