Repository: rucio/rucio

Issue:
Expose limit parameter to REST API
Motivation
----------
I am developing a JupyterLab extension that consumes Rucio REST API as a part of GSoC 2020 project. The extension uses DID search functionality that supports wildcard searches. However, the search endpoint can return plenty of rows that the UI frontend can't handle. Thankfully, Rucio's internal search method has a `limit` parameter, but not exposed to the REST API.


Modification
------------
The REST API handlers should be modified to include `limit` parameter.




Discussion:

Resolving patch:
diff --git a/lib/rucio/web/rest/flaskapi/v1/did.py b/lib/rucio/web/rest/flaskapi/v1/did.py
--- a/lib/rucio/web/rest/flaskapi/v1/did.py
+++ b/lib/rucio/web/rest/flaskapi/v1/did.py
@@ -23,6 +23,7 @@
 # - Martin Baristis <martin.barisits@cern.ch>, 2014-2015
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Muhammad Aditya Hilmy <mhilmy@hey.com>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -138,6 +139,7 @@ def get(self, scope):
              "bytes": 234, "length": 3}
 
         :query type: specify a DID type to search for
+        :query limit: The maximum number of DIDs returned.
         :query long: set to True for long output, otherwise only name
         :query recursive: set to True to recursively list DIDs content
         :query created_before: Date string in RFC-1123 format where the creation date was earlier
@@ -158,12 +160,15 @@ def get(self, scope):
         """
 
         filters = {}
+        limit = None
         long = False
         recursive = False
         type = 'collection'
         for k, v in request.args.items():
             if k == 'type':
                 type = v
+            elif k == 'limit':
+                limit = v[0]
             elif k == 'long':
                 long = v == '1'
             elif k == 'recursive':
@@ -173,7 +178,7 @@ def get(self, scope):
 
         try:
             data = ""
-            for did in list_dids(scope=scope, filters=filters, type=type, long=long, recursive=recursive, vo=request.environ.get('vo')):
+            for did in list_dids(scope=scope, filters=filters, type=type, limit=limit, long=long, recursive=recursive, vo=request.environ.get('vo')):
                 data += dumps(did) + '\n'
             return Response(data, content_type='application/x-json-stream')
         except UnsupportedOperation as error:
@@ -216,6 +221,7 @@ def get(self, scope):
              "bytes": 234, "length": 3}
 
         :query type: specify a DID type to search for
+        :query limit: The maximum number of DIDs returned.
         :query long: set to True for long output, otherwise only name
         :query recursive: set to True to recursively list DIDs content
         :query created_before: Date string in RFC-1123 format where the creation date was earlier
@@ -236,12 +242,15 @@ def get(self, scope):
         """
 
         filters = {}
+        limit = None
         long = False
         recursive = False
         type = 'collection'
         for k, v in request.args.items():
             if k == 'type':
                 type = v
+            elif k == 'limit':
+                limit = v[0]
             elif k == 'long':
                 long = v == '1'
             elif k == 'recursive':
@@ -251,7 +260,7 @@ def get(self, scope):
 
         try:
             data = ""
-            for did in list_dids_extended(scope=scope, filters=filters, type=type, long=long, recursive=recursive, vo=request.environ.get('vo')):
+            for did in list_dids_extended(scope=scope, filters=filters, type=type, limit=limit, long=long, recursive=recursive, vo=request.environ.get('vo')):
                 data += dumps(did) + '\n'
             return Response(data, content_type='application/x-json-stream')
         except UnsupportedOperation as error:
diff --git a/lib/rucio/web/rest/webpy/v1/did.py b/lib/rucio/web/rest/webpy/v1/did.py
--- a/lib/rucio/web/rest/webpy/v1/did.py
+++ b/lib/rucio/web/rest/webpy/v1/did.py
@@ -25,6 +25,7 @@
 # - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Muhammad Aditya Hilmy <mhilmy@hey.com>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -139,6 +140,7 @@ def GET(self, scope):
 
         header('Content-Type', 'application/x-json-stream')
         filters = {}
+        limit = None
         long = False
         recursive = False
         if ctx.query:
@@ -146,6 +148,8 @@ def GET(self, scope):
             for k, v in params.items():
                 if k == 'type':
                     type = v[0]
+                elif k == 'limit':
+                    limit = v[0]
                 elif k == 'long':
                     long = v[0] == '1'
                 elif k == 'recursive':
@@ -154,7 +158,7 @@ def GET(self, scope):
                     filters[k] = v[0]
 
         try:
-            for did in list_dids(scope=scope, filters=filters, type=type, long=long, recursive=recursive, vo=ctx.env.get('vo')):
+            for did in list_dids(scope=scope, filters=filters, type=type, limit=limit, long=long, recursive=recursive, vo=ctx.env.get('vo')):
                 yield dumps(did) + '\n'
         except UnsupportedOperation as error:
             raise generate_http_error(409, 'UnsupportedOperation', error.args[0])
@@ -186,6 +190,7 @@ def GET(self, scope):
 
         header('Content-Type', 'application/x-json-stream')
         filters = {}
+        limit = None
         long = False
         recursive = False
         if ctx.query:
@@ -193,6 +198,8 @@ def GET(self, scope):
             for k, v in params.items():
                 if k == 'type':
                     type = v[0]
+                elif k == 'limit':
+                    limit = v[0]
                 elif k == 'long':
                     long = v[0] == '1'
                 elif k == 'recursive':
@@ -201,7 +208,7 @@ def GET(self, scope):
                     filters[k] = v[0]
 
         try:
-            for did in list_dids_extended(scope=scope, filters=filters, type=type, long=long, recursive=recursive, vo=ctx.env.get('vo')):
+            for did in list_dids_extended(scope=scope, filters=filters, type=type, limit=limit, long=long, recursive=recursive, vo=ctx.env.get('vo')):
                 yield dumps(did) + '\n'
         except UnsupportedOperation as error:
             raise generate_http_error(409, 'UnsupportedOperation', error.args[0])


Test patch: