Repository: Cloud-CV/EvalAI

Issue:
Move connection params out of worker to settings
We are using RabbitMQ to publish messages for submission and execute them. RabbitMQ needs some connection parameters like host name, exchange name, queue name, etc.

To consume message from the queue, a worker is written. This workers lives at `scripts/workers/submission_worker.py`.  

Presently the connection as well as exchange parameters are hardcoded in that file. Ideally they should be present in environment variables. But as of now we can avoid keeping them in environment variables and instead leverage Django settings.

So this task involves moving the connection parameters to settings. Concerned parameters are 
* host
* exchange name
* exchange type
* submission queue name
* submission queue parameters

As of now all the parameters can live in base settings. Also please make sure that you define a dict something like `RABBITMQ_PARAMETERS` and within it multiple keys for host, exchange and all.
So final value of variable should look like

```
RABBITMQ_PARAMETERS = {
    'HOST': 'value',
    'EXCHANGE_NAME': 'value'
    'SUBMISSION_QUEUE': {
        'NAME': 'value',
        ....
    }
}
```




Discussion:
Can I work on this issue?
@mayank-agarwal-96 : sure. you can work on it.
Should I make the changes in 'settings.dev' or 'settings.common'?

Resolving patch:
diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py
--- a/scripts/workers/submission_worker.py
+++ b/scripts/workers/submission_worker.py
@@ -17,7 +17,7 @@
 
 from django.core.files.base import ContentFile
 from django.utils import timezone
-
+from django.conf import settings
 # need to add django project path in sys path
 # root directory : where manage.py lives
 # worker is present in root-directory/scripts/workers
@@ -49,7 +49,6 @@
 
 from jobs.models import Submission          # noqa
 
-
 CHALLENGE_DATA_BASE_DIR = join(COMPUTE_DIRECTORY_PATH, 'challenge_data')
 SUBMISSION_DATA_BASE_DIR = join(COMPUTE_DIRECTORY_PATH, 'submission_files')
 CHALLENGE_DATA_DIR = join(CHALLENGE_DATA_BASE_DIR, 'challenge_{challenge_id}')
@@ -59,7 +58,6 @@
 SUBMISSION_DATA_DIR = join(SUBMISSION_DATA_BASE_DIR, 'submission_{submission_id}')
 SUBMISSION_INPUT_FILE_PATH = join(SUBMISSION_DATA_DIR, '{input_file}')
 CHALLENGE_IMPORT_STRING = 'challenge_data.challenge_{challenge_id}'
-
 EVALUATION_SCRIPTS = {}
 
 # map of challenge id : phase id : phase annotation file name
@@ -446,12 +444,13 @@ def main():
     sys.path.append(COMPUTE_DIRECTORY_PATH)
 
     load_active_challenges()
-
     connection = pika.BlockingConnection(pika.ConnectionParameters(
-        host='localhost'))
+        host=settings.RABBITMQ_PARAMETERS['HOST']))
 
     channel = connection.channel()
-    channel.exchange_declare(exchange='evalai_submissions', type='topic')
+    channel.exchange_declare(
+        exchange=settings.RABBITMQ_PARAMETERS['EVALAI_EXCHANGE']['NAME'],
+        type=settings.RABBITMQ_PARAMETERS['EVALAI_EXCHANGE']['TYPE'])
 
     # name can be a combination of hostname + process id
     # host name : to easily identify that the worker is running on which instance
@@ -459,7 +458,9 @@ def main():
     add_challenge_queue_name = '{hostname}_{process_id}'.format(hostname=socket.gethostname(),
                                                                 process_id=str(os.getpid()))
 
-    channel.queue_declare(queue='submission_task_queue', durable=True)
+    channel.queue_declare(
+        queue=settings.RABBITMQ_PARAMETERS['SUBMISSION_QUEUE'],
+        durable=True)
 
     # reason for using `exclusive` instead of `autodelete` is that
     # challenge addition queue should have only have one consumer on the connection
@@ -470,10 +471,17 @@ def main():
     # create submission base data directory
     create_dir_as_python_package(SUBMISSION_DATA_BASE_DIR)
 
-    channel.queue_bind(exchange='evalai_submissions', queue='submission_task_queue', routing_key='submission.*.*')
-    channel.basic_consume(process_submission_callback, queue='submission_task_queue')
-
-    channel.queue_bind(exchange='evalai_submissions', queue=add_challenge_queue_name, routing_key='challenge.add.*')
+    channel.queue_bind(
+        exchange=settings.RABBITMQ_PARAMETERS['EVALAI_EXCHANGE']['NAME'],
+        queue=settings.RABBITMQ_PARAMETERS['SUBMISSION_QUEUE'],
+        routing_key='submission.*.*')
+    channel.basic_consume(
+        process_submission_callback,
+        queue=settings.RABBITMQ_PARAMETERS['SUBMISSION_QUEUE'])
+
+    channel.queue_bind(
+        exchange=settings.RABBITMQ_PARAMETERS['EVALAI_EXCHANGE']['NAME'],
+        queue=add_challenge_queue_name, routing_key='challenge.add.*')
     channel.basic_consume(add_challenge_callback, queue=add_challenge_queue_name)
 
     channel.start_consuming()
diff --git a/settings/common.py b/settings/common.py
--- a/settings/common.py
+++ b/settings/common.py
@@ -261,3 +261,12 @@
         'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
     }
 }
+
+RABBITMQ_PARAMETERS = {
+    'HOST': 'localhost',
+    'EVALAI_EXCHANGE': {
+        'NAME': 'evalai_submissions',
+        'TYPE': 'topic',
+    },
+    'SUBMISSION_QUEUE': 'submission_task_queue',
+}


Test patch: