Repository: replicate/cog

Issue:
Use hostname for integration tests instead of IP
This seemed to be relying on some very subtle behavior. It seems that
the IP returned as the reverse of the local hostname was a local IP that
was just on the host but not inside the container (maybe 127.0.1.1
instead of 127.0.0.1 or something?).

Either way this wasn't working due to something about my local network
setup and my hostname resolved to 127.0.0.1. This is actually localhost
inside the container, hence it couldn't connect to redis.

This is a bodge to get this working. Instead of relying on IP addresses,
use the hostname which will instead defer to DNS to figure out what the
right IP address based on where it is being accessed.

But this is still fragile and I spent laods of time debugging! See also: #262

Signed-off-by: Ben Firshman <ben@firshman.co.uk>


Discussion:
This works on my local machine but not on CI, lol. I think we need the real fix in #262 unless y'all have any bright ideas for bodges @andreasjansson @zeke.

Resolving patch:
diff --git a/test-integration/upload_server/app.py b/test-integration/upload_server/app.py
new file mode 100644
--- /dev/null
+++ b/test-integration/upload_server/app.py
@@ -0,0 +1,27 @@
+import os
+
+import flask
+from flask import Flask, jsonify, send_from_directory
+
+app = Flask("upload_server")
+
+
+@app.route("/", methods=["GET"])
+def handle_index():
+    return "OK"
+
+
+@app.route("/upload", methods=["PUT"])
+def handle_upload():
+    f = flask.request.files["file"]
+    f.save(os.path.join("/uploads", f.filename))
+    return jsonify({"url": "http://upload-server:5000/download/{}".format(f.filename)})
+
+
+@app.route("/download/<name>")
+def download(name):
+    return send_from_directory("/uploads", name)
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5000, debug=True)


Test patch:
diff --git a/test-integration/test_integration/conftest.py b/test-integration/test_integration/conftest.py
--- a/test-integration/test_integration/conftest.py
+++ b/test-integration/test_integration/conftest.py
@@ -1,25 +1,76 @@
+from pathlib import Path
 import subprocess
 import pytest
 
-from .util import random_string, find_free_port, docker_run, get_local_ip, wait_for_port
+import redis
+
+from .util import random_string, find_free_port, docker_run, wait_for_port
 
 
 @pytest.fixture
-def redis_port():
-    container_name = "cog-test-redis-" + random_string(10)
+def docker_network():
+    name = "cog-test-" + random_string(10)
+    subprocess.run(["docker", "network", "create", name])
+    yield name
+    subprocess.run(["docker", "network", "rm", name])
+
+
+@pytest.fixture
+def redis_port(docker_network):
+    """Start a redis server inside the Docker network.
+    Inside the network, it is available at redis:6379
+    Outside the network, it is available at localhost:redis_port
+    """
     port = find_free_port()
     with docker_run(
         "redis",
-        name=container_name,
+        net_alias="redis",
+        network=docker_network,
         publish=[{"host": port, "container": 6379}],
         detach=True,
     ):
-        wait_for_port(get_local_ip(), port)
+        wait_for_port("localhost", port)
         yield port
 
 
+@pytest.fixture
+def redis_client(redis_port):
+    yield redis.Redis("localhost", redis_port)
+
+
 @pytest.fixture
 def docker_image():
     image = "cog-test-" + random_string(10)
     yield image
     subprocess.run(["docker", "rmi", "-f", image], check=False)
+
+
+@pytest.fixture
+def upload_server_image():
+    """
+    Build the upload server once for the test run. The image doesn't change.
+    """
+    subprocess.run(
+        ["docker", "build", "-t", "cog-test-upload-server", "."],
+        cwd=Path(__file__).parent.parent / "upload_server",
+        check=True,
+    )
+    return "cog-test-upload-server"
+
+
+@pytest.fixture
+def upload_server(docker_network, upload_server_image, tmpdir_factory):
+    """
+    Run a server that can be used to upload and download files from.
+
+    It is accessible at http://upload-server:5000 inside the network. The thing returned is the path for uploads.
+    """
+    tmpdir = tmpdir_factory.mktemp("uploads")
+    with docker_run(
+        upload_server_image,
+        net_alias="upload-server",
+        network=docker_network,
+        volumes=["-v", f"{tmpdir}:/uploads"],
+        detach=True,
+    ):
+        yield tmpdir
diff --git a/test-integration/test_integration/fixtures/file-project/predict.py b/test-integration/test_integration/fixtures/file-project/predict.py
--- a/test-integration/test_integration/fixtures/file-project/predict.py
+++ b/test-integration/test_integration/fixtures/file-project/predict.py
@@ -12,9 +12,7 @@ def setup(self):
     def predict(self, text, path):
         with open(path) as f:
             output = self.foo + text + f.read()
-        tmp = tempfile.NamedTemporaryFile(suffix=".txt")
-        tmp.close()
-        tmp_path = Path(tmp.name)
-        with tmp_path.open("w") as f:
-            f.write(output)
-            return tmp_path
+        tmpdir = Path(tempfile.mkdtemp())
+        with open(tmpdir / "output.txt", "w") as fh:
+            fh.write(output)
+        return tmpdir / "output.txt"
diff --git a/test-integration/test_integration/test_redis_queue.py b/test-integration/test_integration/test_redis_queue.py
--- a/test-integration/test_integration/test_redis_queue.py
+++ b/test-integration/test_integration/test_redis_queue.py
@@ -1,66 +1,41 @@
 import json
-import multiprocessing
-from contextlib import contextmanager
-from unittest import mock
-import pytest
 from pathlib import Path
-
-import flask
-import redis
-from flask import Flask, Response, jsonify
 import subprocess
 
-from .util import (
-    docker_run,
-    find_free_port,
-    get_bridge_ip,
-    get_local_ip,
-    random_string,
-    wait_for_port,
-)
+from .util import docker_run, random_string
 
 
-def test_queue_worker_files(docker_image, redis_port, request):
+def test_queue_worker_files(docker_image, docker_network, redis_client, upload_server):
     project_dir = Path(__file__).parent / "fixtures/file-project"
     subprocess.run(["cog", "build", "-t", docker_image], check=True, cwd=project_dir)
 
-    input_queue = multiprocessing.Queue()
-    output_queue = multiprocessing.Queue()
-    controller_port = find_free_port()
-    local_ip = get_local_ip()
-    upload_url = f"http://{local_ip}:{controller_port}/upload"
-    redis_host = local_ip
-    worker_name = "test-worker"
-    predict_queue_name = "predict-queue"
-    response_queue_name = "response-queue"
-
-    redis_client = redis.Redis(host=redis_host, port=redis_port, db=0)
-
-    with queue_controller(
-        input_queue, output_queue, controller_port, request
-    ), docker_run(
+    with open(upload_server / "input.txt", "w") as f:
+        f.write("test")
+
+    with docker_run(
         image=docker_image,
         interactive=True,
+        network=docker_network,
         command=[
             "python",
             "-m",
             "cog.server.redis_queue",
-            redis_host,
-            str(redis_port),
-            predict_queue_name,
-            upload_url,
-            worker_name,
+            "redis",
+            "6379",
+            "predict-queue",
+            "http://upload-server:5000/upload",
+            "test-worker",
             "model_id",
             "logs",
         ],
     ):
         redis_client.xgroup_create(
-            mkstream=True, groupname=predict_queue_name, name=predict_queue_name, id="$"
+            mkstream=True, groupname="predict-queue", name="predict-queue", id="$"
         )
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
@@ -69,67 +44,57 @@ def test_queue_worker_files(docker_image, redis_port, request):
                             "text": {"value": "baz"},
                             "path": {
                                 "file": {
-                                    "name": "myinput.txt",
-                                    "url": f"http://{local_ip}:{controller_port}/download",
+                                    "name": "input.txt",
+                                    "url": "http://upload-server:5000/download/input.txt",
                                 }
                             },
                         },
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
-        input_queue.put("test")
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {
-            "file": {"name": mock.ANY, "url": "uploaded.txt"},
+            "file": {
+                "name": "output.txt",
+                "url": "http://upload-server:5000/download/output.txt",
+            },
             "status": "success",
         }
-        response_contents = output_queue.get()
-        assert response_contents.decode() == "foobaztest"
 
+        with open(upload_server / "output.txt") as f:
+            assert f.read() == "foobaztest"
 
-def test_queue_worker_yielding(docker_image, redis_port, request):
+
+def test_queue_worker_yielding(docker_network, docker_image, redis_client):
     project_dir = Path(__file__).parent / "fixtures/yielding-project"
     subprocess.run(["cog", "build", "-t", docker_image], check=True, cwd=project_dir)
 
-    input_queue = multiprocessing.Queue()
-    output_queue = multiprocessing.Queue()
-    controller_port = find_free_port()
-    local_ip = get_local_ip()
-    upload_url = f"http://{local_ip}:{controller_port}/upload"
-    redis_host = local_ip
-    worker_name = "test-worker"
-    predict_queue_name = "predict-queue"
-    response_queue_name = "response-queue"
-
-    redis_client = redis.Redis(host=redis_host, port=redis_port, db=0)
-
-    with queue_controller(
-        input_queue, output_queue, controller_port, request
-    ), docker_run(
+    with docker_run(
         image=docker_image,
         interactive=True,
+        network=docker_network,
         command=[
             "python",
             "-m",
             "cog.server.redis_queue",
-            redis_host,
-            str(redis_port),
-            predict_queue_name,
-            upload_url,
-            worker_name,
+            "redis",
+            "6379",
+            "predict-queue",
+            "",
+            "test-worker",
             "model_id",
             "logs",
         ],
     ):
         redis_client.xgroup_create(
-            mkstream=True, groupname=predict_queue_name, name=predict_queue_name, id="$"
+            mkstream=True, groupname="predict-queue", name="predict-queue", id="$"
         )
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
@@ -137,66 +102,53 @@ def test_queue_worker_yielding(docker_image, redis_port, request):
                         "inputs": {
                             "text": {"value": "bar"},
                         },
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"value": "foo", "status": "processing"}
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"value": "bar", "status": "processing"}
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"value": "baz", "status": "success"}
 
-        response = redis_client.rpop(response_queue_name)
+        response = redis_client.rpop("response-queue")
         assert response == None
 
 
-def test_queue_worker_error(docker_image, redis_port, request):
+def test_queue_worker_error(docker_network, docker_image, redis_client):
     project_dir = Path(__file__).parent / "fixtures/failing-project"
     subprocess.run(["cog", "build", "-t", docker_image], check=True, cwd=project_dir)
 
-    input_queue = multiprocessing.Queue()
-    output_queue = multiprocessing.Queue()
-    controller_port = find_free_port()
-    local_ip = get_local_ip()
-    upload_url = f"http://{local_ip}:{controller_port}/upload"
-    redis_host = local_ip
-    worker_name = "test-worker"
-    predict_queue_name = "predict-queue"
-    response_queue_name = "response-queue"
-
-    redis_client = redis.Redis(host=redis_host, port=redis_port, db=0)
-
-    with queue_controller(
-        input_queue, output_queue, controller_port, request
-    ), docker_run(
+    with docker_run(
         image=docker_image,
         interactive=True,
+        network=docker_network,
         command=[
             "python",
             "-m",
             "cog.server.redis_queue",
-            redis_host,
-            str(redis_port),
-            predict_queue_name,
-            upload_url,
-            worker_name,
+            "redis",
+            "6379",
+            "predict-queue",
+            "",
+            "test-worker",
             "model_id",
             "logs",
         ],
     ):
         redis_client.xgroup_create(
-            mkstream=True, groupname=predict_queue_name, name=predict_queue_name, id="$"
+            mkstream=True, groupname="predict-queue", name="predict-queue", id="$"
         )
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
@@ -204,71 +156,58 @@ def test_queue_worker_error(docker_image, redis_port, request):
                         "inputs": {
                             "text": {"value": "bar"},
                         },
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"status": "failed", "error": "over budget"}
 
-        response = redis_client.rpop(response_queue_name)
+        response = redis_client.rpop("response-queue")
         assert response == None
 
 
-def test_queue_worker_logging(docker_image, redis_port, request):
+def test_queue_worker_logging(docker_network, docker_image, redis_client):
     project_dir = Path(__file__).parent / "fixtures/logging-project"
     subprocess.run(["cog", "build", "-t", docker_image], check=True, cwd=project_dir)
 
-    input_queue = multiprocessing.Queue()
-    output_queue = multiprocessing.Queue()
-    controller_port = find_free_port()
-    local_ip = get_local_ip()
-    upload_url = f"http://{local_ip}:{controller_port}/upload"
-    redis_host = local_ip
-    worker_name = "test-worker"
-    predict_queue_name = "predict-queue"
-    response_queue_name = "response-queue"
-
-    redis_client = redis.Redis(host=redis_host, port=redis_port, db=0)
-
-    with queue_controller(
-        input_queue, output_queue, controller_port, request
-    ), docker_run(
+    with docker_run(
         image=docker_image,
         interactive=True,
+        network=docker_network,
         command=[
             "python",
             "-m",
             "cog.server.redis_queue",
-            redis_host,
-            str(redis_port),
-            predict_queue_name,
-            upload_url,
-            worker_name,
+            "redis",
+            "6379",
+            "predict-queue",
+            "",
+            "test-worker",
             "model_id",
             "logs",
         ],
     ):
         redis_client.xgroup_create(
-            mkstream=True, groupname=predict_queue_name, name=predict_queue_name, id="$"
+            mkstream=True, groupname="predict-queue", name="predict-queue", id="$"
         )
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
                         "id": predict_id,
                         "inputs": {},
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"status": "success", "value": "output"}
 
         setup_log_lines = []
@@ -294,48 +233,35 @@ def test_queue_worker_logging(docker_image, redis_port, request):
         ]
 
 
-def test_queue_worker_timeout(docker_image, redis_port, request):
+def test_queue_worker_timeout(docker_network, docker_image, redis_client):
     project_dir = Path(__file__).parent / "fixtures/timeout-project"
     subprocess.run(["cog", "build", "-t", docker_image], check=True, cwd=project_dir)
 
-    input_queue = multiprocessing.Queue()
-    output_queue = multiprocessing.Queue()
-    controller_port = find_free_port()
-    local_ip = get_local_ip()
-    upload_url = f"http://{local_ip}:{controller_port}/upload"
-    redis_host = local_ip
-    worker_name = "test-worker"
-    predict_queue_name = "predict-queue"
-    response_queue_name = "response-queue"
-
-    redis_client = redis.Redis(host=redis_host, port=redis_port, db=0)
-
-    with queue_controller(
-        input_queue, output_queue, controller_port, request
-    ), docker_run(
+    with docker_run(
         image=docker_image,
         interactive=True,
+        network=docker_network,
         command=[
             "python",
             "-m",
             "cog.server.redis_queue",
-            redis_host,
-            str(redis_port),
-            predict_queue_name,
-            upload_url,
-            worker_name,
+            "redis",
+            "6379",
+            "predict-queue",
+            "",
+            "test-worker",
             "model_id",
             "logs",
             "1",  # timeout
         ],
     ):
         redis_client.xgroup_create(
-            mkstream=True, groupname=predict_queue_name, name=predict_queue_name, id="$"
+            mkstream=True, groupname="predict-queue", name="predict-queue", id="$"
         )
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
@@ -343,18 +269,18 @@ def test_queue_worker_timeout(docker_image, redis_port, request):
                         "inputs": {
                             "sleep_time": {"value": 0.5},
                         },
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"status": "success", "value": "it worked!"}
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
@@ -362,58 +288,45 @@ def test_queue_worker_timeout(docker_image, redis_port, request):
                         "inputs": {
                             "sleep_time": {"value": 5.0},
                         },
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"status": "failed", "error": "Prediction timed out"}
 
 
-def test_queue_worker_yielding_timeout(docker_image, redis_port, request):
+def test_queue_worker_yielding_timeout(docker_image, docker_network, redis_client):
     project_dir = Path(__file__).parent / "fixtures/yielding-timeout-project"
     subprocess.run(["cog", "build", "-t", docker_image], check=True, cwd=project_dir)
 
-    input_queue = multiprocessing.Queue()
-    output_queue = multiprocessing.Queue()
-    controller_port = find_free_port()
-    local_ip = get_local_ip()
-    upload_url = f"http://{local_ip}:{controller_port}/upload"
-    redis_host = local_ip
-    worker_name = "test-worker"
-    predict_queue_name = "predict-queue"
-    response_queue_name = "response-queue"
-
-    redis_client = redis.Redis(host=redis_host, port=redis_port, db=0)
-
-    with queue_controller(
-        input_queue, output_queue, controller_port, request
-    ), docker_run(
+    with docker_run(
         image=docker_image,
         interactive=True,
+        network=docker_network,
         command=[
             "python",
             "-m",
             "cog.server.redis_queue",
-            redis_host,
-            str(redis_port),
-            predict_queue_name,
-            upload_url,
-            worker_name,
+            "redis",
+            "6379",
+            "predict-queue",
+            "",
+            "test-worker",
             "model_id",
             "logs",
             "1",  # timeout
         ],
     ):
         redis_client.xgroup_create(
-            mkstream=True, groupname=predict_queue_name, name=predict_queue_name, id="$"
+            mkstream=True, groupname="predict-queue", name="predict-queue", id="$"
         )
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
@@ -422,18 +335,18 @@ def test_queue_worker_yielding_timeout(docker_image, redis_port, request):
                             "sleep_time": {"value": 0.5},
                             "n_iterations": {"value": 1},
                         },
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"status": "success", "value": "yield 0"}
 
         predict_id = random_string(10)
         redis_client.xadd(
-            name=predict_queue_name,
+            name="predict-queue",
             fields={
                 "value": json.dumps(
                     {
@@ -442,57 +355,15 @@ def test_queue_worker_yielding_timeout(docker_image, redis_port, request):
                             "sleep_time": {"value": 0.7},
                             "n_iterations": {"value": 10},
                         },
-                        "response_queue": response_queue_name,
+                        "response_queue": "response-queue",
                     }
                 ),
             },
         )
 
         # TODO(andreas): revisit this test design if it starts being flakey
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"value": "yield 0", "status": "processing"}
 
-        response = json.loads(redis_client.brpop(response_queue_name, timeout=10)[1])
+        response = json.loads(redis_client.brpop("response-queue", timeout=10)[1])
         assert response == {"status": "failed", "error": "Prediction timed out"}
-
-
-@contextmanager
-def queue_controller(input_queue, output_queue, controller_port, request):
-    controller = QueueController(input_queue, output_queue, controller_port)
-    request.addfinalizer(controller.kill)
-    controller.start()
-    yield controller
-    controller.kill()
-
-
-class QueueController(multiprocessing.Process):
-    def __init__(self, input_queue, output_queue, port):
-        super().__init__()
-        self.input_queue = input_queue
-        self.output_queue = output_queue
-        self.port = port
-
-    def run(self):
-        app = Flask("test-queue-controller")
-
-        @app.route("/", methods=["GET"])
-        def handle_index():
-            return "OK"
-
-        @app.route("/upload", methods=["PUT"])
-        def handle_upload():
-            f = flask.request.files["file"]
-            contents = f.read()
-            self.output_queue.put(contents)
-            return jsonify({"url": "uploaded.txt"})
-
-        @app.route("/download", methods=["GET"])
-        def handle_download():
-            contents = self.input_queue.get()
-            return Response(
-                contents,
-                mimetype="text/plain",
-                headers={"Content-disposition": "attachment; filename=myinput.txt"},
-            )
-
-        app.run(host="0.0.0.0", port=self.port, debug=False)
diff --git a/test-integration/test_integration/util.py b/test-integration/test_integration/util.py
--- a/test-integration/test_integration/util.py
+++ b/test-integration/test_integration/util.py
@@ -38,8 +38,11 @@ def docker_run(
     name=None,
     detach=False,
     interactive=False,
+    network=None,
+    net_alias=None,
     publish: Optional[List[dict]] = None,
     command: Optional[List[str]] = None,
+    volumes: Optional[List[str]] = None,
     env: Optional[dict] = None,
 ):
     if name is None:
@@ -54,10 +57,17 @@ def docker_run(
     if env is not None:
         for key, value in env.items():
             cmd += ["-e", f"{key}={value}"]
+    if volumes is not None:
+        for volume in volumes:
+            cmd += ["--volume", volume]
     if detach:
         cmd += ["--detach"]
     if interactive:
         cmd += ["-i"]
+    if network is not None:
+        cmd.extend(["--network", network])
+    if net_alias is not None:
+        cmd.extend(["--net-alias", net_alias])
     cmd += [image]
     if command:
         cmd += command
@@ -66,21 +76,3 @@ def docker_run(
         yield
     finally:
         subprocess.Popen(["docker", "rm", "--force", name]).wait()
-
-
-def get_local_ip():
-    return socket.gethostbyname(socket.gethostname())
-
-
-def get_bridge_ip():
-    """Return the IP address of the docker bridge network"""
-    cmd = [
-        "docker",
-        "network",
-        "inspect",
-        "bridge",
-        "--format='{{ json (index .IPAM.Config 0).Gateway }}'",
-    ]
-    out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()
-
-    return out.decode().strip().replace('"', "").replace("'", "")