Repository: spotify/luigi
File: test/contrib/pai_test.py (Python)

Code before:
import logging
from luigi.contrib.pai import PaiTask
from luigi.contrib.pai import TaskRole

logging.basicConfig(level=logging.DEBUG)

"""
The following configurations are required to run the test
[OpenPai]
pai_url:http://host:port/
username:admin
password:admin-password
expiration:3600

"""


class SklearnJob(PaiTask):
    image = "openpai/pai.example.sklearn"
    name = "test_job_sk_{0}".format(time.time())
    command = 'cd scikit-learn/benchmarks && python bench_mnist.py'
    virtual_cluster = 'spark'
    tasks = [TaskRole('test', 'cd scikit-learn/benchmarks && python bench_mnist.py', memoryMB=4096)]


class FailJob(PaiTask):
    image = "openpai/pai.example.sklearn"
    name = "test_job_fail_{0}".format(time.time())
    command = 'cd scikit-learn/benchmarks && python bench_mnist.py'
    virtual_cluster = 'spark'
    tasks = [TaskRole('test', 'cd scikit-learn/benchmarks && python bench_mnist.py', memoryMB=4096)]


class TestPaiTask(unittest.TestCase):

    @responses.activate
    def test_success(self):
        """
        Here using the responses lib to mock the PAI rest api call, the following specify the response of the call.
        """
        responses.add(responses.POST, 'http://127.0.0.1:9186/api/v1/token',
                      json={"token": "test", "user": "admin", "admin": True}, status=200)
        sk_task = SklearnJob()

        responses.add(responses.POST, 'http://127.0.0.1:9186/api/v1/jobs',
                      json={"message": "update job {0} successfully".format(sk_task.name)}, status=202)

        responses.add(responses.GET, 'http://127.0.0.1:9186/api/v1/jobs/{0}'.format(sk_task.name),
                      json={}, status=404)

        responses.add(responses.GET, 'http://127.0.0.1:9186/api/v1/jobs/{0}'.format(sk_task.name),

Review comment (suggestion):
This class is identical to `SklearnJob`. Just keep `SklearnJob` and use it for both the successful test and the 404 test.

Code after:
import logging
from luigi.contrib.pai import PaiTask
from luigi.contrib.pai import TaskRole

logging.basicConfig(level=logging.DEBUG)

"""
The following configurations are required to run the test
[OpenPai]
pai_url:http://host:port/
username:admin
password:admin-password
expiration:3600

"""


class SklearnJob(PaiTask):
    image = "openpai/pai.example.sklearn"
    name = "test_job_sk_{0}".format(time.time())
    command = 'cd scikit-learn/benchmarks && python bench_mnist.py'
    virtual_cluster = 'spark'
    tasks = [TaskRole('test', 'cd scikit-learn/benchmarks && python bench_mnist.py', memoryMB=4096)]


class TestPaiTask(unittest.TestCase):

    @responses.activate
    def test_success(self):
        """
        Here using the responses lib to mock the PAI rest api call, the following specify the response of the call.
        """
        responses.add(responses.POST, 'http://127.0.0.1:9186/api/v1/token',
                      json={"token": "test", "user": "admin", "admin": True}, status=200)
        sk_task = SklearnJob()

        responses.add(responses.POST, 'http://127.0.0.1:9186/api/v1/jobs',
                      json={"message": "update job {0} successfully".format(sk_task.name)}, status=202)

        responses.add(responses.GET, 'http://127.0.0.1:9186/api/v1/jobs/{0}'.format(sk_task.name),
                      json={}, status=404)

        responses.add(responses.GET, 'http://127.0.0.1:9186/api/v1/jobs/{0}'.format(sk_task.name),
                      body='{"jobStatus": {"state":"SUCCEED"}}', status=200)

        success = luigi.build([sk_task], local_scheduler=True)
        self.assertTrue(success)
        self.assertTrue(sk_task.complete())

    @responses.activate
    def test_fail(self):