Repository: opsdroid/opsdroid

Issue:
Update rasa parser to work with rasa 2
<!-- Before you post an issue or if you are unsure about something join our matrix channel https://app.element.io/#/room/#opsdroid-general:matrix.org and ask away! We are more than happy to help you. -->
# Description

I am still investigating but it looks like Rasa 2 has a different http API and our matcher does not work with it.


Discussion:

Resolving patch:
diff --git a/opsdroid/const.py b/opsdroid/const.py
--- a/opsdroid/const.py
+++ b/opsdroid/const.py
@@ -32,7 +32,7 @@
 REGEX_PARSE_SCORE_FACTOR = 0.6
 
 RASANLU_DEFAULT_URL = "http://localhost:5000"
-RASANLU_DEFAULT_PROJECT = "opsdroid"
+RASANLU_DEFAULT_MODELS_PATH = "models"
 
 LUISAI_DEFAULT_URL = "https://westus.api.cognitive.microsoft.com/luis/v2.0/apps/"
 
diff --git a/opsdroid/parsers/rasanlu.py b/opsdroid/parsers/rasanlu.py
--- a/opsdroid/parsers/rasanlu.py
+++ b/opsdroid/parsers/rasanlu.py
@@ -9,10 +9,10 @@
 import aiohttp
 import arrow
 
-from opsdroid.const import RASANLU_DEFAULT_URL, RASANLU_DEFAULT_PROJECT
+from opsdroid.const import RASANLU_DEFAULT_URL, RASANLU_DEFAULT_MODELS_PATH
 
 _LOGGER = logging.getLogger(__name__)
-CONFIG_SCHEMA = {"url": str, "project": str, "token": str, "min-score": float}
+CONFIG_SCHEMA = {"url": str, "token": str, "models-path": str, "min-score": float}
 
 
 async def _get_all_intents(skills):
@@ -31,21 +31,22 @@ async def _get_intents_fingerprint(intents):
 
 async def _build_training_url(config):
     """Build the url for training a Rasa NLU model."""
-    url = "{}/train?project={}&fixed_model_name={}".format(
+    url = "{}/model/train".format(
         config.get("url", RASANLU_DEFAULT_URL),
-        config.get("project", RASANLU_DEFAULT_PROJECT),
-        config["model"],
     )
 
     if "token" in config:
-        url += "&token={}".format(config["token"])
+        url += "?&token={}".format(config["token"])
 
     return url
 
 
 async def _build_status_url(config):
     """Build the url for getting the status of Rasa NLU."""
-    return "{}/status".format(config.get("url", RASANLU_DEFAULT_URL))
+    url = "{}/status".format(config.get("url", RASANLU_DEFAULT_URL))
+    if "token" in config:
+        url += "?&token={}".format(config["token"])
+    return url
 
 
 async def _init_model(config):
@@ -65,20 +66,88 @@ async def _init_model(config):
     return True
 
 
-async def _get_existing_models(config):
-    """Get a list of models already trained in the Rasa NLU project."""
-    project = config.get("project", RASANLU_DEFAULT_PROJECT)
+async def _get_rasa_nlu_version(config):
+    """Get Rasa NLU version data"""
+    async with aiohttp.ClientSession(trust_env=True) as session:
+        url = config.get("url", RASANLU_DEFAULT_URL) + "/version"
+        try:
+            resp = await session.get(url)
+        except aiohttp.client_exceptions.ClientConnectorError:
+            _LOGGER.error(_("Unable to connect to Rasa NLU."))
+            return None
+        if resp.status == 200:
+            result = await resp.json()
+            _LOGGER.debug(_("Rasa NLU response - %s."), json.dumps(result))
+        else:
+            result = await resp.text()
+            _LOGGER.error(_("Bad Rasa NLU response - %s."), result)
+        return result
+
+
+async def _check_rasanlu_compatibility(config):
+    """Check if Rasa NLU is compatible with the API we implement"""
+    _LOGGER.debug(_("Checking Rasa NLU version."))
+    json_object = await _get_rasa_nlu_version(config)
+    version = json_object["version"]
+    minimum_compatible_version = json_object["minimum_compatible_version"]
+    # Make sure we don't run against a 1.x.x Rasa NLU because it has a different API
+    if int(minimum_compatible_version[0:1]) >= 2:
+        _LOGGER.debug(_("Rasa NLU version {}.".format(version)))
+        return True
+
+    _LOGGER.error(
+        _(
+            "Incompatible Rasa NLU version ({}). Use Rasa Version >= 2.X.X.".format(
+                version
+            )
+        )
+    )
+    return False
+
+
+async def _load_model(config):
+    """Load model from the filesystem of the Rasa NLU environment"""
+    async with aiohttp.ClientSession(trust_env=True) as session:
+        headers = {}
+        data = {
+            "model_file": "{}/{}".format(
+                config.get("models-path", RASANLU_DEFAULT_MODELS_PATH),
+                config["model_filename"],
+            ),
+        }
+        url = config.get("url", RASANLU_DEFAULT_URL) + "/model"
+        if "token" in config:
+            url += "?token={}".format(config["token"])
+        try:
+            resp = await session.put(url, data=json.dumps(data), headers=headers)
+        except aiohttp.client_exceptions.ClientConnectorError:
+            _LOGGER.error(_("Unable to connect to Rasa NLU."))
+            return None
+        if resp.status == 204:
+            result = await resp.json()
+        else:
+            result = await resp.text()
+            _LOGGER.error(_("Bad Rasa NLU response - %s."), result)
+
+        return result
+
+
+async def _is_model_loaded(config):
+    """Check whether the model is loaded in Rasa NLU"""
     async with aiohttp.ClientSession(trust_env=True) as session:
+        url = config.get("url", RASANLU_DEFAULT_URL) + "/status"
+        if "token" in config:
+            url += "?token={}".format(config["token"])
         try:
             resp = await session.get(await _build_status_url(config))
-            if resp.status == 200:
-                result = await resp.json()
-                if project in result["available_projects"]:
-                    project_models = result["available_projects"][project]
-                    return project_models["available_models"]
-        except aiohttp.ClientOSError:
-            pass
-    return []
+        except aiohttp.client_exceptions.ClientConnectorError:
+            _LOGGER.error(_("Unable to connect to Rasa NLU."))
+            return None
+        if resp.status == 200:
+            result = await resp.json()
+            if result["model_file"].find(config["model_filename"]):
+                return True
+        return False
 
 
 async def train_rasanlu(config, skills):
@@ -89,23 +158,19 @@ async def train_rasanlu(config, skills):
         _LOGGER.warning(_("No intents found, skipping training."))
         return False
 
-    config["model"] = await _get_intents_fingerprint(intents)
-    if config["model"] in await _get_existing_models(config):
-        _LOGGER.info(_("This model already exists, skipping training..."))
-        await _init_model(config)
-        return True
+    await _check_rasanlu_compatibility(config)
+
+    """
+    TODO: think about how to correlate intent with trained model
+          so we can just load the model without training it again if it wasn't changed
+    """
 
     async with aiohttp.ClientSession(trust_env=True) as session:
         _LOGGER.info(_("Now training the model. This may take a while..."))
 
         url = await _build_training_url(config)
 
-        # https://github.com/RasaHQ/rasa_nlu/blob/master/docs/http.rst#post-train
-        # Note : The request should always be sent as
-        # application/x-yml regardless of wether you use
-        # json or md for the data format. Do not send json as
-        # application/json for example.+
-        headers = {"content-type": "application/x-yml"}
+        headers = {"Content-Type": "application/x-yaml"}
 
         try:
             training_start = arrow.now()
@@ -115,39 +180,41 @@ async def train_rasanlu(config, skills):
             return False
 
         if resp.status == 200:
-            if resp.content_type == "application/json":
-                result = await resp.json()
-                if "info" in result and "new model trained" in result["info"]:
-                    time_taken = (arrow.now() - training_start).total_seconds()
-                    _LOGGER.info(
-                        _("Rasa NLU training completed in %s seconds."), int(time_taken)
-                    )
-                    await _init_model(config)
-                    return True
-
-                _LOGGER.debug(result)
             if (
-                resp.content_type == "application/zip"
+                resp.content_type == "application/x-tar"
                 and resp.content_disposition.type == "attachment"
             ):
                 time_taken = (arrow.now() - training_start).total_seconds()
                 _LOGGER.info(
                     _("Rasa NLU training completed in %s seconds."), int(time_taken)
                 )
-                await _init_model(config)
+                config["model_filename"] = resp.content_disposition.filename
+                # close the connection and don't retrieve the model tar
+                # because using it is currently not implemented
+                resp.close()
+
                 """
-                As inditated in the issue #886, returned zip file is ignored, this can be changed
-                This can be changed in future release if needed
-                Saving model.zip file example :
+                model_path = "/tmp/{}".format(resp.content_disposition.filename)
                 try:
-                    output_file = open("/target/directory/model.zip","wb")
+                    output_file = open(model_path,"wb")
                     data = await resp.read()
                     output_file.write(data)
                     output_file.close()
-                    _LOGGER.debug("Rasa taining model file saved to /target/directory/model.zip")
+                    _LOGGER.debug("Rasa taining model file saved to {}", model_path)
                 except:
-                    _LOGGER.error("Cannot save rasa taining model file to /target/directory/model.zip")
+                    _LOGGER.error("Cannot save rasa taining model file to {}", model_path)
                 """
+
+                await _load_model(config)
+                # Check if the current trained model is loaded
+                if await _is_model_loaded(config):
+                    _LOGGER.info(_("Successfully loaded Rasa NLU model."))
+                else:
+                    _LOGGER.error(_("Failed getting Rasa NLU server status."))
+                    return False
+
+                # Check if we will get a valid response from Rasa
+                await call_rasanlu("", config)
                 return True
 
         _LOGGER.error(_("Bad Rasa NLU response - %s."), await resp.text())
@@ -159,14 +226,10 @@ async def call_rasanlu(text, config):
     """Call the Rasa NLU api and return the response."""
     async with aiohttp.ClientSession(trust_env=True) as session:
         headers = {}
-        data = {
-            "q": text,
-            "project": config.get("project", "default"),
-            "model": config.get("model", "fallback"),
-        }
+        data = {"text": text}
+        url = config.get("url", RASANLU_DEFAULT_URL) + "/model/parse"
         if "token" in config:
-            data["token"] = config["token"]
-        url = config.get("url", RASANLU_DEFAULT_URL) + "/parse"
+            url += "?&token={}".format(config["token"])
         try:
             resp = await session.post(url, data=json.dumps(data), headers=headers)
         except aiohttp.client_exceptions.ClientConnectorError:


Test patch:
diff --git a/tests/test_parser_rasanlu.py b/tests/test_parser_rasanlu.py
--- a/tests/test_parser_rasanlu.py
+++ b/tests/test_parser_rasanlu.py
@@ -415,33 +415,25 @@ async def test__get_intents_fingerprint(self):
         )
 
     async def test__build_training_url(self):
-        config = {}
-        with self.assertRaises(KeyError):
-            await rasanlu._build_training_url(config)
-
-        config = {"model": "helloworld"}
-        url = await rasanlu._build_training_url(config)
-        self.assertTrue("helloworld" in url)
-
-        config = {"model": "helloworld", "token": "abc123"}
+        config = {"token": "abc123"}
         url = await rasanlu._build_training_url(config)
         self.assertTrue("&token=abc123" in url)
 
         config = {
             "url": "http://example.com",
-            "project": "myproject",
-            "model": "helloworld",
         }
         url = await rasanlu._build_training_url(config)
         self.assertTrue("http://example.com" in url)
-        self.assertTrue("myproject" in url)
-        self.assertTrue("helloworld" in url)
 
     async def test__build_status_url(self):
         config = {"url": "http://example.com"}
         url = await rasanlu._build_status_url(config)
         self.assertTrue("http://example.com" in url)
 
+        config = {"url": "http://example.com", "token": "token123"}
+        url = await rasanlu._build_status_url(config)
+        self.assertTrue("&token=token123" in url)
+
     async def test__init_model(self):
         with amock.patch.object(rasanlu, "call_rasanlu") as mocked_call:
             mocked_call.return_value = {"data": "Some data"}
@@ -450,41 +442,135 @@ async def test__init_model(self):
             mocked_call.return_value = None
             self.assertEqual(await rasanlu._init_model({}), False)
 
-    async def test__get_existing_models(self):
+    async def test__get_rasa_nlu_version(self):
         result = amock.Mock()
         result.status = 200
+        result.text = amock.CoroutineMock()
         result.json = amock.CoroutineMock()
-        result.json.return_value = {
-            "available_projects": {
-                "default": {"available_models": ["fallback"], "status": "ready"},
-                "opsdroid": {"available_models": ["hello", "world"], "status": "ready"},
-            }
-        }
+
         with amock.patch("aiohttp.ClientSession.get") as patched_request:
+            patched_request.side_effect = (
+                aiohttp.client_exceptions.ClientConnectorError("key", amock.Mock())
+            )
+            self.assertEqual(await rasanlu._get_rasa_nlu_version({}), None)
+
+            patched_request.side_effect = None
+            result.content_type = "application/json"
+            result.json.return_value = {
+                "version": "1.0.0",
+                "minimum_compatible_version": "1.0.0",
+            }
+            patched_request.return_value = asyncio.Future()
+            patched_request.return_value.set_result(result)
+            self.assertEqual(
+                await rasanlu._get_rasa_nlu_version({}), result.json.return_value
+            )
+
+            patched_request.side_effect = None
+            result.status = 500
+            result.text.return_value = "Some error happened"
             patched_request.return_value = asyncio.Future()
             patched_request.return_value.set_result(result)
-            models = await rasanlu._get_existing_models({"project": "opsdroid"})
-            self.assertEqual(models, ["hello", "world"])
+            self.assertEqual(
+                await rasanlu._get_rasa_nlu_version({}), result.text.return_value
+            )
 
-    async def test__get_existing_models_exception(self):
+    async def test__check_rasanlu_compatibility(self):
+        with amock.patch.object(rasanlu, "_get_rasa_nlu_version") as mock_crc:
+            mock_crc.return_value = {
+                "version": "1.0.0",
+                "minimum_compatible_version": "1.0.0",
+            }
+            self.assertEqual(await rasanlu._check_rasanlu_compatibility({}), False)
 
-        with amock.patch("aiohttp.ClientSession.get") as patched_request:
+            mock_crc.return_value = {
+                "version": "2.6.2",
+                "minimum_compatible_version": "2.6.0",
+            }
+            self.assertEqual(await rasanlu._check_rasanlu_compatibility({}), True)
+
+            mock_crc.return_value = {
+                "version": "3.1.2",
+                "minimum_compatible_version": "3.0.0",
+            }
+            self.assertEqual(await rasanlu._check_rasanlu_compatibility({}), True)
+
+    async def test__load_model(self):
+        result = amock.Mock()
+        result.status = 204
+        result.text = amock.CoroutineMock()
+        result.json = amock.CoroutineMock()
+
+        with amock.patch("aiohttp.ClientSession.put") as patched_request:
+            patched_request.side_effect = None
+            result.json.return_value = {}
             patched_request.return_value = asyncio.Future()
-            patched_request.side_effect = ClientOSError()
-            models = await rasanlu._get_existing_models({"project": "opsdroid"})
-            self.assertRaises(ClientOSError)
-            self.assertEqual(models, [])
+            patched_request.return_value.set_result(result)
+            self.assertEqual(
+                await rasanlu._load_model({"model_filename": "model.tar.gz"}), {}
+            )
+            self.assertEqual(
+                await rasanlu._load_model(
+                    {"model_filename": "model.tar.gz", "token": "12345"}
+                ),
+                {},
+            )
+
+            result.status = 500
+            result.text.return_value = "some weird error"
+            self.assertEqual(
+                await rasanlu._load_model({"model_filename": "model.tar.gz"}),
+                result.text.return_value,
+            )
+
+            patched_request.side_effect = (
+                aiohttp.client_exceptions.ClientConnectorError("key", amock.Mock())
+            )
+            self.assertEqual(
+                await rasanlu._load_model({"model_filename": "model.tar.gz"}), None
+            )
 
-    async def test__get_existing_models_fails(self):
+    async def test__is_model_loaded(self):
         result = amock.Mock()
-        result.status = 404
+        result.status = 200
         result.json = amock.CoroutineMock()
-        result.json.return_value = {}
+
         with amock.patch("aiohttp.ClientSession.get") as patched_request:
+            result.json.return_value = {
+                "fingerprint": {
+                    "config": ["7625d69d93053ac8520a544d0852c626"],
+                    "domain": ["229b51e41876bbcbbbfbeddf79548d5a"],
+                    "messages": ["cf7eda7edcae128a75ee8c95d3bbd680"],
+                    "stories": ["b5facea681fd00bc7ecc6818c70d9639"],
+                    "trained_at": 1556527123.42784,
+                    "version": "2.6.2",
+                },
+                "model_file": "/app/models/model.tar.gz",
+                "num_active_training_jobs": 2,
+            }
+            patched_request.side_effect = None
             patched_request.return_value = asyncio.Future()
             patched_request.return_value.set_result(result)
-            models = await rasanlu._get_existing_models({})
-            self.assertEqual(models, [])
+            self.assertEqual(
+                await rasanlu._is_model_loaded({"model_filename": "model.tar.gz"}), True
+            )
+            self.assertEqual(
+                await rasanlu._is_model_loaded(
+                    {"model_filename": "model.tar.gz", "token": "12345"}
+                ),
+                True,
+            )
+            result.status = 500
+            self.assertEqual(
+                await rasanlu._is_model_loaded({"model_filename": "model.tar.gz"}),
+                False,
+            )
+            patched_request.side_effect = (
+                aiohttp.client_exceptions.ClientConnectorError("key", amock.Mock())
+            )
+            self.assertEqual(
+                await rasanlu._is_model_loaded({"model_filename": "model.tar.gz"}), None
+            )
 
     async def test_train_rasanlu_fails(self):
         result = amock.Mock()
@@ -498,31 +584,27 @@ async def test_train_rasanlu_fails(self):
         ) as patched_request, amock.patch.object(
             rasanlu, "_get_all_intents"
         ) as mock_gai, amock.patch.object(
-            rasanlu, "_init_model"
-        ) as mock_im, amock.patch.object(
             rasanlu, "_build_training_url"
         ) as mock_btu, amock.patch.object(
-            rasanlu, "_get_existing_models"
-        ) as mock_gem, amock.patch.object(
             rasanlu, "_get_intents_fingerprint"
-        ) as mock_gif:
+        ) as mock_gif, amock.patch.object(
+            rasanlu, "_check_rasanlu_compatibility"
+        ) as mock_crc, amock.patch.object(
+            rasanlu, "_load_model"
+        ) as mock_lmo, amock.patch.object(
+            rasanlu, "_is_model_loaded"
+        ) as mock_iml:
 
             mock_gai.return_value = None
             self.assertEqual(await rasanlu.train_rasanlu({}, {}), False)
 
-            mock_gai.return_value = "Hello World"
-            mock_gem.return_value = ["abc123"]
-            mock_gif.return_value = "abc123"
-            self.assertEqual(await rasanlu.train_rasanlu({}, {}), True)
-            self.assertTrue(mock_im.called)
-            self.assertFalse(mock_btu.called)
-
-            mock_gem.return_value = []
+            # _build_training_url
             mock_btu.return_value = "http://example.com"
-            patched_request.side_effect = (
-                aiohttp.client_exceptions.ClientConnectorError("key", amock.Mock())
-            )
+
+            # Test if no intents specified
             self.assertEqual(await rasanlu.train_rasanlu({}, {}), False)
+            self.assertFalse(mock_btu.called)
+            self.assertTrue(mock_gai.called)
 
             patched_request.side_effect = None
             patched_request.return_value = asyncio.Future()
@@ -535,49 +617,87 @@ async def test_train_rasanlu_fails(self):
             patched_request.return_value.set_result(result)
             self.assertEqual(await rasanlu.train_rasanlu({}, {}), False)
 
+            # Test Rasa client connection error
+            # _get_all_intents
+            mock_gai.return_value = "Hello World"
+            # _get_intents_fingerprint
+            mock_gif.return_value = "abc1234"
+            # _check_rasanlu_compatibility
+            mock_crc.return_value = True
+            patched_request.side_effect = (
+                aiohttp.client_exceptions.ClientConnectorError("key", amock.Mock())
+            )
+            self.assertEqual(await rasanlu.train_rasanlu({}, {}), False)
+            self.assertTrue(mock_btu.called)
+            self.assertFalse(mock_lmo.called)
+
+            # Test if the trained model is not loaded
+            # _load_model
+            mock_lmo.return_value = "{}"
+            # _is_model_loaded
+            mock_iml.return_value = False
+            result.content_type = "application/x-tar"
+            result.content_disposition.type = "attachment"
+            result.content_disposition.filename = "model.tar.gz"
+            result.json.return_value = "Tar file content..."
+            result.status = 200
+            patched_request.side_effect = None
+            self.assertEqual(await rasanlu.train_rasanlu({}, {}), False)
+            self.assertTrue(mock_lmo.called)
+            self.assertTrue(mock_iml.called)
+
     async def test_train_rasanlu_succeeded(self):
         result = amock.Mock()
         result.text = amock.CoroutineMock()
         result.json = amock.CoroutineMock()
         result.status = 200
-        result.json.return_value = {"info": "new model trained: abc1234"}
+        result.json.return_value = {}
 
         with amock.patch(
             "aiohttp.ClientSession.post"
         ) as patched_request, amock.patch.object(
             rasanlu, "_get_all_intents"
         ) as mock_gai, amock.patch.object(
-            rasanlu, "_init_model"
-        ), amock.patch.object(
             rasanlu, "_build_training_url"
         ) as mock_btu, amock.patch.object(
-            rasanlu, "_get_existing_models"
-        ) as mock_gem, amock.patch.object(
             rasanlu, "_get_intents_fingerprint"
-        ) as mock_gif:
-
-            mock_gem.return_value = ["abc123"]
+        ) as mock_gif, amock.patch.object(
+            rasanlu, "_check_rasanlu_compatibility"
+        ) as mock_crc, amock.patch.object(
+            rasanlu, "_load_model"
+        ) as mock_lmo, amock.patch.object(
+            rasanlu, "_is_model_loaded"
+        ) as mock_iml:
+
+            # _build_training_url
             mock_btu.return_value = "http://example.com"
+            # _get_all_intents
             mock_gai.return_value = "Hello World"
+            # _get_intents_fingerprint
             mock_gif.return_value = "abc1234"
-            result.content_type = "application/json"
+            # _check_rasanlu_compatibility
+            mock_crc.return_value = True
+            # _load_model
+            mock_lmo.return_value = "{}"
+            # _is_model_loaded
+            mock_iml.return_value = True
+            result.content_type = "application/x-tar"
+            result.content_disposition.type = "attachment"
+            result.content_disposition.filename = "model.tar.gz"
+            result.json.return_value = "Tar file content..."
 
             patched_request.side_effect = None
             patched_request.return_value = asyncio.Future()
             patched_request.return_value.set_result(result)
             self.assertEqual(await rasanlu.train_rasanlu({}, {}), True)
 
-            result.json.return_value = {}
             patched_request.side_effect = None
             patched_request.return_value = asyncio.Future()
             patched_request.return_value.set_result(result)
-            self.assertEqual(await rasanlu.train_rasanlu({}, {}), False)
-
-            result.content_type = "application/zip"
-            result.content_disposition.type = "attachment"
-            result.json.return_value = {"info": "new model trained: abc1234"}
+            self.assertEqual(await rasanlu.train_rasanlu({}, {}), True)
 
+            result.status = 500
             patched_request.side_effect = None
             patched_request.return_value = asyncio.Future()
             patched_request.return_value.set_result(result)
-            self.assertEqual(await rasanlu.train_rasanlu({}, {}), True)
+            self.assertEqual(await rasanlu.train_rasanlu({}, {}), False)