Repository: StackStorm/st2

Issue:
remote-shell-cmd does not close ssh connections
Hi,

If I run a `core.remote` action an ssh connection opens and it does not close after the action finishes successfully. Running the same action again and again opens new connections until the remote server refuses to accept more.

A simple bash one-liner to reproduce this:

```
for i in {1..1000}; do st2 run core.remote hosts=127.0.0.1 whoami; done
```

And the result:

```
$ netstat -a | grep "tcp \+0 \+0 \+localhost:ssh" | wc -l
1000
```

I tried it on a fresh Ubuntu 16.04 machine with a fresh install of StackStorm 2.2.0.

BR,
k.


Discussion:
Thanks for reporting this.

Can you please post more details - which version of StackStorm you are running and which version of paramiko library is used (`/opt/stackstorm/bin/st2/pip show paramiko`).

We haven't noticed anything like this in the past, but we recently upgraded to latest version of paramiko so it's possible some default behavior is changed or maybe it's related to some setting in the user / global SSH config file .

To rule out some SSH config related keep alive setting keeping the connections open, can you please also run something like this:

```bash
for i in {1..1000}; do st2 run core.remote hosts=127.0.0.1 whoami; sleep 20; done
```


I was able to reproduce the issue - I'm looking into a fix.

Thanks again for reporting this.

Resolving patch:
diff --git a/st2actions/st2actions/container/base.py b/st2actions/st2actions/container/base.py
--- a/st2actions/st2actions/container/base.py
+++ b/st2actions/st2actions/container/base.py
@@ -119,22 +119,29 @@ def _do_run(self, runner, runnertype_db, action_db, liveaction_db):
             extra = {'result': result, 'status': status}
             LOG.debug('Action "%s" completed.' % (action_db.name), extra=extra)
 
-            # Always clean-up the auth_token
             try:
                 LOG.debug('Setting status: %s for liveaction: %s', status, liveaction_db.id)
                 updated_liveaction_db = self._update_live_action_db(liveaction_db.id, status,
                                                                     result, context)
-            except:
-                error = 'Cannot update LiveAction object for id: %s, status: %s, result: %s.' % (
+            except Exception as e:
+                msg = 'Cannot update LiveAction object for id: %s, status: %s, result: %s.' % (
                     liveaction_db.id, status, result)
-                LOG.exception(error)
-                raise
+                LOG.exception(msg)
+                raise e
 
-            executions.update_execution(updated_liveaction_db)
-            extra = {'liveaction_db': updated_liveaction_db}
-            LOG.debug('Updated liveaction after run', extra=extra)
+            try:
+                executions.update_execution(updated_liveaction_db)
+                extra = {'liveaction_db': updated_liveaction_db}
+                LOG.debug('Updated liveaction after run', extra=extra)
+            except Exception as e:
+                msg = 'Cannot update ActionExecution object for id: %s, status: %s, result: %s.' % (
+                    updated_liveaction_db.id, status, result)
+                LOG.exception(msg)
+                raise e
 
             # Always clean-up the auth_token
+            # Note: self._clean_up_auth_token should never throw to ensure post_run is always
+            # called.
             self._clean_up_auth_token(runner=runner, status=status)
 
         LOG.debug('Performing post_run for runner: %s', runner.runner_id)
@@ -179,6 +186,9 @@ def _do_cancel(self, runner, runnertype_db, action_db, liveaction_db):
     def _clean_up_auth_token(self, runner, status):
         """
         Clean up the temporary auth token for the current action.
+
+        Note: This method should never throw since it's called inside finally block which assumes
+        it doesn't throw.
         """
         # Deletion of the runner generated auth token is delayed until the token expires.
         # Async actions such as Mistral workflows uses the auth token to launch other
diff --git a/st2common/st2common/runners/paramiko_ssh_runner.py b/st2common/st2common/runners/paramiko_ssh_runner.py
--- a/st2common/st2common/runners/paramiko_ssh_runner.py
+++ b/st2common/st2common/runners/paramiko_ssh_runner.py
@@ -144,6 +144,13 @@ def pre_run(self):
 
         self._parallel_ssh_client = ParallelSSHClient(**client_kwargs)
 
+    def post_run(self, status, result):
+        super(BaseParallelSSHRunner, self).post_run(status=status, result=result)
+
+        # Ensure we close the connection when the action execution finishes
+        if self._parallel_ssh_client:
+            self._parallel_ssh_client.close()
+
     def _is_private_key_material(self, private_key):
         return private_key and REMOTE_RUNNER_PRIVATE_KEY_HEADER in private_key.lower()
 
diff --git a/st2tests/st2tests/mocks/runner.py b/st2tests/st2tests/mocks/runner.py
--- a/st2tests/st2tests/mocks/runner.py
+++ b/st2tests/st2tests/mocks/runner.py
@@ -64,4 +64,5 @@ def run(self, action_params):
         return (status, json.dumps(result), context)
 
     def post_run(self, status, result):
+        super(MockActionRunner, self).post_run(status=status, result=result)
         self.post_run_called = True


Test patch:
diff --git a/contrib/runners/local_runner/tests/unit/__init__.py b/contrib/runners/local_runner/tests/unit/__init__.py
new file mode 100644
diff --git a/st2actions/tests/unit/test_paramiko_ssh_runner.py b/st2actions/tests/unit/test_paramiko_ssh_runner.py
--- a/st2actions/tests/unit/test_paramiko_ssh_runner.py
+++ b/st2actions/tests/unit/test_paramiko_ssh_runner.py
@@ -190,3 +190,34 @@ def test_pre_run(self, mock_client):
             'connect': True
         }
         mock_client.assert_called_with(**expected_kwargs)
+
+    @mock.patch('st2common.runners.paramiko_ssh_runner.ParallelSSHClient')
+    def test_post_run(self, mock_client):
+        # Verify that the SSH connections are closed on post_run
+        runner = Runner('id')
+        runner.context = {}
+        runner_parameters = {
+            RUNNER_HOSTS: 'localhost',
+            RUNNER_USERNAME: 'someuser1',
+            RUNNER_PASSWORD: 'somepassword'
+        }
+        runner.runner_parameters = runner_parameters
+        runner.pre_run()
+
+        expected_kwargs = {
+            'hosts': ['localhost'],
+            'user': 'someuser1',
+            'password': 'somepassword',
+            'port': None,
+            'concurrency': 1,
+            'bastion_host': None,
+            'raise_on_any_error': False,
+            'connect': True
+        }
+        mock_client.assert_called_with(**expected_kwargs)
+        self.assertEqual(runner._parallel_ssh_client.close.call_count, 0)
+
+        runner.post_run(result=None, status=None)
+
+        # Verify connections are closed
+        self.assertEqual(runner._parallel_ssh_client.close.call_count, 1)
diff --git a/contrib/runners/local_runner/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py
similarity index 74%
rename from contrib/runners/local_runner/tests/unit/test_runner_container.py
rename to st2actions/tests/unit/test_runner_container.py
--- a/contrib/runners/local_runner/tests/unit/test_runner_container.py
+++ b/st2actions/tests/unit/test_runner_container.py
@@ -93,6 +93,84 @@ def test_pre_run_runner_is_disabled(self):
         expected_msg = 'Runner "test-runner-1" has been disabled by the administrator'
         self.assertRaisesRegexp(ValueError, expected_msg, runner.pre_run)
 
+    def test_post_run_is_always_called_after_run(self):
+        # 1. post_run should be called on success, failure, etc.
+        runner_container = get_runner_container()
+        params = {
+            'actionstr': 'bar',
+            'mock_status': action_constants.LIVEACTION_STATUS_SUCCEEDED
+        }
+        liveaction_db = self._get_failingaction_exec_db_model(params)
+        liveaction_db = LiveAction.add_or_update(liveaction_db)
+        executions.create_execution_object(liveaction_db)
+
+        global global_runner
+        global_runner = None
+        original_get_runner = runner_container._get_runner
+
+        def mock_get_runner(*args, **kwargs):
+            global global_runner
+            runner = original_get_runner(*args, **kwargs)
+            global_runner = runner
+            return runner
+        runner_container._get_runner = mock_get_runner
+
+        # Note: We can't assert here that post_run hasn't been called yet because runner instance
+        # is only instantiated later inside dispatch method
+        runner_container.dispatch(liveaction_db)
+        self.assertTrue(global_runner.post_run_called)
+
+        # 2. Verify post_run is called if run() throws
+        runner_container = get_runner_container()
+        params = {
+            'actionstr': 'bar',
+            'raise': True
+        }
+        liveaction_db = self._get_failingaction_exec_db_model(params)
+        liveaction_db = LiveAction.add_or_update(liveaction_db)
+        executions.create_execution_object(liveaction_db)
+
+        global_runner = None
+        original_get_runner = runner_container._get_runner
+
+        def mock_get_runner(*args, **kwargs):
+            global global_runner
+            runner = original_get_runner(*args, **kwargs)
+            global_runner = runner
+            return runner
+        runner_container._get_runner = mock_get_runner
+
+        # Note: We can't assert here that post_run hasn't been called yet because runner instance
+        # is only instantiated later inside dispatch method
+        runner_container.dispatch(liveaction_db)
+        self.assertTrue(global_runner.post_run_called)
+
+        # 2. Verify post_run is also called if _delete_auth_token throws
+        runner_container = get_runner_container()
+        runner_container._delete_auth_token = mock.Mock(side_effect=ValueError('throw'))
+        params = {
+            'actionstr': 'bar',
+            'mock_status': action_constants.LIVEACTION_STATUS_SUCCEEDED
+        }
+        liveaction_db = self._get_failingaction_exec_db_model(params)
+        liveaction_db = LiveAction.add_or_update(liveaction_db)
+        executions.create_execution_object(liveaction_db)
+
+        global_runner = None
+        original_get_runner = runner_container._get_runner
+
+        def mock_get_runner(*args, **kwargs):
+            global global_runner
+            runner = original_get_runner(*args, **kwargs)
+            global_runner = runner
+            return runner
+        runner_container._get_runner = mock_get_runner
+
+        # Note: We can't assert here that post_run hasn't been called yet because runner instance
+        # is only instantiated later inside dispatch method
+        runner_container.dispatch(liveaction_db)
+        self.assertTrue(global_runner.post_run_called)
+
     def test_get_runner_module_fail(self):
         runnertype_db = RunnerTypeDB(name='dummy', runner_module='absent.module')
         runner = None