Repository: fedora-infra/bodhi

Issue:
Bodhi is logging stderr at error level in bodhi.server.util.cmd
Bodhi is [logging subprocess stderr output at the error level](https://github.com/fedora-infra/bodhi/blob/2.5.0/bodhi/server/util.py#L604-L605), which is not correct. Many programs use stderr for debug or info type output, and it doesn't necessarily indicate that there was an error.

Instead, Bodhi should check the exit code of the process that was called. If the exit code is 0, the stderr should be logged at info. If it is non-zero, then it is OK to log messages at error.

This is causing e-mails like this to get sent:

```
Message
-------
[2017-04-03 22:56:16][bodhi.server   ERROR]
python
From https://pagure.io/fedora-comps
   8cfc441..3ea2f69  master     -> origin/master



Process Details
---------------
- host:     bodhi-backend01.phx2.fedoraproject.org
- PID:      52590
- name:     fedmsg-hub
- command:  /usr/bin/python /usr/bin/fedmsg-hub
- msg_id:   

Callstack that lead to the logging statement
--------------------------------------------
python
  File "/usr/lib64/python2.7/threading.py", line 777 in __bootstrap
    self.__bootstrap_inner()
  File "/usr/lib64/python2.7/threading.py", line 804 in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 259 in run
    self.work()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 317 in work
    self.update_comps()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 641 in update_comps
    util.cmd(['git', 'pull'], comps_dir)
  File "/usr/lib/python2.7/site-packages/bodhi/server/util.py", line 581 in cmd
    log.error(err)
```


Discussion:

Resolving patch:
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -601,7 +601,10 @@ def cmd(cmd, cwd=None):
     if out:
         log.debug(out)
     if err:
-        log.error(err)
+        if p.returncode == 0:
+            log.debug(err)
+        else:
+            log.error(err)
     if p.returncode != 0:
         log.error('return code %s', p.returncode)
     return out, err, p.returncode


Test patch:
diff --git a/bodhi/tests/server/test_utils.py b/bodhi/tests/server/test_utils.py
--- a/bodhi/tests/server/test_utils.py
+++ b/bodhi/tests/server/test_utils.py
@@ -12,6 +12,9 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
+import mock
+import unittest
+import subprocess
 from bodhi.server.buildsys import setup_buildsystem, teardown_buildsystem
 from bodhi.server.config import config
 from bodhi.server.util import (get_critpath_pkgs, get_nvr, markup,
@@ -70,3 +73,59 @@ def test_sorted_builds(self):
         b1, b2 = sorted_builds([new, old])
         assert b1 == new, b1
         assert b2 == old, b2
+
+
+class TestCMDFunctions(unittest.TestCase):
+    @mock.patch('bodhi.server.log.debug')
+    @mock.patch('bodhi.server.log.error')
+    @mock.patch('subprocess.Popen')
+    def test_err_nonzero_return_code(self, mock_popen, mock_error, mock_debug):
+        """
+        Ensures proper behavior when there is err output and the exit code isn't 0.
+        See https://github.com/fedora-infra/bodhi/issues/1412
+        """
+        mock_popen.return_value = mock.Mock()
+        mock_popen_obj = mock_popen.return_value
+        mock_popen_obj.communicate.return_value = ('output', 'error')
+        mock_popen_obj.returncode = 1
+        cmd('/bin/echo', '"home/imgs/catpix"')
+        mock_popen.assert_called_once_with(['/bin/echo'], cwd='"home/imgs/catpix"',
+                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        mock_error.assert_any_call('error')
+        mock_debug.assert_called_once_with('output')
+
+    @mock.patch('bodhi.server.log.debug')
+    @mock.patch('bodhi.server.log.error')
+    @mock.patch('subprocess.Popen')
+    def test_no_err_zero_return_code(self, mock_popen, mock_error, mock_debug):
+        """
+        Ensures proper behavior when there is no err output and the exit code is 0.
+        See https://github.com/fedora-infra/bodhi/issues/1412
+        """
+        mock_popen.return_value = mock.Mock()
+        mock_popen_obj = mock_popen.return_value
+        mock_popen_obj.communicate.return_value = ('output', None)
+        mock_popen_obj.returncode = 0
+        cmd('/bin/echo', '"home/imgs/catpix"')
+        mock_popen.assert_called_once_with(['/bin/echo'], cwd='"home/imgs/catpix"',
+                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        mock_error.assert_not_called()
+        mock_debug.assert_called_once_with('output')
+
+    @mock.patch('bodhi.server.log.debug')
+    @mock.patch('bodhi.server.log.error')
+    @mock.patch('subprocess.Popen')
+    def test_err_zero_return_code(self, mock_popen, mock_error, mock_debug):
+        """
+        Ensures proper behavior when there is err output, but the exit code is 0.
+        See https://github.com/fedora-infra/bodhi/issues/1412
+        """
+        mock_popen.return_value = mock.Mock()
+        mock_popen_obj = mock_popen.return_value
+        mock_popen_obj.communicate.return_value = ('output', 'error')
+        mock_popen_obj.returncode = 0
+        cmd('/bin/echo', '"home/imgs/catpix"')
+        mock_popen.assert_called_once_with(['/bin/echo'], cwd='"home/imgs/catpix"',
+                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        mock_error.assert_not_called()
+        mock_debug.assert_called_with('error')