Repository: spyder-ide/spyder

Issue:
Tests are fragile
## Problem Description

Tests are not robust ; in particular, the following lines induce failures of Azur pipeline Windows-3.6-slow:
https://github.com/spyder-ide/spyder/blob/1142a7643c754d2edb0154c18fbdb6baa0af7fc6/spyder/plugins/editor/widgets/tests/conftest.py#L236-L237

Even if it's possible to live with this problem since it doesn't affect users, it's very frustrating for contributors to spend way more time trying to fix "random" behavior of tests than actually working on the proposed fix. Notice that this failure can't be tested locally.

### What steps reproduce the problem?

1. copy content of /spyder/plugins/editor/widgets/tests/test_completion.py into /spyder/plugins/editor/widgets/tests/test_introspection.py
2. Run azur pipeline, Windows 3.6-slow will (maybe) fail

### What is the expected output?

More robust tests.

### What was tried to fix the issue?
I tried changing the timeout to 60000 (in #11916) , without effect.


Discussion:

Resolving patch:
diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -15,6 +15,13 @@
 import os.path as osp
 import shutil
 import sys
+import warnings
+
+
+if sys.version_info[0] == 2:
+    # Hide warnings on py2 due to qtawesome as it makes the results unreadable
+    warnings.filterwarnings("ignore")
+
 
 # To activate/deactivate certain things for pytest's only
 # NOTE: Please leave this before any other import here!!
diff --git a/spyder/plugins/ipythonconsole/plugin.py b/spyder/plugins/ipythonconsole/plugin.py
--- a/spyder/plugins/ipythonconsole/plugin.py
+++ b/spyder/plugins/ipythonconsole/plugin.py
@@ -45,7 +45,7 @@
 from spyder.plugins.ipythonconsole.utils.style import create_qss_style
 from spyder.plugins.ipythonconsole.widgets import (ClientWidget,
                                                    KernelConnectionDialog)
-from spyder.py3compat import is_string, to_text_string, PY38_OR_MORE
+from spyder.py3compat import is_string, to_text_string, PY2, PY38_OR_MORE
 from spyder.utils import encoding
 from spyder.utils import icon_manager as ima
 from spyder.utils import programs, sourcecode
@@ -844,10 +844,17 @@ def interpreter_versions(self):
             import subprocess
             versions = {}
             pyexec = CONF.get('main_interpreter', 'executable')
-            py_cmd = '%s -c "import sys; print(sys.version)"' % pyexec
-            ipy_cmd = ('%s -c "import IPython.core.release as r; print(r.version)"'
-                       % pyexec)
+            py_cmd = u'%s -c "import sys; print(sys.version)"' % pyexec
+            ipy_cmd = (
+                u'%s -c "import IPython.core.release as r; print(r.version)"'
+                % pyexec
+            )
             for cmd in [py_cmd, ipy_cmd]:
+                if PY2:
+                    # We need to encode as run_shell_command will treat the
+                    # string as str
+                    cmd = cmd.encode('utf-8')
+
                 try:
                     proc = programs.run_shell_command(cmd)
                     output, _err = proc.communicate()


Test patch:
diff --git a/continuous_integration/circle/modules_test.sh b/.github/scripts/modules_test.sh
similarity index 96%
rename from continuous_integration/circle/modules_test.sh
rename to .github/scripts/modules_test.sh
--- a/continuous_integration/circle/modules_test.sh
+++ b/.github/scripts/modules_test.sh
@@ -2,9 +2,6 @@
 
 set -ex
 
-source $HOME/miniconda/etc/profile.d/conda.sh
-conda activate test
-
 export TEST_CI_WIDGETS=True
 export PYTHONPATH=.:./external-deps/spyder-kernels
 
@@ -70,6 +67,9 @@ for f in spyder/*/*/*.py; do
     if [[ $f == spyder/plugins/completion/*.py ]]; then
         continue
     fi
+    if [[ $f == spyder/plugins/onlinehelp/widgets.py ]]; then
+        continue
+    fi
     if [[ $f == spyder/plugins/*/__init__.py ]]; then
         continue
     fi
diff --git a/spyder/app/tests/test_cli_options.py b/spyder/app/tests/test_cli_options.py
--- a/spyder/app/tests/test_cli_options.py
+++ b/spyder/app/tests/test_cli_options.py
@@ -11,6 +11,7 @@
 from spyder.app import cli_options
 
 
+@pytest.mark.first
 def test_get_options():
     getopt = cli_options.get_options
 
diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py
--- a/spyder/app/tests/test_mainwindow.py
+++ b/spyder/app/tests/test_mainwindow.py
@@ -237,9 +237,10 @@ def remove_test_dir():
 # =============================================================================
 @flaky(max_runs=3)
 @pytest.mark.slow
+@pytest.mark.first
 @pytest.mark.single_instance
 @pytest.mark.skipif((os.environ.get('CI', None) is None or (PY2
-                     and sys.platform == 'darwin')),
+                     and not sys.platform.startswith('linux'))),
                     reason="It's not meant to be run outside of CIs")
 def test_single_instance_and_edit_magic(main_window, qtbot, tmpdir):
     """Test single instance mode and %edit magic."""
@@ -303,6 +304,8 @@ def test_lock_action(main_window):
 
 
 @pytest.mark.slow
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason="Fails on win and py2")
 def test_default_plugin_actions(main_window, qtbot):
     """Test the effect of dock, undock, close and toggle view actions."""
     # Use a particular plugin
@@ -348,7 +351,10 @@ def test_opengl_implementation(main_window, qtbot):
 
 @pytest.mark.slow
 @flaky(max_runs=3)
-@pytest.mark.skipif(np.__version__ < '1.14.0', reason="This only happens in Numpy 1.14+")
+@pytest.mark.skipif(
+    np.__version__ < '1.14.0' or (os.name == 'nt' and PY2),
+    reason="This only happens in Numpy 1.14+"
+)
 @pytest.mark.parametrize('main_window', [{'spy_config': ('variable_explorer', 'minmax', True)}], indirect=True)
 def test_filter_numpy_warning(main_window, qtbot):
     """
@@ -690,8 +696,7 @@ def test_move_to_first_breakpoint(main_window, qtbot, debugcell):
 
 @pytest.mark.slow
 @flaky(max_runs=3)
-@pytest.mark.skipif(os.environ.get('CI', None) is None or sys.platform == 'darwin',
-                    reason="It's not meant to be run locally and fails in macOS")
+@pytest.mark.skipif(os.name == 'nt', reason='Fails on windows!')
 def test_runconfig_workdir(main_window, qtbot, tmpdir):
     """Test runconfig workdir options."""
     CONF.set('run', 'configurations', [])
@@ -746,7 +751,7 @@ def test_runconfig_workdir(main_window, qtbot, tmpdir):
 
 @pytest.mark.slow
 @flaky(max_runs=3)
-@pytest.mark.skipif((os.name == 'nt' and PY2) or sys.platform == 'darwin',
+@pytest.mark.skipif(os.name == 'nt' or sys.platform == 'darwin',
                     reason="It's failing there")
 def test_dedicated_consoles(main_window, qtbot):
     """Test running code in dedicated consoles."""
diff --git a/spyder/plugins/completion/kite/utils/tests/test_install.py b/spyder/plugins/completion/kite/utils/tests/test_install.py
--- a/spyder/plugins/completion/kite/utils/tests/test_install.py
+++ b/spyder/plugins/completion/kite/utils/tests/test_install.py
@@ -28,10 +28,7 @@
 
 @pytest.mark.slow
 @pytest.mark.first
-@pytest.mark.skipif((not sys.platform.startswith('linux')
-                     or os.environ.get('CI', None) is None),
-                    reason=("Only works reliably on Linux and "
-                            "it's not meant to be run outside of CIs"))
+@pytest.mark.skipif(bool(os.environ.get('CI', None)), reason='Fails on CI!')
 def test_kite_install(qtbot):
     """Test the correct execution of the installation process of kite."""
     install_manager = KiteInstallationThread(None)
diff --git a/spyder/plugins/editor/widgets/tests/conftest.py b/spyder/plugins/editor/widgets/tests/conftest.py
--- a/spyder/plugins/editor/widgets/tests/conftest.py
+++ b/spyder/plugins/editor/widgets/tests/conftest.py
@@ -146,14 +146,7 @@ def teardown():
     return editor, kite
 
 
-# Windows tests fail if using module scope
-if os.name == 'nt':
-    LSP_PLUGIN_SCOPE = 'module'
-else:
-    LSP_PLUGIN_SCOPE = 'function'
-
-
-@pytest.fixture(scope=LSP_PLUGIN_SCOPE)
+@pytest.fixture(scope='function')
 def lsp_plugin(qtbot_module, request):
     # Activate pycodestyle and pydocstyle
     CONF.set('lsp-server', 'pycodestyle', True)
diff --git a/spyder/plugins/editor/widgets/tests/test_codeeditor.py b/spyder/plugins/editor/widgets/tests/test_codeeditor.py
--- a/spyder/plugins/editor/widgets/tests/test_codeeditor.py
+++ b/spyder/plugins/editor/widgets/tests/test_codeeditor.py
@@ -243,9 +243,9 @@ def test_comment(editorbot):
     cursor.setPosition(8)
     cursor.setPosition(11, QTextCursor.KeepAnchor)
     widget.setTextCursor(cursor)
-    qtbot.keyPress(widget, "1", modifier=Qt.ControlModifier)
+    widget.toggle_comment()
     assert widget.toPlainText() == "# import numpy"
-    qtbot.keyPress(widget, "1", modifier=Qt.ControlModifier)
+    widget.toggle_comment()
     assert widget.toPlainText() == "import numpy"
 
 
diff --git a/spyder/plugins/editor/widgets/tests/test_goto.py b/spyder/plugins/editor/widgets/tests/test_goto.py
--- a/spyder/plugins/editor/widgets/tests/test_goto.py
+++ b/spyder/plugins/editor/widgets/tests/test_goto.py
@@ -29,7 +29,7 @@
 TEST_FILE_REL = 'conftest.py'
 
 
-@pytest.mark.skipif(os.name == 'nt', reason="It fails on Windows")
+@pytest.mark.skipif(bool(os.environ.get('CI', None)), reason='Fails on CI!')
 @pytest.mark.parametrize('params', [
             # Parameter, expected output 1, full file path, expected output 2
             # ----------------------------------------------------------------
diff --git a/spyder/plugins/editor/widgets/tests/test_hints_and_calltips.py b/spyder/plugins/editor/widgets/tests/test_hints_and_calltips.py
--- a/spyder/plugins/editor/widgets/tests/test_hints_and_calltips.py
+++ b/spyder/plugins/editor/widgets/tests/test_hints_and_calltips.py
@@ -63,6 +63,10 @@ def test_hide_calltip(lsp_codeeditor, qtbot):
 
 @pytest.mark.slow
 @pytest.mark.second
+@pytest.mark.skipif(
+    os.name == 'nt' and PY2,
+    reason='Fails on Win',
+)
 @pytest.mark.parametrize('params', [
             # Parameter, Expected Output
             ('dict', 'dict'),
@@ -126,8 +130,7 @@ def test_get_calltips(qtbot, lsp_codeeditor, params):
 
 @pytest.mark.slow
 @pytest.mark.second
-@pytest.mark.skipif(sys.platform != 'darwin' and bool(os.environ.get('CI')),
-                    reason="Fails on Windows and Linux on CI")
+@pytest.mark.skipif(sys.platform == 'darwin', reason='Fails on Mac')
 @pytest.mark.parametrize('params', [
             # Parameter, Expected Output
             ('"".format', '-> str'),
@@ -169,8 +172,7 @@ def test_get_hints(qtbot, lsp_codeeditor, params, capsys):
 
 @pytest.mark.slow
 @pytest.mark.second
-@pytest.mark.skipif(sys.platform != 'darwin',
-                    reason="Fails on Windows and Linux")
+@pytest.mark.skipif(sys.platform == 'darwin', reason='Fails on Mac')
 def test_get_hints_not_triggered(qtbot, lsp_codeeditor):
     """Test that the editor is not returning hover hints for empty docs."""
     code_editor, _ = lsp_codeeditor
diff --git a/spyder/plugins/editor/widgets/tests/test_introspection.py b/spyder/plugins/editor/widgets/tests/test_introspection.py
--- a/spyder/plugins/editor/widgets/tests/test_introspection.py
+++ b/spyder/plugins/editor/widgets/tests/test_introspection.py
@@ -93,9 +93,7 @@ def test_space_completion(lsp_codeeditor, qtbot):
 @pytest.mark.slow
 @pytest.mark.first
 @flaky(max_runs=5)
-@pytest.mark.skipif(
-    os.environ.get('CI') and (PY2 or os.name != 'nt'),
-    reason='Fails consistently on CI with Linux/Mac or Python 2')
+@pytest.mark.skipif(bool(os.environ.get('CI', None)), reason='Fails on CI!')
 def test_hide_widget_completion(lsp_codeeditor, qtbot):
     """Validate hiding completion widget after a delimeter or operator."""
     code_editor, _ = lsp_codeeditor
diff --git a/spyder/plugins/explorer/widgets/tests/test_explorer.py b/spyder/plugins/explorer/widgets/tests/test_explorer.py
--- a/spyder/plugins/explorer/widgets/tests/test_explorer.py
+++ b/spyder/plugins/explorer/widgets/tests/test_explorer.py
@@ -24,6 +24,7 @@
                                                       ProjectExplorerTest)
 from spyder.plugins.projects.widgets.explorer import (
     ProjectExplorerTest as ProjectExplorerTest2)
+from spyder.py3compat import PY2
 
 
 @pytest.fixture
@@ -227,6 +228,7 @@ def run_test_helper(single_click, initial_index):
     run_test_helper(single_click=False, initial_index=initial_index)
 
 
+@pytest.mark.first
 def test_get_common_file_associations(qtbot, file_explorer_associations):
     widget = file_explorer_associations.explorer.treewidget
     associations = widget.get_common_file_associations(
@@ -244,6 +246,7 @@ def test_get_common_file_associations(qtbot, file_explorer_associations):
     assert associations[0][-1] == '/some/fake/some_app_1' + ext
 
 
+@pytest.mark.first
 def test_get_file_associations(qtbot, file_explorer_associations):
     widget = file_explorer_associations.explorer.treewidget
     associations = widget.get_file_associations('/some/path/file.txt')
@@ -256,6 +259,8 @@ def test_get_file_associations(qtbot, file_explorer_associations):
     assert associations[0][-1] == '/some/fake/some_app_1' + ext
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on win and py2!')
 def test_create_file_manage_actions(qtbot, file_explorer_associations,
                                     tmp_path):
     widget = widget = file_explorer_associations.explorer.treewidget
@@ -285,6 +290,8 @@ def test_create_file_manage_actions(qtbot, file_explorer_associations,
     assert not action_texts
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on win and py2!')
 def test_clicked(qtbot, file_explorer_associations, tmp_path):
     widget = file_explorer_associations.explorer.treewidget
     some_dir = tmp_path / 'some_dir'
@@ -317,6 +324,8 @@ def interact_2():
     qtbot.keyClick(widget, Qt.Key_Return)
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on win and py2!')
 def test_check_launch_error_codes(qtbot, file_explorer_associations):
     widget = file_explorer_associations.explorer.treewidget
 
@@ -347,6 +356,8 @@ def interact_2():
     assert not res
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on win and py2!')
 def test_open_association(qtbot, file_explorer_associations, tmp_path):
     widget = file_explorer_associations.explorer.treewidget
     some_dir = tmp_path / 'some_dir'
diff --git a/spyder/plugins/explorer/widgets/tests/test_fileassociations.py b/spyder/plugins/explorer/widgets/tests/test_fileassociations.py
--- a/spyder/plugins/explorer/widgets/tests/test_fileassociations.py
+++ b/spyder/plugins/explorer/widgets/tests/test_fileassociations.py
@@ -21,8 +21,11 @@
 # Local imports
 from spyder.plugins.explorer.widgets.fileassociations import (
     ApplicationsDialog, FileAssociationsWidget, InputTextDialog)
+from spyder.py3compat import PY2
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on win and py2!')
 def test_input_text_dialog(qtbot):
     widget = InputTextDialog()
     qtbot.addWidget(widget)
@@ -53,6 +56,8 @@ def test_input_text_dialog(qtbot):
     widget.validate()
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on win and py2!')
 def test_apps_dialog(qtbot, tmp_path):
     widget = ApplicationsDialog()
     qtbot.addWidget(widget)
@@ -140,6 +145,8 @@ def create_timer(func, interval=500):
     return timer
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on win and py2!')
 def test_file_assoc_widget(file_assoc_widget):
     qtbot, widget = file_assoc_widget
 
diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
--- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
+++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py
@@ -1580,6 +1580,7 @@ def test_calltip(ipyconsole, qtbot):
 
 
 @flaky(max_runs=3)
+@pytest.mark.first
 @pytest.mark.test_environment_interpreter
 def test_conda_env_activation(ipyconsole, qtbot):
     """
diff --git a/spyder/plugins/plots/widgets/tests/test_plots_widgets.py b/spyder/plugins/plots/widgets/tests/test_plots_widgets.py
--- a/spyder/plugins/plots/widgets/tests/test_plots_widgets.py
+++ b/spyder/plugins/plots/widgets/tests/test_plots_widgets.py
@@ -98,6 +98,7 @@ def png_to_qimage(png):
 # =============================================================================
 # ---- Tests
 # =============================================================================
+@pytest.mark.first
 @pytest.mark.parametrize("fmt, fext",
                          [('image/png', '.png'), ('image/svg+xml', '.svg')])
 def test_handle_new_figures(figbrowser, tmpdir, fmt, fext):
diff --git a/spyder/plugins/variableexplorer/widgets/tests/test_arrayeditor.py b/spyder/plugins/variableexplorer/widgets/tests/test_arrayeditor.py
--- a/spyder/plugins/variableexplorer/widgets/tests/test_arrayeditor.py
+++ b/spyder/plugins/variableexplorer/widgets/tests/test_arrayeditor.py
@@ -239,6 +239,7 @@ def test_arrayeditor_edit_2d_array(qtbot):
     assert np.sum(diff_arr != dlg.get_value()) == 2
 
 
+@pytest.mark.slow
 def test_arrayeditor_edit_complex_array(qtbot):
     """See: spyder-ide/spyder#7848"""
     cnum = -1+0.5j
diff --git a/spyder/plugins/variableexplorer/widgets/tests/test_collectioneditor.py b/spyder/plugins/variableexplorer/widgets/tests/test_collectioneditor.py
--- a/spyder/plugins/variableexplorer/widgets/tests/test_collectioneditor.py
+++ b/spyder/plugins/variableexplorer/widgets/tests/test_collectioneditor.py
@@ -37,6 +37,8 @@
     NamespacesBrowserFinder)
 from spyder.plugins.variableexplorer.widgets.tests.test_dataframeeditor import \
     generate_pandas_indexes
+from spyder.py3compat import PY2
+
 
 # =============================================================================
 # Constants
@@ -311,6 +313,7 @@ def test_shows_dataframeeditor_when_editing_index(qtbot, monkeypatch):
         mockDataFrameEditor_instance.show.assert_called_once_with()
 
 
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on Win and py2')
 def test_sort_collectionsmodel():
     var_list1 = [0, 1, 2]
     var_list2 = [3, 4, 5, 6]
diff --git a/spyder/utils/tests/test_encoding.py b/spyder/utils/tests/test_encoding.py
--- a/spyder/utils/tests/test_encoding.py
+++ b/spyder/utils/tests/test_encoding.py
@@ -23,6 +23,8 @@
                                              os.path.dirname(__file__)))
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.name == 'nt' and PY2, reason='Fails on Win!')
 def test_symlinks(tmpdir):
     """
     Check that modifing symlinks files changes source file and keeps symlinks.
diff --git a/spyder/utils/tests/test_environ.py b/spyder/utils/tests/test_environ.py
--- a/spyder/utils/tests/test_environ.py
+++ b/spyder/utils/tests/test_environ.py
@@ -45,7 +45,10 @@ def test_environ(environ_dialog, qtbot):
     assert environ_dialog
 
 
-@pytest.mark.skipif(PY3, reason=("This tests only applies to Python 2."))
+@pytest.mark.skipif(
+    PY3 or os.name == 'nt',
+    reason=("This tests only applies to Python 2."),
+)
 def test_clean_env():
     env = {
         'foo': '/foo/bar/測試',
diff --git a/spyder/utils/tests/test_programs.py b/spyder/utils/tests/test_programs.py
--- a/spyder/utils/tests/test_programs.py
+++ b/spyder/utils/tests/test_programs.py
@@ -24,19 +24,19 @@
                                    run_python_script_in_terminal, shell_split)
 
 if os.name == 'nt':
-    python_dir = os.environ['PYTHON'] if os.environ.get('CI', None) else ''
+    python_dir = 'C:\\Miniconda\\'
     VALID_INTERPRETER = os.path.join(python_dir, 'python.exe')
     VALID_W_INTERPRETER = os.path.join(python_dir, 'pythonw.exe')
     INVALID_INTERPRETER = os.path.join(python_dir, 'Scripts', 'ipython.exe')
 else:
     if sys.platform.startswith('linux'):
-        home_dir = os.environ['HOME']
+        home_dir = '/usr/share/miniconda/'
     else:
-        # Parent Miniconda dir in macOS Azure VMs
-        home_dir = os.path.join('/usr', 'local')
-    VALID_INTERPRETER = os.path.join(home_dir, 'miniconda', 'bin', 'python')
-    VALID_W_INTERPRETER = os.path.join(home_dir, 'miniconda', 'bin', 'pythonw')
-    INVALID_INTERPRETER = os.path.join(home_dir, 'miniconda', 'bin', 'ipython')
+        home_dir = '/usr/local/miniconda/'
+
+    VALID_INTERPRETER = os.path.join(home_dir, 'bin', 'python')
+    VALID_W_INTERPRETER = os.path.join(home_dir, 'bin', 'pythonw')
+    INVALID_INTERPRETER = os.path.join(home_dir, 'bin', 'ipython')
 
 
 # =============================================================================
@@ -78,9 +78,7 @@ def test_is_valid_w_interpreter():
 
 
 @flaky(max_runs=3)
-@pytest.mark.skipif(
-    os.environ.get('CI', None) is None,
-    reason='fails sometimes locally')
+@pytest.mark.skipif(bool(os.environ.get('CI', None)), reason='Only on CI!')
 def test_run_python_script_in_terminal(scriptpath, qtbot):
     """
     Test running a Python script in an external terminal when specifying
@@ -98,9 +96,11 @@ def test_run_python_script_in_terminal(scriptpath, qtbot):
 
 
 @flaky(max_runs=3)
+@pytest.mark.first
 @pytest.mark.skipif(
-    os.environ.get('CI', None) is None,
-    reason='fails sometimes locally')
+    os.environ.get('CI', None) is None or os.name == 'nt',
+    reason='Only on CI and not on windows!',
+)
 def test_run_python_script_in_terminal_blank_wdir(scriptpath_with_blanks,
                                                   qtbot):
     """
@@ -120,9 +120,11 @@ def test_run_python_script_in_terminal_blank_wdir(scriptpath_with_blanks,
 
 
 @flaky(max_runs=3)
+@pytest.mark.first
 @pytest.mark.skipif(
-    os.environ.get('CI', None) is None,
-    reason='fails sometimes locally')
+    os.environ.get('CI', None) is None or os.name == 'nt',
+    reason='Only on CI and not on windows!',
+)
 def test_run_python_script_in_terminal_with_wdir_empty(scriptpath, qtbot):
     """
     Test running a Python script in an external terminal without specifying
@@ -142,38 +144,44 @@ def test_run_python_script_in_terminal_with_wdir_empty(scriptpath, qtbot):
     assert res == 'done'
 
 
-@pytest.mark.skipif(os.environ.get('CI', None) is None,
-                    reason='It only runs in CI services.')
+@pytest.mark.first
+@pytest.mark.skipif(os.environ.get('CI', None) is None, reason='Only on CI!')
 def test_is_valid_interpreter():
     assert is_python_interpreter(VALID_INTERPRETER)
 
 
-@pytest.mark.skipif(os.environ.get('CI', None) is None,
-                    reason='It only runs in CI services.')
+@pytest.mark.first
+@pytest.mark.skipif(os.environ.get('CI', None) is None, reason='Only on CI!')
 def test_is_invalid_interpreter():
     assert not is_python_interpreter(INVALID_INTERPRETER)
 
 
+@pytest.mark.first
+@pytest.mark.skipif(os.environ.get('CI', None) is None, reason='Only on CI!')
 def test_is_valid_interpreter_name():
     names = ['python', 'pythonw', 'python2.7', 'python3.5', 'python.exe', 'pythonw.exe']
     assert all([is_python_interpreter_valid_name(n) for n in names])
 
+
 def test_find_program():
     """Test if can find the program."""
     assert find_program('git')
 
+
 def test_shell_split():
     """Test if the text can be split using shell-like sintax."""
     assert shell_split('-q -o -a') == ['-q', '-o', '-a']
     assert shell_split('-q "d:\\Python de xxxx\\t.txt" -o -a') == \
            ['-q', 'd:\\Python de xxxx\\t.txt', '-o', '-a']
 
+
 def test_check_version():
     """Test the compare function for versions."""
     assert check_version('0.9.4-1', '0.9.4', '>=')
     assert check_version('3.0.0rc1', '3.0.0', '<')
     assert check_version('1.0', '1.0b2', '>')
 
+
 def test_is_module_installed():
     """Test if a module with the proper version is installed"""
     assert is_module_installed('qtconsole', '>=4.5')
@@ -181,8 +189,6 @@ def test_is_module_installed():
     assert is_module_installed('jedi', '>=0.7.0')
 
 
-@pytest.mark.skipif(os.name == 'nt' and os.environ.get('AZURE') is not None,
-                    reason="Fails on Windows/Azure")
 def test_is_module_installed_with_custom_interpreter():
     """Test if a module with the proper version is installed"""
     current = sys.executable