Repository: EdinburghGenomics/EGCG-Core

Issue:
clarity.get_list_of_samples crashes if a sample doesn't exist in the Lims
The error is because of the way we're processing sample names. The stacktrace is misleading:
```
File "clarity.py", line 98, in get_list_of_samples
    results.extend(_get_list_of_samples(sample_names[start:start+max_query]))
  File "clarity.py", line 114, in _get_list_of_samples
    samples.extend(_get_list_of_samples(remainder, sub + 1))
  File "clarity.py", line 114, in _get_list_of_samples
    samples.extend(_get_list_of_samples(remainder, sub + 1))
  File "clarity.py", line 114, in _get_list_of_samples
    samples.extend(_get_list_of_samples(remainder, sub + 1))
  File "clarity.py", line 102, in _get_list_of_samples
    pattern, repl = substitutions[sub]
```
We should return None for non-existent samples, or leave them out of the returned list.

Comments:

Patch:
diff --git a/egcg_core/__init__.py b/egcg_core/__init__.py
index e1424ed..58d168b 100644
--- a/egcg_core/__init__.py
+++ b/egcg_core/__init__.py
@@ -1,1 +1,1 @@
-__version__ = '0.3.1'
+__version__ = '0.4'
diff --git a/egcg_core/clarity.py b/egcg_core/clarity.py
index 4691f7c..e896d04 100644
--- a/egcg_core/clarity.py
+++ b/egcg_core/clarity.py
@@ -2,13 +2,16 @@ import re
 from genologics.lims import Lims
 from egcg_core.config import cfg
 from egcg_core.app_logging import logging_default as log_cfg
-from egcg_core.exceptions import LimsCommunicationError
+from egcg_core.exceptions import EGCGError
 
 app_logger = log_cfg.get_logger('clarity')
 try:
     from egcg_core.ncbi import get_species_name
 except ImportError:
-    app_logger.error('Could not import egcg_core.ncbi. Is sqlite3 available?')
+    app_logger.warning('Could not import egcg_core.ncbi. Is sqlite3 available?')
+
+    def get_species_name(query_species):
+        raise EGCGError('Could not import egcg_core.ncbi.get_species_name - sqlite3 seems to be unavailable.')
 
 
 _lims = None
@@ -91,8 +94,8 @@ def sanitize_user_id(user_id):
 
 substitutions = (
     (None, None),
-    (re.compile('_(\d{2})$'), ':\g<1>'),
-    (re.compile('__(\w):(\d{2})'), ' _\g<1>:\g<2>')
+    (re.compile('_(\d{2})$'), ':\g<1>'),  # '_01' -> ':01'
+    (re.compile('__(\w):(\d{2})'), ' _\g<1>:\g<2>')  # '__L:01' -> ' _L:01'
 )
 
 
@@ -115,11 +118,12 @@ def _get_list_of_samples(sample_names, sub=0):
     lims.get_batch(samples)
 
     if len(samples) != len(sample_names):  # haven't got all the samples because some had _01/__L:01
+        sub += 1
+        remainder = sorted(set(_sample_names).difference(set([s.name for s in samples])))
         if sub < len(substitutions):
-            remainder = sorted(set(_sample_names).difference(set([s.name for s in samples])))
-            samples.extend(_get_list_of_samples(remainder, sub + 1))
-        else:
-            raise LimsCommunicationError('Expected %s back, got %s' % (_sample_names, len(samples)))
+            samples.extend(_get_list_of_samples(remainder, sub))
+        else:  # end recursion
+            app_logger.warning('Could not find %s in Lims' % remainder)
 
     return samples
 
diff --git a/egcg_core/executor/cluster_executor.py b/egcg_core/executor/cluster_executor.py
index ce66a44..f9b29d1 100644
--- a/egcg_core/executor/cluster_executor.py
+++ b/egcg_core/executor/cluster_executor.py
@@ -38,9 +38,8 @@ class ClusterExecutor(AppLogger):
             sleep(30)
         return self._job_exit_code()
 
-    @classmethod
-    def _get_writer(cls, job_name, working_dir, walltime=None, cpus=1, mem=2, jobs=1, log_commands=True):
-        return cls.script_writer(job_name, working_dir, cfg['job_queue'], cpus, mem, walltime, jobs, log_commands)
+    def _get_writer(self, job_name, working_dir, walltime=None, cpus=1, mem=2, jobs=1, log_commands=True):
+        return self.script_writer(job_name, working_dir, self.job_queue, cpus, mem, walltime, jobs, log_commands)
 
     def _job_status(self):
         raise NotImplementedError


Test patch:
diff --git a/tests/test_clarity.py b/tests/test_clarity.py
index b4c3684..6e963a2 100644
--- a/tests/test_clarity.py
+++ b/tests/test_clarity.py
@@ -104,10 +104,9 @@ def test_get_list_of_samples():
     exp_lims_sample_ids = ['this', 'that:01', 'other _L:01']
     calling_sample_ids = ['this', 'that_01', 'other__L_01']
     fake_list_samples = [[FakeEntity(n)] for n in exp_lims_sample_ids]
-    pbatch = patched_lims('get_batch')
     psamples = patched_lims('get_samples', side_effect=fake_list_samples)
 
-    with pbatch, psamples as mocked_get_samples:
+    with patched_lims('get_batch'), psamples as mocked_get_samples:
         samples = clarity.get_list_of_samples(calling_sample_ids)
         assert [s.name for s in samples] == exp_lims_sample_ids
         mocked_get_samples.assert_any_call(name=['this', 'that_01', 'other__L_01'])
@@ -115,6 +114,23 @@ def test_get_list_of_samples():
         mocked_get_samples.assert_any_call(name=['other _L:01'])
 
 
+def test_get_list_of_samples_broken():
+    exp_lims_sample_ids = ['this', 'that:01', 'other _L:01']
+    calling_sample_ids = ['this', 'that_01', 'other__L_01']
+    fake_list_samples = [[FakeEntity(n)] for n in exp_lims_sample_ids]
+    psamples = patched_lims('get_samples', side_effect=fake_list_samples)
+    log_msgs = []
+    pwarn = patched('app_logger.warning', new=log_msgs.append)
+
+    with patched_lims('get_batch'), psamples as mocked_get_samples, pwarn:
+        samples = clarity.get_list_of_samples(calling_sample_ids + ['sample_not_in_lims'])
+        assert [s.name for s in samples] == exp_lims_sample_ids
+        mocked_get_samples.assert_any_call(name=['this', 'that_01', 'other__L_01', 'sample_not_in_lims'])
+        mocked_get_samples.assert_any_call(name=['other__L:01', 'sample_not_in_lims', 'that:01'])
+        mocked_get_samples.assert_any_call(name=['other _L:01', 'sample_not_in_lims'])
+        assert log_msgs == ["Could not find ['sample_not_in_lims'] in Lims"]
+
+
 @patched_lims('get_samples', side_effect=[[], [], [None]])
 def test_get_samples(mocked_lims):
     assert clarity.get_samples('a_sample_name__L_01') == [None]
@@ -206,8 +222,7 @@ def test_get_output_containers_from_sample_and_step_name(mocked_get_sample, mock
 
 
 @patched_clarity('get_sample_names_from_plate', ['this', 'that', 'other'])
-@patched_clarity('get_sample',
-                 Mock(artifact=Mock(container=FakeEntity('a_container', type=FakeEntity('96 well plate')))))
+@patched_clarity('get_sample', Mock(artifact=Mock(container=FakeEntity('a_container', type=FakeEntity('96 well plate')))))
 def test_get_samples_arrived_with(mocked_get_sample, mocked_names_from_plate):
     assert clarity.get_samples_arrived_with('a_sample_name') == ['this', 'that', 'other']
     mocked_get_sample.assert_called_with('a_sample_name')