Repository: fujiisoup/sif_parser (language: python)

Issue:
np_open has trouble reading cascaded sif file
In kinetic mode of Andor whenever the number of images exceeds 4GB it splits the excess files and saves. np_open can't open the extra file if the file isn't 4GB big. It shows the following error in the code section of 

sif_parser\_sif_open.py: 204 in _open(fp)

201 info['ybin'] = ybin
203 for f in range(no_images):
204        info['timestamp_of_{0:d}'.format(f)] = int(fp.readline())  <---
206 offset = fp.tell()
207 try: # remove extra 0 if it exits.

ValueError: invalid literal for int() with base 10: b'      

This only happens when the last file of the split ".sif" files set is not  a complete 4GB file. Any other file the comes in between and is 4GB is read perfectly with the info present. 

I used both the pip version 0.3.0 and  built and installed github version 0.3.5. Both of them has this issue. This issue is not a computer issue either since it has been seen on multiple systems we use.


Discussion / hints:

Fix patch:
diff --git a/sif_parser/_sif_open.py b/sif_parser/_sif_open.py
index be6ce38..d99ad54 100644
--- a/sif_parser/_sif_open.py
+++ b/sif_parser/_sif_open.py
@@ -12,7 +12,7 @@ def _to_string(c):
     ''' convert bytes to string. c: string or bytes'''
     return c if not isinstance(c, bytes) else c.decode('utf-8')
 
-def _read_string(fp, length = None):
+def _read_string(fp, length=None):
     '''Read a string of the given length. If no length is provided, the
     length is read from the file.'''
     if length is None:
@@ -30,6 +30,16 @@ def _read_until(fp, terminator=' '):
         word += c
     return word
 
+def _skip_spaces(fp):
+    '''Read until something other than space or line end '''
+    while True:
+        offset = fp.tell()
+        c = _to_string(fp.read(1))
+        if c not in [' ', '\n']:
+            fp.seek(offset)
+            return fp
+    raise ValueError('Reached the end of the file')
+
 def _read_int(fp):
     return int(_read_until(fp, ' '))
 
@@ -121,7 +131,7 @@ def _open(fp):
     elif info['SifVersion'] == 65558:
         for _ in range(5):
             fp.readline()
-    elif info['SifVersion'] == 65559:
+    elif info['SifVersion'] in [65559, 65564]:
         for _ in range(8):
             fp.readline() # Skip to Line 18
         info['spectrograph'] = _to_string(fp.readline().split()[1])
@@ -165,7 +175,7 @@ def _open(fp):
 
     fp.readline() # 13 newline or 6.5
     fp.readline() # 13 newline or 6.5
-
+    
     info['FrameAxis'] = _read_string(fp) # Line 26 or 39
     info['DataType'] = _read_string(fp)
     info['ImageAxis'] = _read_string(fp)    
@@ -199,10 +209,11 @@ def _open(fp):
     tile = []
     info['xbin'] = xbin
     info['ybin'] = ybin
-
+    
+    fp = _skip_spaces(fp)
     for f in range(no_images):
         info['timestamp_of_{0:d}'.format(f)] = int(fp.readline())
-
+    
     offset = fp.tell()
     try: # remove extra 0 if it exits.
         flag = int(fp.readline())
diff --git a/sif_parser/_version.py b/sif_parser/_version.py
index b3057d9..0b449ba 100644
--- a/sif_parser/_version.py
+++ b/sif_parser/_version.py
@@ -14,4 +14,4 @@
 
 
 __version_info__ = (0, 3, 4)
-__version__ = "0.3.4"  # pragma: no cover
+__version__ = "0.3.5"  # pragma: no cover


Test patch:
diff --git a/testings/issue27/test.sif b/testings/issue27/test.sif
new file mode 100644
index 0000000..a89075b
Binary files /dev/null and b/testings/issue27/test.sif differ
diff --git a/testings/test_open.py b/testings/test_open.py
index 113b89d..9c266ec 100644
--- a/testings/test_open.py
+++ b/testings/test_open.py
@@ -8,6 +8,7 @@
 # data directories that will be tested
 DATA_DIR = THIS_DIR + '/sif_parser_testdata/'
 PUBLIC_DATA_DIR = THIS_DIR + '/public_testdata/'
+EXTRA_DATA_DIR = THIS_DIR + '/extra_data/'
 
 import PIL.Image
 import numpy as np
@@ -21,7 +22,7 @@
 
 
 filenames = []
-for d in [DATA_DIR, PUBLIC_DATA_DIR]:
+for d in [DATA_DIR, PUBLIC_DATA_DIR, EXTRA_DATA_DIR]:
     if os.path.exists(d):
         files = os.listdir(d)
         filenames += [d + f for f in files if f[-4:] in ['.sif', '.SIF']]
@@ -42,6 +43,12 @@
     if os.path.exists(e):
         spool_encoding_dirs += sorted([e + dd for dd in os.listdir(e) if not dd.startswith(".DS")], key=str.lower)
 
+def test_issue27():
+    filename = THIS_DIR + '/issue27/test.sif'
+    with open(filename, 'rb') as f:
+        data, info = sif_parser.np_open(f)
+    assert np.sum(np.isnan(data)) == 0
+
 
 @pytest.mark.parametrize('filename', filenames)
 def test_open(filename):