Repository: bids-standard/pybids

Issue:
crash with AttributeError: Can only use .str accessor with string values, which use np.object_ dtype in pandas
```shell
$> python -c 'from bids import BIDSLayout; b=BIDSLayout("ds001976", derivatives=False); b.get_collections(level="dataset")'
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/layout/layout.py", line 848, in get_collections
    skip_empty=skip_empty, **kwargs)
  File "/home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/variables/io.py", line 83, in load_variables
    **kwargs)
  File "/home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/variables/io.py", line 375, in _load_tsv_variables
    _data['subject'] = _data['subject'].str.replace('sub-', '')
  File "/usr/lib/python3/dist-packages/pandas/core/generic.py", line 4374, in __getattr__
    return object.__getattribute__(self, name)
  File "/usr/lib/python3/dist-packages/pandas/core/accessor.py", line 133, in __get__
    accessor_obj = self._accessor(obj)
  File "/usr/lib/python3/dist-packages/pandas/core/strings.py", line 1895, in __init__
    self._validate(data)
  File "/usr/lib/python3/dist-packages/pandas/core/strings.py", line 1917, in _validate
    raise AttributeError("Can only use .str accessor with string "
AttributeError: Can only use .str accessor with string values, which use np.object_ dtype in pandas

$> welp bids
PATH       : /home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/__init__.py
SRC PATH   : /home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/__init__.py
VERSION    : Not found
__version__: '0.9.3+3.g6f77de6'

$> welp pandas
PATH       : /usr/lib/python3/dist-packages/pandas/__init__.py
SRC PATH   : /usr/lib/python3/dist-packages/pandas/__init__.py
VERSION    : 0.23.3+dfsg
__version__: '0.23.3'
```
participants.tsv itself seems to read ok: 
```shell
$> for f in ds001976*/participants.tsv ; do echo -n "$f: "; python -c "import pandas as pd; pd.read_csv('$f', sep='\t')" && echo "Ok"; done 
ds001976/participants.tsv: Ok
```
crash with AttributeError: Can only use .str accessor with string values, which use np.object_ dtype in pandas
```shell
$> python -c 'from bids import BIDSLayout; b=BIDSLayout("ds001976", derivatives=False); b.get_collections(level="dataset")'
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/layout/layout.py", line 848, in get_collections
    skip_empty=skip_empty, **kwargs)
  File "/home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/variables/io.py", line 83, in load_variables
    **kwargs)
  File "/home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/variables/io.py", line 375, in _load_tsv_variables
    _data['subject'] = _data['subject'].str.replace('sub-', '')
  File "/usr/lib/python3/dist-packages/pandas/core/generic.py", line 4374, in __getattr__
    return object.__getattribute__(self, name)
  File "/usr/lib/python3/dist-packages/pandas/core/accessor.py", line 133, in __get__
    accessor_obj = self._accessor(obj)
  File "/usr/lib/python3/dist-packages/pandas/core/strings.py", line 1895, in __init__
    self._validate(data)
  File "/usr/lib/python3/dist-packages/pandas/core/strings.py", line 1917, in _validate
    raise AttributeError("Can only use .str accessor with string "
AttributeError: Can only use .str accessor with string values, which use np.object_ dtype in pandas

$> welp bids
PATH       : /home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/__init__.py
SRC PATH   : /home/yoh/proj/datalad/datalad-crawler/venvs/dev3/lib/python3.7/site-packages/bids/__init__.py
VERSION    : Not found
__version__: '0.9.3+3.g6f77de6'

$> welp pandas
PATH       : /usr/lib/python3/dist-packages/pandas/__init__.py
SRC PATH   : /usr/lib/python3/dist-packages/pandas/__init__.py
VERSION    : 0.23.3+dfsg
__version__: '0.23.3'
```
participants.tsv itself seems to read ok: 
```shell
$> for f in ds001976*/participants.tsv ; do echo -n "$f: "; python -c "import pandas as pd; pd.read_csv('$f', sep='\t')" && echo "Ok"; done 
ds001976/participants.tsv: Ok
```


Discussion:
Hmm. Well, if we find that it's not a string, we can probably assume it doesn't start with `sub-` and either raise an exception or just convert the column to a string.

Also, what is this `welp`?
I think we should just force the `subject` column to a str dtype when the .tsv is first read in, seeing as the spec explicitly says that subject labels are strings. (And the same for any other columns that get similar treatment.)
> Also, what is this `welp`?

just a little helper I have in my shell -- shows the version of the python pkg and where it comes from
Hmm. Well, if we find that it's not a string, we can probably assume it doesn't start with `sub-` and either raise an exception or just convert the column to a string.

Also, what is this `welp`?
I think we should just force the `subject` column to a str dtype when the .tsv is first read in, seeing as the spec explicitly says that subject labels are strings. (And the same for any other columns that get similar treatment.)
> Also, what is this `welp`?

just a little helper I have in my shell -- shows the version of the python pkg and where it comes from

Resolving patch:
diff --git a/bids/layout/models.py b/bids/layout/models.py
--- a/bids/layout/models.py
+++ b/bids/layout/models.py
@@ -132,7 +132,6 @@ def __init__(self, filename):
         self.filename = os.path.basename(self.path)
         self.dirname = os.path.dirname(self.path)
         self.is_dir = not self.filename
-        self._init_on_load()
 
     def __getattr__(self, attr):
         # Ensures backwards compatibility with old File_ namedtuple, which is
@@ -155,10 +154,6 @@ def __repr__(self):
     def __fspath__(self):
         return self.path
 
-    @reconstructor
-    def _init_on_load(self):
-        self._data = None
-
     def get_associations(self, kind=None, include_parents=False):
         """ Get associated files, optionally limiting by association kind.
 
@@ -284,7 +279,8 @@ class BIDSDataFile(BIDSFile):
         'polymorphic_identity': 'data_file'
     }
 
-    def get_df(self, include_timing=True, adjust_onset=False):
+    def get_df(self, include_timing=True, adjust_onset=False,
+               enforce_dtypes=True):
         """ Return the contents of a tsv file as a pandas DataFrame.
 
         Args:
@@ -294,20 +290,34 @@ def get_df(self, include_timing=True, adjust_onset=False):
                 timeseries file is shifted to reflect the "StartTime" value in
                 the JSON sidecar. If False, the first sample starts at 0 secs.
                 Ignored if include_timing=False.
+            enforce_dtypes (bool): If True, enforces the data types defined in
+                the BIDS spec on core columns (e.g., subject_id and session_id
+                must be represented as strings).
 
         Returns: A pandas DataFrame.
         """
         import pandas as pd
         import numpy as np
 
-        # TODO: memoize method instead of just caching the raw data
-        if self._data is None:
-            self._data = pd.read_csv(self.path, sep='\t', na_values='n/a')
+        if enforce_dtypes:
+            dtype = {
+                'subject_id': str,
+                'session_id': str,
+                'participant_id': str
+            }
+        else:
+            dtype = None
+
+        # TODO: memoize this for efficiency. (Note: caching is insufficient,
+        # because the dtype enforcement will break if we ignore the value of
+        # enforce_dtypes).
+        self.data = pd.read_csv(self.path, sep='\t', na_values='n/a',
+                                dtype=dtype)
 
-        data = self._data.copy()
-        md = self.get_metadata()
+        data = self.data.copy()
 
         if self.entities['extension'] == 'tsv.gz':
+            md = self.get_metadata()
             # We could potentially include some validation here, but that seems
             # like a job for the BIDS Validator.
             data.columns = md['Columns']
diff --git a/bids/variables/io.py b/bids/variables/io.py
--- a/bids/variables/io.py
+++ b/bids/variables/io.py
@@ -365,7 +365,7 @@ def _load_tsv_variables(layout, suffix, dataset=None, columns=None,
 
     for f in files:
 
-        _data = pd.read_csv(f.path, sep='\t')
+        _data = f.get_df(include_timing=False)
 
         # Entities can be defined either within the first column of the .tsv
         # file (for entities that vary by row), or from the full file path


Test patch:
diff --git a/bids/layout/tests/test_models.py b/bids/layout/tests/test_models.py
--- a/bids/layout/tests/test_models.py
+++ b/bids/layout/tests/test_models.py
@@ -187,6 +187,17 @@ def test_bidsfile_get_df_from_tsv_gz(layout_synthetic):
     assert np.allclose(df3.iloc[:,0], df1.iloc[:, 0] + 22.8)
 
 
+def test_bidsdatafile_enforces_dtype(layout_synthetic):
+    bf = layout_synthetic.get(suffix='participants', extension='tsv')[0]
+    df = bf.get_df(enforce_dtypes=False)
+    assert df.loc[:, 'subject_id'].dtype == int
+    assert df.loc[:, 'subject_id'][0] == 1
+    df = bf.get_df(enforce_dtypes=True)
+    assert df.loc[:, 'subject_id'].dtype == 'O'
+    assert df.loc[:, 'subject_id'][0] == '001'
+    assert df.loc[:, 'subject_id'][1] == '2'
+
+
 def test_bidsimagefile_get_image():
     path = "synthetic/sub-01/ses-01/func/sub-01_ses-01_task-nback_run-01_bold.nii.gz"
     path = path.split('/')
diff --git a/bids/tests/data/synthetic/participants.tsv b/bids/tests/data/synthetic/participants.tsv
--- a/bids/tests/data/synthetic/participants.tsv
+++ b/bids/tests/data/synthetic/participants.tsv
@@ -1,6 +1,6 @@
-participant_id	age	sex
-sub-01	34	F
-sub-02	38	M
-sub-03	22	M
-sub-04	21	F
-sub-05	42	M
+participant_id	age	sex	subject_id
+sub-01	34	F	001
+sub-02	38	M	2
+sub-03	22	M	3
+sub-04	21	F	4
+sub-05	42	M	5
\ No newline at end of file