Repository: nipy/heudiconv

Issue:
enhance explicitness about what json files heudiconv fails to read
<!-- DO NOT DELETE THIS! 
This template is used to facilitate issue resolution.
All text in <!-> tags will not be displayed.
-->

### Summary
I was helping a user troubleshoot a problem with heudiconv, as it was tripping up on an invalid json file, but the user only thought heudiconv was touching files in the subject directory inside the bids directory, and not looking in other places inside the bids directory (which it does).

A clue that could have gotten this user on track faster would be to print which file was the invalid json file.

To help, the `load_json` function could be changed to something like:

```
try:
    with open(filename, 'r') as fp:
            data = json.load(fp)
except json.JSONDecodeError:
    print("{fname} is not a valid json file".format(fname=filename)
    raise
```
Thoughts on this?
If positive I can open a pull request.
<!-- If you are having conversion troubles, please share as much
relevant information as possible. This includes, but is not limited to:
- log of conversion
- heuristic
-->



### Platform details:

Choose one:
- [ ] Local environment
<!-- If selected, please provide OS and python version -->
- [x] Container
<!-- If selected, please provide container name and tag"-->

- Heudiconv version: 0.5.4
<!-- To check: run heudiconv with just the --version flag -->



Comments:

Patch:
diff --git a/heudiconv/bids.py b/heudiconv/bids.py
index 1bbfcf6..9569278 100644
--- a/heudiconv/bids.py
+++ b/heudiconv/bids.py
@@ -240,6 +240,27 @@ def add_participant_record(studydir, subject, age, sex):
             known_subjects = {l.split('\t')[0] for l in f.readlines()}
         if participant_id in known_subjects:
             return
+    else:
+        # Populate particpants.json (an optional file to describe column names in
+        # participant.tsv). This auto generation will make BIDS-validator happy.
+        participants_json = op.join(studydir, 'participants.json')
+        if not op.lexists(participants_json):
+            save_json(participants_json,
+                OrderedDict([
+                    ("participant_id", OrderedDict([
+                        ("Description", "Participant identifier")])),
+                    ("age", OrderedDict([
+                        ("Description", "Age in years (TODO - verify) as in the initial"
+                            " session, might not be correct for other sessions")])),
+                    ("sex", OrderedDict([
+                        ("Description", "self-rated by participant, M for male/F for "
+                            "female (TODO: verify)")])),
+                    ("group", OrderedDict([
+                        ("Description", "(TODO: adjust - by default everyone is in "
+                            "control group)")])),
+                ]),
+                sort_keys=False,
+                indent=2)
     # Add a new participant
     with open(participants_tsv, 'a') as f:
         f.write(
@@ -311,7 +332,8 @@ def save_scans_key(item, bids_files):
 
 def add_rows_to_scans_keys_file(fn, newrows):
     """
-    Add new rows to file fn for scans key filename
+    Add new rows to file fn for scans key filename and generate accompanying json
+    descriptor to make BIDS validator happy.
 
     Parameters
     ----------
@@ -334,6 +356,25 @@ def add_rows_to_scans_keys_file(fn, newrows):
         os.unlink(fn)
     else:
         fnames2info = newrows
+        # Populate _scans.json (an optional file to describe column names in
+        # _scans.tsv). This auto generation will make BIDS-validator happy.
+        scans_json = '.'.join(fn.split('.')[:-1] + ['json'])
+        if not op.lexists(scans_json):
+            save_json(scans_json,
+                OrderedDict([
+                    ("filename", OrderedDict([
+                        ("Description", "Name of the nifti file")])),
+                    ("acq_time", OrderedDict([
+                        ("LongName", "Acquisition time"),
+                        ("Description", "Acquisition time of the particular scan")])),
+                    ("operator", OrderedDict([
+                        ("Description", "Name of the operator")])),
+                    ("randstr", OrderedDict([
+                        ("LongName", "Random string"),
+                        ("Description", "md5 hash of UIDs")])),
+                ]),
+                sort_keys=False,
+                indent=2)
 
     header = ['filename', 'acq_time', 'operator', 'randstr']
     # prepare all the data rows
diff --git a/heudiconv/info.py b/heudiconv/info.py
index b970b3b..feeae46 100644
--- a/heudiconv/info.py
+++ b/heudiconv/info.py
@@ -8,6 +8,19 @@ __longdesc__ = """Convert DICOM dirs based on heuristic info - HeuDiConv
 uses the dcmstack package and dcm2niix tool to convert DICOM directories or
 tarballs into collections of NIfTI files following pre-defined heuristic(s)."""
 
+CLASSIFIERS = [
+    'Environment :: Console',
+    'Intended Audience :: Science/Research',
+    'License :: OSI Approved :: Apache Software License',
+    'Programming Language :: Python :: 2.7',
+    'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+    'Topic :: Scientific/Engineering'
+]
+
+PYTHON_REQUIRES = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+
 REQUIRES = [
     'nibabel',
     'pydicom',
diff --git a/heudiconv/utils.py b/heudiconv/utils.py
index abea820..2453e9d 100644
--- a/heudiconv/utils.py
+++ b/heudiconv/utils.py
@@ -19,6 +19,11 @@ from nipype.utils.filemanip import which
 import logging
 lgr = logging.getLogger(__name__)
 
+if sys.version_info[0] > 2:
+    from json.decoder import JSONDecodeError
+else:
+    JSONDecodeError = ValueError
+
 
 seqinfo_fields = [
     'total_files_till_now',  # 0
@@ -172,10 +177,15 @@ def load_json(filename):
     -------
     data : dict
     """
-    with open(filename, 'r') as fp:
-        data = json.load(fp)
-    return data
+    try:
+        with open(filename, 'r') as fp:
+            data = json.load(fp)
+    except JSONDecodeError:
+        lgr.error("{fname} is not a valid json file".format(fname=filename))
+        raise
 
+    return data
+    
 
 def assure_no_file_exists(path):
     """Check if file or symlink (git-annex?) exists, and if so -- remove"""
diff --git a/setup.py b/setup.py
index 6017447..c3a804a 100755
--- a/setup.py
+++ b/setup.py
@@ -47,11 +47,13 @@ def main():
         description=ldict['__description__'],
         long_description=ldict['__longdesc__'],
         license=ldict['__license__'],
+        classifiers=ldict['CLASSIFIERS'],
         packages=heudiconv_pkgs,
         entry_points={'console_scripts': [
             'heudiconv=heudiconv.cli.run:main',
             'heudiconv_monitor=heudiconv.cli.monitor:main',
         ]},
+        python_requires=ldict['PYTHON_REQUIRES'],
         install_requires=ldict['REQUIRES'],
         extras_require=ldict['EXTRA_REQUIRES'],
         package_data={


Test patch:
diff --git a/heudiconv/tests/test_heuristics.py b/heudiconv/tests/test_heuristics.py
index c9485c6..f36bbb4 100644
--- a/heudiconv/tests/test_heuristics.py
+++ b/heudiconv/tests/test_heuristics.py
@@ -165,7 +165,13 @@ def test_notop(tmpdir, bidsoptions):
     runner(args)
 
     assert op.exists(pjoin(tmppath, 'Halchenko/Yarik/950_bids_test4'))
-    for fname in ['CHANGES', 'dataset_description.json', 'participants.tsv', 'README']:
+    for fname in [
+        'CHANGES',
+        'dataset_description.json',
+        'participants.tsv',
+        'README',
+        'participants.json'
+    ]:
         if 'notop' in bidsoptions:
             assert not op.exists(pjoin(tmppath, 'Halchenko/Yarik/950_bids_test4', fname))
         else:
diff --git a/heudiconv/tests/test_main.py b/heudiconv/tests/test_main.py
index 7dcd32e..f4b7700 100644
--- a/heudiconv/tests/test_main.py
+++ b/heudiconv/tests/test_main.py
@@ -22,6 +22,7 @@ from mock import patch
 from os.path import join as opj
 from six.moves import StringIO
 import stat
+import os.path as op
 
 
 @patch('sys.stdout', new_callable=StringIO)
@@ -205,6 +206,7 @@ def test_add_rows_to_scans_keys_file(tmpdir):
         assert dates == sorted(dates)
 
     _check_rows(fn, rows)
+    assert op.exists(opj(tmpdir.strpath, 'file.json'))
     # add a new one
     extra_rows = {
         'a_new_file.nii.gz': ['2016adsfasd23', '', 'fasadfasdf'],
diff --git a/heudiconv/tests/test_utils.py b/heudiconv/tests/test_utils.py
index 47faec4..ad84673 100644
--- a/heudiconv/tests/test_utils.py
+++ b/heudiconv/tests/test_utils.py
@@ -1,10 +1,16 @@
+import json
 import os
 import os.path as op
+
 from heudiconv.utils import (
     get_known_heuristics_with_descriptions,
     get_heuristic_description,
     load_heuristic,
-    json_dumps_pretty)
+    json_dumps_pretty,
+    load_json,
+    create_tree,
+    save_json,
+    JSONDecodeError)
 
 import pytest
 from .utils import HEURISTICS_PATH
@@ -58,4 +64,24 @@ def test_json_dumps_pretty():
           'Mar  3 2017 10:46:13 by eja'
     # just the date which reveals the issue
     # tstr = 'Mar  3 2017 10:46:13 by eja'
-    assert pretty({'WipMemBlock': tstr}) == '{\n  "WipMemBlock": "%s"\n}' % tstr
\ No newline at end of file
+    assert pretty({'WipMemBlock': tstr}) == '{\n  "WipMemBlock": "%s"\n}' % tstr
+
+
+def test_load_json(tmp_path, caplog):
+    # test invalid json
+    ifname = 'invalid.json'
+    invalid_json_file = str(tmp_path / ifname)
+    create_tree(str(tmp_path), {ifname: u"I'm Jason Bourne"})
+
+    with pytest.raises(JSONDecodeError):
+        load_json(str(invalid_json_file))
+
+    assert ifname in caplog.text
+
+    # test valid json
+    vcontent = {"secret": "spy"}
+    vfname = "valid.json"
+    valid_json_file = str(tmp_path / vfname)
+    save_json(valid_json_file, vcontent)
+    
+    assert load_json(valid_json_file) == vcontent