Repository: NSLS-II/sirepo-bluesky (language: python)

Issue:
Implement exporting of sirepo-data.json to YAML
This was discussed with the LUME collaborators and can be a pretty useful addition to being used by other tools to load virtual beamlines.


Discussion / hints:
May be a good starting point for @rlb131 and @noahjwilson.

Fix patch:
diff --git a/setup.py b/setup.py
index 9108d41a..4cfd850c 100644
--- a/setup.py
+++ b/setup.py
@@ -49,6 +49,7 @@
     entry_points={
         "console_scripts": [
             # 'command = some.module:some_function',
+            "json-yaml-converter = sirepo_bluesky.utils.json_yaml_converter:cli_converter"
         ],
     },
     include_package_data=True,
diff --git a/sirepo_bluesky/utils/__init__.py b/sirepo_bluesky/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/sirepo_bluesky/utils/json_yaml_converter.py b/sirepo_bluesky/utils/json_yaml_converter.py
new file mode 100644
index 00000000..753aa519
--- /dev/null
+++ b/sirepo_bluesky/utils/json_yaml_converter.py
@@ -0,0 +1,125 @@
+import argparse
+import json
+import os
+from enum import Enum
+
+import yaml
+
+
+class Filetype(Enum):
+    JSON = 1
+    YAML = 2
+
+
+def json_to_yaml(json_fp, yaml_fp, openmode="x", **kwargs):
+    """
+    Converts a .json file into a .yaml or .yml file.
+
+    Parameters
+    ----------
+    json_fp : str
+        Filepath of .json file.
+    yaml_fp : str
+        Filepath of new .yaml or .yml file.
+    openmode : str
+        Mode to be passed to open. Set to "x" by default to avoid overwriting files.
+    kwargs : dict
+        Will be passed to yaml.dump.
+    """
+    with open(json_fp, "r") as fp:
+        data = json.load(fp)
+    with open(yaml_fp, mode=openmode) as fp:
+        yaml.dump(data, fp, **kwargs)
+
+
+def yaml_to_json(yaml_fp, json_fp, indent=2, openmode="x", **kwargs):
+    """
+    Converts a .yaml or .yml file into a .json file.
+
+    Parameters
+    ----------
+    yaml_fp : str
+        Filepath of new .yaml or .yml file.
+    json_fp : str
+        Filepath of .json file.
+    indent : int
+        Indentation of .json file, default is 2.
+    openmode : str
+        Mode to be passed to open. Set to "x" by default to avoid overwriting files.
+    kwargs : dict
+        Will be passed to json.dump.
+    """
+    with open(yaml_fp, "r") as fp:
+        data = yaml.safe_load(fp)
+    with open(json_fp, mode=openmode) as fp:
+        json.dump(data, fp, indent=indent, **kwargs)
+
+
+def dict_to_file(dict, filename, indent=2, openmode="x", **kwargs):
+    """
+    Converts a dict into a .json, .yaml, or .yml file.
+
+    Parameters
+    ----------
+    dict : dict
+        Dictionary to be converted.
+    filename : str
+        Filepath of new file.
+    indent : int
+        Indentation of .json file to be created, default is 2.
+    openmode : str
+        Mode to be passed to open. Set to "x" by default to avoid overwriting files.
+    kwargs : dict
+        Will be passed to json.dump or yaml.dump.
+    """
+    filetype = get_file_type(filename)
+    if filetype == Filetype.JSON:
+        with open(filename, mode=openmode) as jsonfile:
+            json.dump(dict, jsonfile, indent=indent, **kwargs)
+    elif filetype == Filetype.YAML:
+        with open(filename, mode=openmode) as yamlfile:
+            yaml.dump(dict, yamlfile, **kwargs)
+    else:
+        raise RuntimeError("Invalid file type: must be .json or .yaml")
+
+
+def cli_converter():
+    # Uses command json-yaml-converter on command line
+    parser = argparse.ArgumentParser(description="Converts from .json to .yaml/.yml and vice versa")
+    parser.add_argument("-i", "--input-file", dest="input_file", help="The file to be converted")
+    parser.add_argument("-o", "--output-file", dest="output_file", help="The target file after conversion")
+    parser.add_argument(
+        "--indent", default=2, type=int, dest="indent", help="The indentation for .json files, default is 2"
+    )
+    parser.add_argument(
+        "-v", "--verbose", dest="verbose", action="store_true", help="Gives more information when running"
+    )
+    args = parser.parse_args()
+    if args.verbose:
+        print(f"\n{args.input_file = }")
+        print(f"{args.output_file = }")
+        print(f"{args.indent = }")
+    if not (args.input_file and args.output_file):
+        parser.error("Input_file and/or output_file is not specified")
+    filetype_input = get_file_type(args.input_file)
+    filetype_output = get_file_type(args.output_file)
+    if filetype_input == Filetype.JSON and filetype_output == Filetype.YAML:
+        json_to_yaml(args.input_file, args.output_file)
+        if args.verbose:
+            print(".yaml file created.")
+    elif filetype_input == Filetype.YAML and filetype_output == Filetype.JSON:
+        yaml_to_json(args.input_file, args.output_file, indent=args.indent)
+        if args.verbose:
+            print(".json file created.")
+    else:
+        raise RuntimeError("Invalid conversion: must convert from .json to .yaml/.yml or vice versa")
+
+
+def get_file_type(filename):
+    filetype = os.path.splitext(filename)[-1]
+    if filetype == ".json":
+        return Filetype.JSON
+    elif filetype == ".yaml" or filetype == ".yml":
+        return Filetype.YAML
+    else:
+        raise ValueError("Invalid file type: file must be .json or .yaml/.yml")


Test patch:
diff --git a/sirepo_bluesky/tests/test_converter.py b/sirepo_bluesky/tests/test_converter.py
new file mode 100644
index 00000000..d599d13c
--- /dev/null
+++ b/sirepo_bluesky/tests/test_converter.py
@@ -0,0 +1,97 @@
+import json
+import os
+import subprocess
+from pprint import pprint
+
+import pytest
+import yaml
+
+from sirepo_bluesky.utils.json_yaml_converter import dict_to_file, json_to_yaml, yaml_to_json
+
+
+def test_dict_to_file(tmp_path):
+    test_dict = {"a": 1, "b": 2, "c": 3}
+
+    dict_to_file(test_dict, tmp_path / "test1.json")
+    with open(tmp_path / "test1.json", "r") as fp:
+        assert test_dict == json.load(fp)
+
+    dict_to_file(test_dict, tmp_path / "test2.json", indent=4)
+    with open(tmp_path / "test2.json", "r") as fp:
+        assert test_dict == json.load(fp)
+
+    dict_to_file(test_dict, tmp_path / "test.yaml")
+    with open(tmp_path / "test.yaml", "r") as fp:
+        assert test_dict == yaml.safe_load(fp)
+
+    with pytest.raises(ValueError):
+        dict_to_file(test_dict, "test.jpeg")
+
+    test_dict = {"a": 1, "b": 2, "c": 4}
+
+    with pytest.raises(FileExistsError):
+        dict_to_file(test_dict, tmp_path / "test2.json")
+
+    dict_to_file(test_dict, tmp_path / "test2.json", openmode="w")
+    with open(tmp_path / "test2.json", "r") as fp:
+        assert test_dict == json.load(fp)
+
+
+def test_json_yaml_converters(tmp_path):
+    test_dict = {"a": 1, "b": 2, "c": 3}
+    dict_to_file(test_dict, tmp_path / "test1.json")
+
+    json_to_yaml(tmp_path / "test1.json", tmp_path / "test.yaml")
+    with open(tmp_path / "test.yaml", "r") as fp:
+        assert test_dict == yaml.safe_load(fp)
+
+    yaml_to_json(tmp_path / "test.yaml", tmp_path / "test2.json")
+    with open(tmp_path / "test2.json", "r") as fp:
+        assert test_dict == json.load(fp)
+
+    yaml_to_json(tmp_path / "test.yaml", tmp_path / "test3.json", indent=4)
+    with open(tmp_path / "test3.json", "r") as fp:
+        assert test_dict == json.load(fp)
+
+
+@pytest.mark.docker
+def test_dict_to_file_from_simulation(tmp_path, shadow_tes_simulation):
+    # Sirepo must be running for this test to pass, deselect with '-m "not docker"'
+    test_dict = shadow_tes_simulation.data
+
+    dict_to_file(test_dict, tmp_path / "test1.json")
+    with open(tmp_path / "test1.json", "r") as fp:
+        assert test_dict == json.load(fp)
+
+    dict_to_file(test_dict, tmp_path / "test2.json", indent=4)
+    with open(tmp_path / "test2.json", "r") as fp:
+        assert test_dict == json.load(fp)
+
+    dict_to_file(test_dict, tmp_path / "test.yaml")
+    with open(tmp_path / "test.yaml", "r") as fp:
+        assert test_dict == yaml.safe_load(fp)
+
+    pprint(test_dict)
+
+
+def test_cli_converter(tmp_path):
+    test_dict = {"a": 1, "b": 2, "c": 3}
+
+    dict_to_file(test_dict, tmp_path / "test1.json")
+
+    assert os.path.isfile(tmp_path / "test1.json")
+
+    subprocess.run(
+        f"json-yaml-converter -i {tmp_path / 'test1.json'} -o {tmp_path / 'test.yaml'} -v".split(), check=True
+    )
+
+    assert os.path.isfile(tmp_path / "test.yaml")
+
+    with open(str(tmp_path / "test.yaml"), "r") as fp:
+        assert test_dict == yaml.safe_load(fp)
+
+    subprocess.run(
+        f"json-yaml-converter -i {tmp_path / 'test.yaml'} -o {tmp_path / 'test2.json'} -v".split(), check=True
+    )
+    with open(str(tmp_path / "test2.json"), "r") as fp:
+        assert test_dict == yaml.safe_load(fp)