Repository: ddionrails/ddionrails

Issue:
ddionrails/workspace/datafiles.json causes HTTP 500 errors 
### Subject of the issue

When initiating a script generator, [`workspace.scripts.soep_datasets.SoepDatasets.get_dict()`](https://github.com/ddionrails/ddionrails/blob/154afbd487a2b2776a48dd02ebea8f67a43c7a02/ddionrails/workspace/scripts/soep_datasets.py#L77) is called.
The `SoepDatasets` class is initiated with the content of a json file: https://github.com/ddionrails/ddionrails/blob/master/ddionrails/workspace/datafiles.json

If a variable inside a basket does not belong to a dataset in this json file, a KeyError will be raise [here](https://github.com/ddionrails/ddionrails/blob/154afbd487a2b2776a48dd02ebea8f67a43c7a02/ddionrails/workspace/scripts/soep_datasets.py#L78).
Causing the user to receive a HTTP 500 internal server error.

A variable that causes this problem would be soep-core/data/biol/lb0191 for example.

Datasets from the soep-core repository that are not inside the json file are:

```
biol
hbrutt
hbrutto
hgen
hl
hpath
hpathl
jugendl
pbrutto
pequiv
pgen
pkal
pl
plueckel
ppath
ppathl
vpl
```

@kwenzig told me, that these are datasets not compatible with the script generator.

### Expected behaviour

I see two ways to address this.

The missing datasets should be added to the datafiles.json If the system and the schema for the `datafiles.json` support the inclusion of datasets, that should be ignored during the script generation.

Otherwise the UI for the script generator should be changed to display an error that identifies the unsupported variables to the user. So that they can remove them from their baskets


Discussion:
Issue-Label Bot is automatically applying the label `bug` to this issue, with a confidence of 0.91. Please mark this comment with :thumbsup: or :thumbsdown: to give our bot feedback! 

 Links: [app homepage](https://github.com/marketplace/issue-label-bot), [dashboard](https://mlbot.net/data/ddionrails/ddionrails) and [code](https://github.com/hamelsmu/MLapp) for this bot.

Resolving patch:
diff --git a/ddionrails/workspace/mixins.py b/ddionrails/workspace/mixins.py
--- a/ddionrails/workspace/mixins.py
+++ b/ddionrails/workspace/mixins.py
@@ -11,9 +11,7 @@ class SoepMixin:
     """SoepMixin - utilities/helpers to get SOEP years and year prefixes"""
 
     def _generate_script_dict(self):
-        """
-        Creates a dictionary with dataset names as keys and variables from basket as values
-        """
+        """ Map variables from basket to their dataset names """
         script_dict = dict()
         for variable in self.basket.variables.all():
             dataset_name = variable.dataset.name
@@ -24,20 +22,25 @@ def _generate_script_dict(self):
                 self._enrich_dataset_dict(dataset_dict)
         return script_dict
 
-    def _create_dataset_dict(self, dataset_name: str) -> Dict:
-        """
-        Returns dict with dataset information
+    @staticmethod
+    def _create_dataset_dict(dataset_name: str) -> Dict:
+        """ Returns dict with dataset information
+
+        A possible empty dict from SoepDatasets().get_dict()
+        Is handled with "" as default values.
+        Variables from such a dataset will be included in the
+        `NOT PROCESSED` section of the created script.
         """
         dataset = SoepDatasets().get_dict(dataset_name)
         return dict(
-            name = dataset_name,
-            analysis_unit = dataset["analyse_unit"],
-            period = dataset["syear"],
-            prefix = dataset["prefix"],
-            is_matchable = dataset["is_matchable"],
-            is_special = dataset["is_special"],
-            curr_hid = dataset["curr_hid"],
-            variables = set(),
+            name=dataset_name,
+            analysis_unit=dataset.get("analyse_unit", ""),
+            period=dataset.get("syear", ""),
+            prefix=dataset.get("prefix", ""),
+            is_matchable=dataset.get("is_matchable", ""),
+            is_special=dataset.get("is_special", ""),
+            curr_hid=dataset.get("curr_hid", ""),
+            variables=set(),
         )
 
     @staticmethod
@@ -46,18 +49,17 @@ def _enrich_dataset_dict(dataset_dict):
         Set merge_id variables and add variables to dataset
         """
 
-        d = dataset_dict
-        analysis_unit = d["analysis_unit"]
+        analysis_unit = dataset_dict["analysis_unit"]
         if analysis_unit == "h":
-            d["merge_id"] = d["curr_hid"]
-            d["variables"].add(d["curr_hid"])
+            dataset_dict["merge_id"] = dataset_dict["curr_hid"]
+            dataset_dict["variables"].add(dataset_dict["curr_hid"])
         elif analysis_unit == "p":
-            d["merge_id"] = "persnr"
-            d["variables"].add("persnr")
-            if d["curr_hid"] != "":
-                d["variables"].add(d["curr_hid"])
+            dataset_dict["merge_id"] = "persnr"
+            dataset_dict["variables"].add("persnr")
+            if dataset_dict["curr_hid"] != "":
+                dataset_dict["variables"].add(dataset_dict["curr_hid"])
         else:
-            d["merge_id"] = ""
+            dataset_dict["merge_id"] = ""
 
     @staticmethod
     def _validate_datasets(script_dict, analysis_unit, valid=True):
@@ -84,7 +86,10 @@ def _validate_datasets(script_dict, analysis_unit, valid=True):
         for dataset_name in script_dict.keys():
             if dataset_name in all_datasets:
                 if script_dict[dataset_name]["prefix"] != "":
-                    if analysis_unit == "h" and script_dict[dataset_name]["analysis_unit"] == "p":
+                    if (
+                        analysis_unit == "h"
+                        and script_dict[dataset_name]["analysis_unit"] == "p"
+                    ):
                         invalid_list.append(dataset_name)
                     else:
                         valid_list.append(dataset_name)
@@ -99,6 +104,4 @@ def _validate_datasets(script_dict, analysis_unit, valid=True):
 
         if valid:
             return valid_list
-        else:
-            return invalid_list
-
+        return invalid_list
diff --git a/ddionrails/workspace/scripts/soep_datasets.py b/ddionrails/workspace/scripts/soep_datasets.py
--- a/ddionrails/workspace/scripts/soep_datasets.py
+++ b/ddionrails/workspace/scripts/soep_datasets.py
@@ -1,10 +1,4 @@
 #!/usr/bin/python3
-
-############################################################
-# File   : soep_datasets.py
-# Date   : 2020-02-04 / ISO 8601: YYYY-MM-DD
-# Author : mpahl
-############################################################
 """
 Example:
 # from soep_datasets import SoepDatasets
@@ -23,14 +17,23 @@
 
 """
 
+import json
+from os import path
+from typing import Dict, Union
+
+############################################################
+# File   : soep_datasets.py
+# Date   : 2020-02-04 / ISO 8601: YYYY-MM-DD
+# Author : mpahl
+############################################################
+
 ############################################################
 # imports
 
-import json
-from os import path
 
 ############################################################
 
+
 class SoepDatasetsJsonLoader:
     """class SoepDatasetsJsonLoader -
     utility class to load SoepDatasets from a json file
@@ -44,15 +47,17 @@ def load_json_data(json_file):
         _json_data = {}
 
         if path.exists(json_file):
-            with open(json_file) as f:
-                _json_data = json.load(f)
+            with open(json_file) as file:
+                _json_data = json.load(file)
 
         return _json_data
 
+
 class SoepDataset:
     """
     TODO: Describe this module and methodes ..
     """
+
     def __init__(self):
         self.sub_path = ""
         self.syear = ""
@@ -60,6 +65,7 @@ def __init__(self):
         self.analyse_unit = ""
         self.is_matchable = ""
 
+
 class SoepDatasets:
     """
     TODO: Describe this module and methodes ..
@@ -67,15 +73,21 @@ class SoepDatasets:
 
     # Default path to json file
     # _default_json_datasets_file = "../datafiles.json" # bash tests
-    _default_json_datasets_file = "ddionrails/workspace/datafiles.json" # local tests
+    _default_json_datasets_file = "ddionrails/workspace/datafiles.json"  # local tests
     # _default_json_datasets_file = "ddionrails/workspace/datafiles_with_long.json"
 
     ########################################################
     def __init__(self, json_file=_default_json_datasets_file):
         self.data = SoepDatasetsJsonLoader().load_json_data(json_file)
 
-    def get_dict(self, dataset_name):
-        return self.data[dataset_name]
+    def get_dict(self, dataset_name) -> Union[Dict[str, str], Dict]:
+        """ Return dataset information
+
+        Returns:
+            The dataset information from the datafiles.json file
+            or an empty dictionairy if dataset_name is not present in JSON file.
+        """
+        return self.data.get(dataset_name, dict())
 
     def get_dataset(self, dataset_name):
         dataset_property = self.get_dict(dataset_name)
@@ -90,9 +102,10 @@ def get_dataset(self, dataset_name):
 
         return _dataset
 
+
 if __name__ == "__main__":
 
     TEST_DATASET = "bap"
     x = SoepDatasets()
     data_dict = x.get_dict(TEST_DATASET)
-    data_object = x.get_dataset(TEST_DATASET)
\ No newline at end of file
+    data_object = x.get_dataset(TEST_DATASET)


Test patch: