#NEWLINE# def make_dataset( name ) : #NEWLINE# #INDENT# '\n    Create the dataset defined with name in SETTINGS json. A dataset definition\n    in SETTINGS takes the form:\n        { "DatasetName": {\n            "input_data": ["NamesOfAnyDatasetsRequiredAsInput", ...],\n            "transforms": [[ "TrainsformName1", { "TransformArg1": Value1, \n                                                  "TransformArg2": Value2, ... }].\n                            ... ]\n          }\n        }\n    "DatasetName" is the name that will be used for the dataset throughout the model\n    this name is used for accessing configuration information, and when passing the\n    dataset as input into other dataset definitions or model definitions\n    \n    "input_data" is a list of other dataset names that are required as input for\n    the creation of the dataset\n    \n    "transforms" is a list of lists. Each sub-list must have length 2 and contain\n    a transform name, which is a valid name of a function defined in data_transforms.py\n    and a dict of arguments required to pass into the transform. All transforms defined\n    in data_transforms.py must have the structure:\n        \n        def transform_name(input_data, **args):\n            ...\n            \n    The "transforms" list allows for chaining of transforms for creating complex\n    datasets. The input for the first transform in the chain is always defined in \n    "input_data", the next transform in the chain takes the output of the previous\n    transform as input_data. \n    ' #NEWLINE# cfgs = config . dataset_configs ( name ) #NEWLINE# data = [ load_dataset ( ds ) for ds in cfgs [ 'input_data' ] ] #NEWLINE# if ( len ( data ) == 1 ) : #NEWLINE# #INDENT# data = data [ 0 ] #NEWLINE# #UNINDENT# logger . info ( ( 'Creating dataset %s' % name ) ) #NEWLINE# for ( tname , args ) in cfgs [ 'transforms' ] : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# transform = getattr ( data_transforms , tname ) #NEWLINE# #UNINDENT# except AttributeError : #NEWLINE# #INDENT# raise AttributeError ( ( 'Unable to find transform                                    %s in data_transforms.py' % tname ) ) #NEWLINE# #UNINDENT# logger . info ( ( 'Applying %s on %s' % ( tname , name ) ) ) #NEWLINE# data = transform ( tname , ** args ) #NEWLINE# #UNINDENT# return data