Repository: microsoft/nni

Issue:
ERROR (nni.msg_dispatcher_base/Thread-1) index 823 is out of bounds for axis 0 with size 790
nni Environment:

nni version: latest
nni mode local
OS: windows 10 64bit
python version:3.6
is conda or virtualenv used?: yes
is running in docker?:no


when i run my experiment on nni，after 20+ trial  it's error 
and i search for the logs, i see the message below:

[01/09/2020, 05:55:34 PM] DEBUG (nni.msg_dispatcher_base/Thread-2) process_command: command: [CommandType.TrialEnd], data: [OrderedDict([('trial_job_id', 'gM4jc'), ('event', 'SUCCEEDED'), ('hyper_params', '{"parameter_id": 23, "parameter_source": "algorithm", "parameters": {"num_leaves": 170, "max_bin": 833, "min_data_in_leaf": 160, "learning_rate": 0.022876755463143227, "lambda_l1": 0.2, "feature_fraction": 0.7524212365076838, "bagging_fraction": 0.9063591484877684, "bagging_freq": 10}, "parameter_index": 0}')])]
[01/09/2020, 05:55:34 PM] ERROR (nni.msg_dispatcher_base/Thread-1) index 823 is out of bounds for axis 0 with size 790
Traceback (most recent call last):
  File "E:\miniconda\lib\site-packages\nni\msg_dispatcher_base.py", line 90, in command_queue_worker
    self.process_command(command, data)
  File "E:\miniconda\lib\site-packages\nni\msg_dispatcher_base.py", line 149, in process_command
    command_handlers[command](data)
  File "E:\miniconda\lib\site-packages\nni\msg_dispatcher.py", line 101, in handle_request_trial_jobs
    params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback)
  File "E:\miniconda\lib\site-packages\nni\tuner.py", line 132, in generate_multiple_parameters
    res = self.generate_parameters(parameter_id, **kwargs)
  File "E:\miniconda\lib\site-packages\nni\hyperopt_tuner\hyperopt_tuner.py", line 267, in generate_parameters
    total_params = self.get_suggestion(random_search=False)
  File "E:\miniconda\lib\site-packages\nni\hyperopt_tuner\hyperopt_tuner.py", line 444, in get_suggestion
    new_trials = algorithm(new_ids, rval.domain, trials, random_state)
  File "E:\miniconda\lib\site-packages\hyperopt\tpe.py", line 900, in suggest
    print_node_on_error=False)
  File "E:\miniconda\lib\site-packages\hyperopt\pyll\base.py", line 913, in rec_eval
    rval = scope._impls[node.name](*args, **kwargs)
  File "E:\miniconda\lib\site-packages\hyperopt\tpe.py", line 61, in categorical_lpdf
    return np.log(np.asarray(p)[sample])
IndexError: index 823 is out of bounds for axis 0 with size 790
[01/09/2020, 05:55:35 PM] DEBUG (nni.protocol/MainThread) Received command, header: [b'PI000000']
[01/09/2020, 05:55:35 PM] DEBUG (nni.protocol/MainThread) Received command, data: []
[01/09/2020, 05:55:35 PM] INFO (nni.msg_dispatcher_base/MainThread) Dispatcher exiting...
[01/09/2020, 05:55:37 PM] INFO (nni.msg_dispatcher_base/MainThread) Terminated by NNI manager

the search space is this:
{
    "num_leaves":{"_type":"randint","_value":[60,140]},
    "max_bin":{"_type":"randint","_value":[300,600]},
    "min_data_in_leaf":{"_type":"randint","_value":[80, 180]},
    "learning_rate":{"_type":"uniform","_value":[0.01, 0.03]},
    "lambda_l1":{"_type":"choice","_value":[0.2,0.4,0.5,0.7,0.8]},
    "feature_fraction":{"_type":"uniform","_value":[0.6, 1.0]},
    "bagging_fraction":{"_type":"uniform","_value":[0.6, 1.0]},
    "bagging_freq":{"_type":"choice","_value":[1, 2, 4, 8, 10]}
}

the config_windows is this 
authorName: default
experimentName: example_auto-gbdt
trialConcurrency: 1
maxExecDuration: 30h
maxTrialNum: 250
#choice: local, remote, pai
trainingServicePlatform: local
searchSpacePath: search_space.json
#choice: true, false
useAnnotation: false
tuner:
  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
  #SMAC (SMAC should be installed through nnictl)
  builtinTunerName: TPE
  classArgs:
    #choice: maximize, minimize
    optimize_mode: maximize
trial:
  command: python find_para.py
  codeDir: e:/match/
  gpuNum: 0


the code is this 

import pandas as pd
import lightgbm as lgb
import logging
from sklearn.metrics import f1_score

LOG = logging.getLogger('auto-gbdt')


def under_fit(df, q):
    df_0 = df[df.label == 0]
    df_1 = df[df.label != 0]
    print(len(df_1))
    df_0 = df_0.sample(frac=q, random_state=1)
    print(len(df_0))
    df = pd.concat([df_0, df_1], axis=0)
    return df


from sklearn.model_selection import train_test_split


def get_default_parameters():
    params = {
        "boosting": "gbdt",
        'objective': 'multiclass',
        'metric': 'multi_logloss',
        'num_class': 5,
        'num_leaves': 80,
        'learning_rate': 0.02,
        'feature_fraction': 0.9,
        'bagging_fraction': 0.8,
        'bagging_freq': 5,
        "max_bin": 256,
        "lambda_l1": 0.1,
        'verbose': -1
    }
    return params

def load_data():
     ----read data from csv and feature works---
     return lgb_train, lgb_eval, test_X, test_y

import nni


def run(lgb_train, lgb_eval, params, test_X, test_y):
    print('Start training...')
    params['num_leaves'] = int(params['num_leaves'])
    # train
    gbm = lgb.train(params,
                    lgb_train,
                    num_boost_round=1500,
                    valid_sets=lgb_eval,
                    categorical_feature='auto',
                    early_stopping_rounds=5,
                    )
    print('Start predicting...')

    # predict
    y_pred = gbm.predict(test_X, num_iteration=gbm.best_iteration)
    y_pred = [list(x).index(max(x)) for x in y_pred]
    # eval
    f1 = f1_score(test_y, y_pred, average='micro')
    print('The f1 of prediction is:', f1)
    nni.report_final_result(f1)


if __name__ == '__main__':
    lgb_train, lgb_eval, test_X, test_y = load_data()
    try:
        # get parameters from tuner
        RECEIVED_PARAMS = nni.get_next_parameter()
        LOG.debug(RECEIVED_PARAMS)
        PARAMS = get_default_parameters()
        PARAMS.update(RECEIVED_PARAMS)
        LOG.debug(PARAMS)
        # train
        run(lgb_train, lgb_eval, PARAMS, test_X, test_y)
    except Exception as exception:
        LOG.exception(exception)
        raise

i don't know how to fix it . some one can help? thanks


Discussion:
and by the way 
i set the max_bin "max_bin":{"_type":"randint","_value":[300,600]},
but in the log i see this "max_bin": 833,  why its >600?

and nnimanager.log is here

[2020-1-9 17:55:33] INFO [ 'Trial job gM4jc status changed from RUNNING to SUCCEEDED' ]
[2020-1-9 17:55:33] DEBUG [ 'storeTrialJobEvent: event: SUCCEEDED, data: undefined, jobDetail: {"id":"gM4jc","status":"SUCCEEDED","submitTime":1578563503303,"workingDirectory":"C:\\\\Users\\\\hp680\\\\nni\\\\experiments\\\\OHl57N2z\\\\trials\\\\gM4jc","form":{"sequenceId":23,"hyperParameters":{"value":"{\\"parameter_id\\": 23, \\"parameter_source\\": \\"algorithm\\", \\"parameters\\": {\\"num_leaves\\": 170, \\"max_bin\\": 833, \\"min_data_in_leaf\\": 160, \\"learning_rate\\": 0.022876755463143227, \\"lambda_l1\\": 0.2, \\"feature_fraction\\": 0.7524212365076838, \\"bagging_fraction\\": 0.9063591484877684, \\"bagging_freq\\": 10}, \\"parameter_index\\": 0}","index":0}},"url":"file://localhost:C:\\\\Users\\\\hp680\\\\nni\\\\experiments\\\\OHl57N2z\\\\trials\\\\gM4jc","gpuIndices":[],"startTime":1578563505772,"pid":21572,"endTime":1578563730048}' ]
[2020-1-9 17:55:34] DEBUG [ 'ipcInterface command type: [EN], content:[{"trial_job_id":"gM4jc","event":"SUCCEEDED","hyper_params":"{\\"parameter_id\\": 23, \\"parameter_source\\": \\"algorithm\\", \\"parameters\\": {\\"num_leaves\\": 170, \\"max_bin\\": 833, \\"min_data_in_leaf\\": 160, \\"learning_rate\\": 0.022876755463143227, \\"lambda_l1\\": 0.2, \\"feature_fraction\\": 0.7524212365076838, \\"bagging_fraction\\": 0.9063591484877684, \\"bagging_freq\\": 10}, \\"parameter_index\\": 0}"}]' ]
[2020-1-9 17:55:34] DEBUG [ 'ipcInterface command type: [GE], content:[1]' ]
[2020-1-9 17:55:35] DEBUG [ 'ipcInterface command type: [PI], content:[]' ]
[2020-1-9 17:55:40] DEBUG [ 'ipcInterface command type: [PI], content:[]' ]
[2020-1-9 17:55:40] ERROR [ 'Dispatcher error: This socket has been ended by the other party' ]
[2020-1-9 17:55:40] ERROR [ 'Error: Dispatcher stream error, tuner may have crashed.\n    at EventEmitter.dispatcher.onError (e:\\miniconda\\nni\\core\\nnimanager.js:505:32)\n    at EventEmitter.emit (events.js:182:13)\n    at Socket.IpcInterface.outgoingStream.on (e:\\miniconda\\nni\\core\\ipcInterface.js:42:72)\n    at Socket.emit (events.js:182:13)\n    at Socket.writeAfterFIN [as write] (net.js:405:8)\n    at IpcInterface.sendCommand (e:\\miniconda\\nni\\core\\ipcInterface.js:49:38)\n    at NNIManager.pingDispatcher (e:\\miniconda\\nni\\core\\nnimanager.js:354:29)' ]
[2020-1-9 17:55:40] INFO [ 'Change NNIManager status from: RUNNING to: ERROR' ]
[2020-1-9 17:55:40] WARNING [ 'Commands jammed in buffer!' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /experiment: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /check-status: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /trial-jobs: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /metric-data: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'getTrialJobsByReplayEvents begin' ]
[2020-1-9 17:55:40] DEBUG [ 'getTrialJobsByReplayEvents done' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /experiment: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /check-status: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /trial-jobs: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'GET: /metric-data: body:\n{}' ]
[2020-1-9 17:55:40] DEBUG [ 'getTrialJobsByReplayEvents begin' ]
[2020-1-9 17:55:40] DEBUG [ 'getTrialJobsByReplayEvents done' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /experiment: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /check-status: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /trial-jobs: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /metric-data: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'getTrialJobsByReplayEvents begin' ]
[2020-1-9 18:08:30] DEBUG [ 'getTrialJobsByReplayEvents done' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /experiment: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /trial-jobs: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /metric-data: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'GET: /version: body:\n{}' ]
[2020-1-9 18:08:30] DEBUG [ 'getTrialJobsByReplayEvents begin' ]
[2020-1-9 18:08:30] DEBUG [ 'getTrialJobsByReplayEvents done' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /experiment: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /check-status: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /trial-jobs: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /metric-data: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'getTrialJobsByReplayEvents begin' ]
[2020-1-9 18:08:40] DEBUG [ 'getTrialJobsByReplayEvents done' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /experiment: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /check-status: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /trial-jobs: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'GET: /metric-data: body:\n{}' ]
[2020-1-9 18:08:40] DEBUG [ 'getTrialJobsByReplayEvents begin' ]
[2020-1-9 18:08:40] DEBUG [ 'getTrialJobsByReplayEvents done' ]


[dispatcher.log](https://github.com/microsoft/nni/files/4039999/dispatcher.log)

[nnimanager.log](https://github.com/microsoft/nni/files/4040015/nnimanager.log)


Hello @lastrei .

Based on the [tutorial](https://github.com/microsoft/nni/blob/master/docs/en_US/Tutorial/SearchSpaceSpec.md), I think you can use `quniform` to instead `randint`. Because TPE treats integers from lower to upper as unordered ones.

![image](https://user-images.githubusercontent.com/6280746/72120832-35434980-3394-11ea-8676-6a9dcec49a4f.png)

As for why the `max_bin` is out of the boundary, I don't have a certain answer now. But could you check the hyperopt version that you are using? thanks.

@xuehui1991 
thanks for reply
hyperopt.__version__
'0.2'
the index out of bound  error  because of randint? i'll use quniform to have a try 
thank you so much
Thanks @lastrei for reporting this issue. Currently, NNI only supports hyperopt 0.1.2. Could you try 0.1.2 to check whether the problem still exists. Thank you.
> Thanks @lastrei for reporting this issue. Currently, NNI only supports hyperopt 0.1.2. Could you try 0.1.2 to check whether the problem still exists. Thank you.

ok i will change hyperopt to 0.1.2 
and i use quniform to instead randint  the out of bound error is not appear ..but i don't know 
why randint can cause out of bound?

Hello @lastrei. I reproduce your error using your search space. I think it a bug happened in the period of code refinement. NNI should fix it asap.

Resolving patch:
diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py
--- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py
+++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py
@@ -102,6 +102,9 @@ def _parse_randint(self, param_value):
         """
         Parse type of randint parameter and return a list
         """
+        if param_value[0] >= param_value[1]:
+            raise ValueError("Randint should contain at least 1 candidate, but [%s, %s) contains none.",
+                             param_value[0], param_value[1])
         return np.arange(param_value[0], param_value[1]).tolist()
 
     def _expand_parameters(self, para):
diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py
--- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py
+++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py
@@ -118,6 +118,8 @@ def json2vals(in_x, vals, out_y, name=NodeType.ROOT):
                           vals[NodeType.VALUE],
                           out_y,
                           name=name + '[%d]' % _index)
+            if _type == 'randint':
+                out_y[name] -= in_x[NodeType.VALUE][0]
         else:
             for key in in_x.keys():
                 json2vals(in_x[key], vals[key], out_y,


Test patch:
diff --git a/src/sdk/pynni/tests/assets/search_space.json b/src/sdk/pynni/tests/assets/search_space.json
--- a/src/sdk/pynni/tests/assets/search_space.json
+++ b/src/sdk/pynni/tests/assets/search_space.json
@@ -1,8 +1,7 @@
 {
   "choice_str": {
     "_type": "choice",
-    "_value": ["cat", "dog", "elephant", "cow", "sheep", "panda"],
-    "fail": ["metis", "gp"]
+    "_value": ["cat", "dog", "elephant", "cow", "sheep", "panda"]
   },
   "choice_int": {
     "_type": "choice",
@@ -10,8 +9,7 @@
   },
   "choice_mixed": {
     "_type": "choice",
-    "_value": [0.3, "cat", 1, null],
-    "fail": ["metis", "gp"]
+    "_value": [0.3, "cat", 1, null]
   },
   "choice_float": {
     "_type": "choice",
diff --git a/src/sdk/pynni/tests/test_builtin_tuners.py b/src/sdk/pynni/tests/test_builtin_tuners.py
--- a/src/sdk/pynni/tests/test_builtin_tuners.py
+++ b/src/sdk/pynni/tests/test_builtin_tuners.py
@@ -5,6 +5,7 @@
 import json
 import logging
 import os
+import random
 import shutil
 import sys
 from unittest import TestCase, main
@@ -15,6 +16,7 @@
 from nni.gridsearch_tuner.gridsearch_tuner import GridSearchTuner
 from nni.hyperopt_tuner.hyperopt_tuner import HyperoptTuner
 from nni.metis_tuner.metis_tuner import MetisTuner
+
 try:
     from nni.smac_tuner.smac_tuner import SMACTuner
 except ImportError:
@@ -34,20 +36,28 @@ class BuiltinTunersTestCase(TestCase):
         - [X] generate_multiple_parameters
         - [ ] import_data
         - [ ] trial_end
-        - [ ] receive_trial_result
+        - [x] receive_trial_result
     """
 
+    def setUp(self):
+        self.test_round = 3
+        self.params_each_round = 50
+        self.exhaustive = False
+
     def search_space_test_one(self, tuner_factory, search_space):
         tuner = tuner_factory()
         self.assertIsInstance(tuner, Tuner)
         tuner.update_search_space(search_space)
 
-        parameters = tuner.generate_multiple_parameters(list(range(0, 50)))
-        logger.info(parameters)
-        self.check_range(parameters, search_space)
-        if not parameters:  # TODO: not strict
-            raise ValueError("No parameters generated")
-        return parameters
+        for i in range(self.test_round):
+            parameters = tuner.generate_multiple_parameters(list(range(i * self.params_each_round,
+                                                                       (i + 1) * self.params_each_round)))
+            logger.debug(parameters)
+            self.check_range(parameters, search_space)
+            for k in range(min(len(parameters), self.params_each_round)):
+                tuner.receive_trial_result(self.params_each_round * i + k, parameters[k], random.uniform(-100, 100))
+            if not parameters and not self.exhaustive:
+                raise ValueError("No parameters generated")
 
     def check_range(self, generated_params, search_space):
         EPS = 1E-6
@@ -91,7 +101,8 @@ def check_range(self, generated_params, search_space):
                     for layer_name in item["_value"].keys():
                         self.assertIn(v[layer_name]["chosen_layer"], item["layer_choice"])
 
-    def search_space_test_all(self, tuner_factory, supported_types=None, ignore_types=None):
+    def search_space_test_all(self, tuner_factory, supported_types=None, ignore_types=None, fail_types=None):
+        # Three types: 1. supported; 2. ignore; 3. fail.
         # NOTE(yuge): ignore types
         # Supported types are listed in the table. They are meant to be supported and should be correct.
         # Other than those, all the rest are "unsupported", which are expected to produce ridiculous results
@@ -103,16 +114,18 @@ def search_space_test_all(self, tuner_factory, supported_types=None, ignore_type
         if supported_types is None:
             supported_types = ["choice", "randint", "uniform", "quniform", "loguniform", "qloguniform",
                                "normal", "qnormal", "lognormal", "qlognormal"]
+        if fail_types is None:
+            fail_types = []
+        if ignore_types is None:
+            ignore_types = []
         full_supported_search_space = dict()
         for single in search_space_all:
-            single_keyword = single.split("_")
             space = search_space_all[single]
-            expected_fail = not any([t in single_keyword for t in supported_types]) or "fail" in single_keyword
-            if ignore_types is not None and any([t in ignore_types for t in single_keyword]):
+            if any(single.startswith(t) for t in ignore_types):
                 continue
-            if "fail" in space:
-                if self._testMethodName.split("_", 1)[1] in space.pop("fail"):
-                    expected_fail = True
+            expected_fail = not any(single.startswith(t) for t in supported_types) or \
+                            any(single.startswith(t) for t in fail_types) or \
+                            "fail" in single  # name contains fail (fail on all)
             single_search_space = {single: space}
             if not expected_fail:
                 # supports this key
@@ -129,11 +142,14 @@ def search_space_test_all(self, tuner_factory, supported_types=None, ignore_type
             self.search_space_test_one(tuner_factory, full_supported_search_space)
 
     def test_grid_search(self):
+        self.exhaustive = True
         self.search_space_test_all(lambda: GridSearchTuner(),
                                    supported_types=["choice", "randint", "quniform"])
 
     def test_tpe(self):
-        self.search_space_test_all(lambda: HyperoptTuner("tpe"))
+        self.search_space_test_all(lambda: HyperoptTuner("tpe"),
+                                   ignore_types=["uniform_equal", "qloguniform_equal", "loguniform_equal", "quniform_clip_2"])
+        # NOTE: types are ignored because `tpe.py line 465, in adaptive_parzen_normal assert prior_sigma > 0`
 
     def test_random_search(self):
         self.search_space_test_all(lambda: HyperoptTuner("random_search"))
@@ -148,6 +164,7 @@ def test_smac(self):
                                    supported_types=["choice", "randint", "uniform", "quniform", "loguniform"])
 
     def test_batch(self):
+        self.exhaustive = True
         self.search_space_test_all(lambda: BatchTuner(),
                                    supported_types=["choice"])
 
@@ -156,14 +173,18 @@ def test_evolution(self):
         self.search_space_test_all(lambda: EvolutionTuner(population_size=100))
 
     def test_gp(self):
+        self.test_round = 1  # NOTE: GP tuner got hanged for multiple testing round
         self.search_space_test_all(lambda: GPTuner(),
                                    supported_types=["choice", "randint", "uniform", "quniform", "loguniform",
                                                     "qloguniform"],
-                                   ignore_types=["normal", "lognormal", "qnormal", "qlognormal"])
+                                   ignore_types=["normal", "lognormal", "qnormal", "qlognormal"],
+                                   fail_types=["choice_str", "choice_mixed"])
 
     def test_metis(self):
+        self.test_round = 1  # NOTE: Metis tuner got hanged for multiple testing round
         self.search_space_test_all(lambda: MetisTuner(),
-                                   supported_types=["choice", "randint", "uniform", "quniform"])
+                                   supported_types=["choice", "randint", "uniform", "quniform"],
+                                   fail_types=["choice_str", "choice_mixed"])
 
     def test_networkmorphism(self):
         pass