GitHub PullRequestReviewCommentEvent at 2014-07-22 21:13:39
Repository: scikit-learn
Actor: arjoly
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/15256710","id":15256710,"diff_hunk":"@@ -0,0 +1,768 @@\n+\"\"\"Extreme Learning Machines\n+\"\"\"\n+\n+# Author: Issam H. Laradji <issam.laradji@gmail.com>\n+# Licence: BSD 3 clause\n+\n+from abc import ABCMeta, abstractmethod\n+\n+import numpy as np\n+\n+from ..base import BaseEstimator, ClassifierMixin, RegressorMixin\n+from ..externals import six\n+from ..preprocessing import LabelBinarizer\n+from ..metrics.pairwise import linear_kernel, polynomial_kernel, rbf_kernel\n+from ..metrics.pairwise import sigmoid_kernel\n+from ..linear_model import ridge\n+from ..utils import gen_even_slices\n+from ..utils import atleast2d_or_csr, check_random_state, column_or_1d\n+from ..utils.extmath import safe_sparse_dot\n+from ..utils.fixes import expit as logistic_sigmoid\n+\n+\n+def _tanh(X):\n+    \"\"\"Compute the hyperbolic tan function.\"\"\"\n+    return np.tanh(X, X)\n+\n+\n+def _relu(X):\n+    \"\"\"Compute the rectified linear unit function.\"\"\"\n+    # print X.shape\n+    tmp = 1 + np.exp(X, X)\n+    return np.log(tmp, tmp)\n+\n+\n+def _softmax(Z):\n+    \"\"\"Compute the K-way softmax function. \"\"\"\n+    Z = np.exp(Z - Z.max(axis=1)[:, np.newaxis])\n+    Z /= Z.sum(axis=1)[:, np.newaxis]\n+\n+    return Z\n+\n+\n+def _square_error(y, y_pred, n_samples):\n+    \"\"\"Compute the square error.\"\"\"\n+    return (np.sum(y - y_pred) ** 2) / (2 * n_samples)\n+\n+\n+class BaseELM(six.with_metaclass(ABCMeta, BaseEstimator)):\n+    \"\"\"Base class for ELM classification and regression.\n+\n+    Warning: This class should not be used directly.\n+    Use derived classes instead.\n+    \"\"\"\n+    _activation_functions = {\n+        'tanh': _tanh,\n+        'logistic': logistic_sigmoid,\n+        'relu': _relu\n+    }\n+\n+    @abstractmethod\n+    def __init__(self, n_hidden, activation, algorithm, kernel, C, degree,\n+                 gamma, coef0, class_weight, weight_scale, batch_size, verbose,\n+                 random_state):\n+        self.C = C\n+        self.activation = activation\n+        self.algorithm = algorithm\n+        self.kernel = kernel\n+        self.degree = degree\n+        self.gamma = gamma\n+        self.coef0 = coef0\n+        self.class_weight = class_weight\n+        self.weight_scale = weight_scale\n+        self.batch_size = batch_size\n+        self.n_hidden = n_hidden\n+        self.verbose = verbose\n+        self.random_state = random_state\n+\n+        self.coef_hidden_ = None\n+        self.coef_output_ = None\n+\n+    def _validate_params(self):\n+        \"\"\"Validate input params.\"\"\"\n+        if self.n_hidden <= 0:\n+            raise ValueError(\"n_hidden must be greater or equal zero\")\n+        if self.C <= 0.0:\n+            raise ValueError(\"C must be > 0\")\n+\n+        if self.activation not in self._activation_functions:\n+            raise ValueError(\"The activation %s\"\n+                             \" is not supported. \" % self.activation)\n+\n+        if self.algorithm not in ['standard', 'sequential']:\n+            raise ValueError(\"The algorithm %s\"\n+                             \" is not supported. \" % self.algorithm)\n+\n+        if self.kernel not in ['random', 'linear', 'poly', 'rbf', 'sigmoid']:\n+            raise ValueError(\"The kernel %s\"\n+                             \" is not supported. \" % self.kernel)\n+\n+        if self.algorithm is not 'standard' and self.class_weight is not None:\n+                raise NotImplementedError(\"class_weight is only supported \"\n+                                          \"when algorithm='standard'.\")\n+\n+    def _init_param(self, X):\n+        \"\"\"Set initial parameters.\"\"\"\n+        self._activation_func = self._activation_functions[self.activation]\n+\n+        if (self.kernel in ['poly', 'rbf', 'sigmoid']) and (self.gamma == 0):\n+            # if custom gamma is not provided ...\n+            self.gamma = 1.0 / self._n_features\n+\n+        if self.kernel != 'random':\n+            self._X_train = X\n+\n+    def _scaled_weight_init(self, fan_in, fan_out):\n+        \"\"\"Scale the initial, random parameters for a specific layer.\"\"\"\n+        if self.weight_scale == 'auto':\n+            if self.activation == 'tanh':\n+                interval = np.sqrt(6. / (fan_in + fan_out))\n+            elif self.activation == 'logistic':\n+                interval = 4. * np.sqrt(6. / (fan_in + fan_out))\n+            else:\n+                interval = np.sqrt(1. / (fan_in))\n+        else:\n+            interval = self.weight_scale\n+\n+        return interval\n+\n+    def _init_hidden_weights(self):\n+        \"\"\"Initialize coef and intercept parameters for the hidden layer.\"\"\"\n+        rng = check_random_state(self.random_state)\n+        fan_in, fan_out = self._n_features, self.n_hidden\n+\n+        interval = self._scaled_weight_init(fan_in, fan_out)\n+\n+        coef = rng.uniform(-interval, interval, (fan_in, fan_out))\n+        intercept = rng.uniform(-interval, interval, (fan_out))\n+\n+        self.coef_hidden_ = coef\n+        self.intercept_hidden_ = intercept\n+\n+    def _assign_weights(self, y):\n+        \"\"\"Assign weight to each sample based on its class.\"\"\"\n+        n_samples = y.shape[0]\n+        diagonals = np.zeros(n_samples)\n+\n+        y_original = self._lbin.inverse_transform(y)\n+\n+        if self.class_weight == 'auto':\n+            # assign weights as, w[i] = 0.618 / (n_samples of class i)\n+            class_weight = {}\n+\n+            for class_ in np.unique(y_original):\n+                class_size = len(np.where(y_original == class_)[0])\n+                class_weight[class_] = 0.618 / class_size\n+        else:\n+            class_weight = dict(self.class_weight)\n+\n+        for class_ in self.classes_:\n+            indices = np.where(y_original == class_)[0]\n+            if class_ in class_weight.keys():\n+                diagonals[indices] = class_weight[class_]\n+            else:\n+                diagonals[indices] = 1\n+\n+        return diagonals\n+\n+    def _get_hidden_activations(self, X):\n+        \"\"\"Compute the hidden activations using the set kernel.\n+        \"\"\"\n+        if self.kernel == 'random':\n+            if self.coef_hidden_ is None:\n+                self._init_hidden_weights()\n+\n+            A = safe_sparse_dot(X, self.coef_hidden_)\n+            A += self.intercept_hidden_\n+            H = self._activation_func(A)\n+\n+        elif self.kernel == 'linear':\n+            H = linear_kernel(X, self._X_train)\n+\n+        elif self.kernel == 'poly':\n+            H = polynomial_kernel(X, self._X_train, self.degree, self.gamma,\n+                                  self.coef0)\n+\n+        elif self.kernel == 'rbf':\n+            H = rbf_kernel(X, self._X_train, self.gamma)\n+\n+        elif self.kernel == 'sigmoid':\n+            H = sigmoid_kernel(X, self._X_train, self.gamma, self.coef0)\n+\n+        return H\n+\n+    def _solve_lsqr(self, X, y):\n+        \"\"\"Compute the least-square solutions for the whole dataset.\"\"\"\n+        H = self._get_hidden_activations(X)\n+\n+        # compute output coefficients using ridge_regression\n+        if self.class_weight is not None:\n+            W = self._assign_weights(y)\n+        else:\n+            W = None\n+\n+        self.coef_output_ = ridge.ridge_regression(H, y, 1.0 / self.C,\n+                                                   sample_weight=W).T\n+\n+    def _sequential_solve_procedure(self, X_batch, y_batch):\n+        \"\"\"Compute the least-square solutions for one batch.\"\"\"\n+        H = self._get_hidden_activations(X_batch)\n+\n+        if self._K is None:\n+            # initialize K and coef_output_\n+            self.coef_output_ = np.zeros((self.n_hidden, self.n_outputs_))\n+            self._K = safe_sparse_dot(H.T, H)\n+            y_ = safe_sparse_dot(H.T, y_batch)\n+\n+        else:\n+            self._K += safe_sparse_dot(H.T, H)\n+            H_updated = safe_sparse_dot(H, self.coef_output_)\n+            y_ = safe_sparse_dot(H.T, (y_batch - H_updated))\n+\n+        self.coef_output_ += ridge.ridge_regression(self._K, y_,\n+                                                    1.0 / self.C).T\n+\n+    def fit(self, X, y):\n+        \"\"\"Fit the model to the data X and target y.\n+\n+        Parameters\n+        ----------\n+        X : {array-like, sparse matrix}, shape (n_samples, n_features)\n+            Training data, where n_samples is the number of samples\n+            and n_features is the number of features.\n+\n+        y : array-like, shape (n_samples,)\n+            Target values.\n+\n+        Returns\n+        -------\n+        self : returns an instance of self.\n+        \"\"\"\n+        X = atleast2d_or_csr(X)\n+\n+        self._validate_params()\n+\n+        n_samples, self._n_features = X.shape\n+        self.n_outputs_ = y.shape[1]\n+        self._init_param(X)\n+\n+        if self.algorithm == 'standard':\n+            # compute the least-square solutions for the whole dataset\n+            self._solve_lsqr(X, y)\n+\n+        elif self.algorithm == 'sequential':\n+            # compute the least-square solutions in batches\n+            batch_size = np.clip(self.batch_size, 0, n_samples)\n+            n_batches = n_samples // batch_size\n+            batch_slices = list(gen_even_slices(n_batches * batch_size,\n+                                                n_batches))\n+            self._K = None\n+\n+            for batch, batch_slice in enumerate(batch_slices):\n+                self._sequential_solve_procedure(X[batch_slice],\n+                                                 y[batch_slice])\n+\n+                if self.verbose:\n+                    # compute training square error\n+                    y_pred = self.decision_function(X[batch_slice])\n+                    cost = _square_error(y[batch_slice], y_pred, batch_size)\n+\n+                    print(\"Training square error for batch %d = %f\" % (batch,\n+                                                                       cost))\n+\n+        if self.verbose:\n+            # compute training square error\n+            y_pred = self.decision_function(X)\n+            cost = _square_error(y, y_pred, n_samples)\n+\n+            print(\"Training square error for the dataset = %f\" % cost)\n+\n+        return self\n+\n+    def decision_function(self, X):\n+        \"\"\"Predict using the trained model\n+\n+        Parameters\n+        ----------\n+        X : {array-like, sparse matrix}, shape (n_samples, n_features)\n+            Data, where n_samples is the number of samples\n+            and n_features is the number of features.\n+\n+        Returns\n+        -------\n+        array, shape (n_samples,)\n+            Predicted target values per element in X.\n+        \"\"\"\n+        X = atleast2d_or_csr(X)\n+\n+        self.hidden_activations_ = self._get_hidden_activations(X)\n+        output = safe_sparse_dot(self.hidden_activations_, self.coef_output_)\n+\n+        return output\n+\n+    def partial_fit(self, X, y):\n+        \"\"\"Fit the model to the data X and target y.\n+\n+        Parameters\n+        ----------\n+        X : {array-like, sparse matrix}, shape (n_samples, n_features)\n+            Subset of training data.\n+\n+        y : array-like, shape (n_samples,)\n+            Subset of target values.\n+\n+        Returns\n+        -------\n+        self : returns an instance of self.\n+        \"\"\"\n+        X = atleast2d_or_csr(X)\n+\n+        self.n_outputs_ = y.shape[1]\n+\n+        n_samples, self._n_features = X.shape\n+        self._validate_params()\n+        self._init_param(X)\n+\n+        if self.coef_output_ is None:\n+            self._K = None\n+\n+        self._sequential_solve_procedure(X, y)\n+\n+        if self.verbose:\n+            # compute training square error\n+            y_pred = self.decision_function(X)\n+            cost = _square_error(y, y_pred, n_samples)\n+\n+            print(\"Training square error for the batch = %f\" % (cost))\n+\n+        return self\n+\n+\n+class ELMClassifier(BaseELM, ClassifierMixin):\n+    \"\"\"Extreme learning machines classifier.\n+\n+    The algorithm trains a single-hidden layer feedforward network by computing\n+    the hidden layer values using randomized parameters, then solving\n+    for the output weights using least-square solutions. For prediction,\n+    after computing the forward pass, the continuous output values pass\n+    through a gate function converting them to integers that represent classes.\n+\n+    This implementation works with data represented as dense and sparse numpy\n+    arrays of floating point values for the features.\n+\n+    Parameters\n+    ----------\n+    C : float, optional, default 1\n+        Regularization term.\n+\n+    class_weight : {dict, 'auto'}, optional\n+        Set the parameter C of class i to class_weight[i]*C for ELMClassifier.\n+        If not given, all classes are supposed to have weight one. The 'auto'\n+        mode uses the values of y to automatically adjust weights inversely\n+        proportional to class frequencies.\n+\n+    weight_scale : float or 'auto', default 'auto'\n+        Scales the weights that initialize the outgoing weights of the first\n+        hidden layer. The weight values will range between plus and minus an\n+        interval based on the uniform distribution. That interval\n+        is 1 / (n_features + n_hidden) if weight_scale='auto'; otherwise,\n+        the interval is the value given to weight_scale.\n+\n+    n_hidden: int, default 100\n+        The number of neurons in the hidden layer, it only applies to\n+        kernel='random'.\n+\n+    activation : {'logistic', 'tanh', 'relu'}, default 'tanh'\n+        Activation function for the hidden layer. It only applies to\n+        kernel='random'.\n+\n+         - 'logistic' for 1 / (1 + exp(x)).\n+\n+         - 'tanh' for the hyperbolic tangent.\n+\n+         - 'relu' for log(1 + exp(x))\n+\n+    algorithm : {'standard', 'sequential'}, default 'standard'\n+        The algorithm for computing least-square solutions.\n+        Defaults to 'sequential'\n+\n+        - 'standard' computes the least-square solutions using the\n+          whole matrix at once.\n+\n+        - 'sequential' computes the least-square solutions by training\n+          on the dataset in batches using a recursive least-square\n+          algorithm.\n+\n+    kernel : {'random', 'linear', 'poly', 'rbf', 'sigmoid'},\n+             optional, default 'random'\n+        Specifies the kernel type to be used in the algorithm.\n+\n+    degree : int, optional, default 3\n+        Degree of the polynomial kernel function 'poly'.\n+        Ignored by all other kernels.\n+\n+    gamma : float, optional, default 0.0\n+        Kernel coefficient for 'rbf', 'poly' and 'sigmoid'. If gamma is\n+        0.0 then 1/n_features will be used instead.\n+\n+    coef0 : float, optional (default=0.0)\n+        Independent term in kernel function. It only applies to\n+        'poly' and 'sigmoid'.\n+\n+    batch_size : int, optional, default 200\n+        Size of minibatches for the 'sequential' algoritm.\n+        Minibatches do not apply to the 'standard' ELM algorithm.\n+\n+    verbose : bool, optional, default False\n+        Whether to print training score to stdout.\n+\n+    random_state : int or RandomState, optional, default None\n+        State of or seed for random number generator.\n+\n+\n+    Attributes\n+    ----------\n+    `classes_` : array or list of array of shape = [n_classes]\n+        Class labels for each output.\n+\n+    `n_outputs_` : int,\n+        Number of output neurons.\n+\n+    References\n+    ----------\n+    Zong, Weiwei, Guang-Bin Huang, and Yiqiang Chen.\n+        \"Weighted extreme learning machine for imbalance learning.\"\n+        Neurocomputing 101 (2013): 229-242.\n+\n+    Liang, Nan-Ying, et al.\n+        \"A fast and accurate online sequential learning algorithm for\n+        feedforward networks.\" Neural Networks, IEEE Transactions on\n+        17.6 (2006): 1411-1423.\n+        http://www.ntu.edu.sg/home/egbhuang/pdf/OS-ELM-TNN.pdf\n+    \"\"\"\n+    def __init__(self, n_hidden=500, activation='tanh', algorithm='standard',\n+                 kernel='random', C=1, degree=3, gamma=0.0, coef0=0.0,\n+                 class_weight=None, weight_scale='auto',\n+                 batch_size=200, verbose=False, random_state=None):\n+\n+        super(ELMClassifier, self).__init__(n_hidden, activation, algorithm,\n+                                            kernel, C, degree, gamma, coef0,\n+                                            class_weight, weight_scale,\n+                                            batch_size, verbose, random_state)\n+\n+        self._lbin = LabelBinarizer(-1, 1)\n+        self.classes_ = None\n+\n+    def fit(self, X, y):\n+        \"\"\"Fit the model to the data X and target y.\n+\n+        Parameters\n+        ----------\n+        X : {array-like, sparse matrix}, shape (n_samples, n_features)\n+            Training data, where n_samples is the number of samples\n+            and n_features is the number of features.\n+\n+        y : array-like, shape (n_samples,)\n+            Target values.\n+\n+        Returns\n+        -------\n+        self : returns an instance of self.\n+        \"\"\"\n+        y = column_or_1d(y, warn=True)\n+        self.classes_ = np.unique(y)\n+        y = self._lbin.fit_transform(y)\n+\n+        super(ELMClassifier, self).fit(X, y)\n+\n+        return self\n+\n+    def predict(self, X):\n+        \"\"\"Predict using the extreme learning machines model\n+\n+        Parameters\n+        ----------\n+        X : {array-like, sparse matrix}, shape (n_samples, n_features)\n+            Data, where n_samples is the number of samples\n+            and n_features is the number of features.\n+\n+        Returns\n+        -------\n+        array, shape (n_samples,)\n+            Predicted target values per element in X.\n+        \"\"\"\n+        X = atleast2d_or_csr(X)\n+        scores = self.decision_function(X)\n+\n+        return self._lbin.inverse_transform(scores)\n+\n+    def predict_proba(self, X):\n+        \"\"\"Probability estimates.\n+\n+        Parameters\n+        ----------\n+        X : {array-like, sparse matrix}, shape (n_samples, n_features)\n+            Data, where n_samples is the number of samples\n+            and n_features is the number of features.\n+\n+        Returns\n+        -------\n+        array, shape (n_samples, n_outputs)\n+            Returns the probability of the sample for each class in the model,\n+            where classes are ordered as they are in `self.classes_`.\n+        \"\"\"\n+        scores = self.decision_function(X)\n+\n+        if len(self.classes_) == 2:\n+            scores = logistic_sigmoid(scores)\n+            return np.hstack([1 - scores, scores])\n+        else:\n+            return _softmax(scores)\n+\n+    def predict_log_proba(self, X):\n+        \"\"\"Return the log of probability estimates.\n+\n+        Parameters\n+        ----------\n+        X : array-like, shape (n_samples, n_features)\n+            Data, where n_samples is the number of samples\n+            and n_features is the number of features.\n+\n+        Returns\n+        -------\n+        T : array-like, shape (n_samples, n_outputs)\n+            Returns the log-probability of the sample for each class in the\n+            model, where classes are ordered as they are in\n+            `self.classes_`. Equivalent to log(predict_proba(X))\n+        \"\"\"\n+        tmp = self.predict_proba(X)\n+        return np.log(tmp, tmp)\n+\n+    def partial_fit(self, X, y, classes=None):\n+        \"\"\"Fit the model to the data X and target y.\n+\n+        Parameters\n+        ----------\n+        X : {array-like, sparse matrix}, shape (n_samples, n_features)\n+            Training data, where n_samples in the number of samples\n+            and n_features is the number of features.\n+\n+        classes : array, shape (n_classes)\n+            Classes across all calls to partial_fit.\n+            Can be obtained by via `np.unique(y_all)`, where y_all is the\n+            target vector of the entire dataset.\n+            This argument is required for the first call to partial_fit\n+            and can be omitted in the subsequent calls.\n+            Note that y doesn't need to contain all labels in `classes`.\n+\n+        y : array-like, shape (n_samples,)\n+            Subset of the target values.\n+\n+        Returns\n+        -------\n+        self : returns an instance of self.\n+        \"\"\"\n+        if self.algorithm != 'sequential':\n+            raise ValueError(\n+                \"only 'sequential' algorithm supports partial fit\")\n+\n+        if self.classes_ is None and classes is None:\n+            raise ValueError(\"classes must be passed on the first call \"\n+                             \"to partial_fit.\")\n+        elif self.classes_ is not None and classes is not None:\n+            if np.any(self.classes_ != np.unique(classes)):\n+                raise ValueError(\"`classes` is not the same as on last call \"\n+                                 \"to partial_fit.\")\n+        elif classes is not None:\n+            self.classes_ = classes\n+\n+        if not hasattr(self, '_lbin'):\n+            self._lbin = LabelBinarizer()\n+            self._lbin._classes = classes","path":"sklearn/neural_network/extreme_learning_machines.py","position":581,"original_position":581,"commit_id":"4cbf70943bd1ef5c7de5fb172ee15f457e54b255","original_commit_id":"4cbf70943bd1ef5c7de5fb172ee15f457e54b255","user":{"login":"arjoly","id":1274722,"avatar_url":"https://avatars.githubusercontent.com/u/1274722?","gravatar_id":"9fb38ab677ea6f73bb07faa43b0677f9","url":"https://api.github.com/users/arjoly","html_url":"https://github.com/arjoly","followers_url":"https://api.github.com/users/arjoly/followers","following_url":"https://api.github.com/users/arjoly/following{/other_user}","gists_url":"https://api.github.com/users/arjoly/gists{/gist_id}","starred_url":"https://api.github.com/users/arjoly/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/arjoly/subscriptions","organizations_url":"https://api.github.com/users/arjoly/orgs","repos_url":"https://api.github.com/users/arjoly/repos","events_url":"https://api.github.com/users/arjoly/events{/privacy}","received_events_url":"https://api.github.com/users/arjoly/received_events","type":"User","site_admin":false},"body":"I would avoid this. Better patching the label binarizer to accept a classes argument.","created_at":"2014-07-22T21:13:39Z","updated_at":"2014-07-22T21:13:39Z","html_url":"https://github.com/scikit-learn/scikit-learn/pull/3306#discussion_r15256710","pull_request_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306","_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/15256710"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/3306#discussion_r15256710"},"pull_request":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306"}}},"pull_request":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306","id":17446637,"html_url":"https://github.com/scikit-learn/scikit-learn/pull/3306","diff_url":"https://github.com/scikit-learn/scikit-learn/pull/3306.diff","patch_url":"https://github.com/scikit-learn/scikit-learn/pull/3306.patch","issue_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3306","number":3306,"state":"open","title":"GSoC 2014: Standard Extreme Learning Machines","user":{"login":"IssamLaradji","id":3382128,"avatar_url":"https://avatars.githubusercontent.com/u/3382128?","gravatar_id":"e35d22a037b8b35422d3ee982f133076","url":"https://api.github.com/users/IssamLaradji","html_url":"https://github.com/IssamLaradji","followers_url":"https://api.github.com/users/IssamLaradji/followers","following_url":"https://api.github.com/users/IssamLaradji/following{/other_user}","gists_url":"https://api.github.com/users/IssamLaradji/gists{/gist_id}","starred_url":"https://api.github.com/users/IssamLaradji/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/IssamLaradji/subscriptions","organizations_url":"https://api.github.com/users/IssamLaradji/orgs","repos_url":"https://api.github.com/users/IssamLaradji/repos","events_url":"https://api.github.com/users/IssamLaradji/events{/privacy}","received_events_url":"https://api.github.com/users/IssamLaradji/received_events","type":"User","site_admin":false},"body":"Finished implementing the standard extreme learning machines (ELMs). I am getting the following results with 550 hidden neurons against the digits datasets,\r\n\r\nTraining accuracy using the logistic activation function: 0.999444\r\nTraining accuracy using the tanh activation function: 1.000000\r\n\r\nFortunately, this algorithm is much easier to implement and debug than multi-layer perceptron :).\r\nI will push a test file soon.\r\n\r\n@ogrisel , @larsmans","created_at":"2014-06-22T08:34:21Z","updated_at":"2014-07-22T21:13:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0bcf4f5ec0dc86e9aaef49d1f51e2ad9545c5c17","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306/commits","review_comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306/comments","review_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/{number}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3306/comments","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/4cbf70943bd1ef5c7de5fb172ee15f457e54b255","head":{"label":"IssamLaradji:Extreme-Learning-Machines","ref":"Extreme-Learning-Machines","sha":"4cbf70943bd1ef5c7de5fb172ee15f457e54b255","user":{"login":"IssamLaradji","id":3382128,"avatar_url":"https://avatars.githubusercontent.com/u/3382128?","gravatar_id":"e35d22a037b8b35422d3ee982f133076","url":"https://api.github.com/users/IssamLaradji","html_url":"https://github.com/IssamLaradji","followers_url":"https://api.github.com/users/IssamLaradji/followers","following_url":"https://api.github.com/users/IssamLaradji/following{/other_user}","gists_url":"https://api.github.com/users/IssamLaradji/gists{/gist_id}","starred_url":"https://api.github.com/users/IssamLaradji/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/IssamLaradji/subscriptions","organizations_url":"https://api.github.com/users/IssamLaradji/orgs","repos_url":"https://api.github.com/users/IssamLaradji/repos","events_url":"https://api.github.com/users/IssamLaradji/events{/privacy}","received_events_url":"https://api.github.com/users/IssamLaradji/received_events","type":"User","site_admin":false},"repo":{"id":10518207,"name":"scikit-learn","full_name":"IssamLaradji/scikit-learn","owner":{"login":"IssamLaradji","id":3382128,"avatar_url":"https://avatars.githubusercontent.com/u/3382128?","gravatar_id":"e35d22a037b8b35422d3ee982f133076","url":"https://api.github.com/users/IssamLaradji","html_url":"https://github.com/IssamLaradji","followers_url":"https://api.github.com/users/IssamLaradji/followers","following_url":"https://api.github.com/users/IssamLaradji/following{/other_user}","gists_url":"https://api.github.com/users/IssamLaradji/gists{/gist_id}","starred_url":"https://api.github.com/users/IssamLaradji/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/IssamLaradji/subscriptions","organizations_url":"https://api.github.com/users/IssamLaradji/orgs","repos_url":"https://api.github.com/users/IssamLaradji/repos","events_url":"https://api.github.com/users/IssamLaradji/events{/privacy}","received_events_url":"https://api.github.com/users/IssamLaradji/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/IssamLaradji/scikit-learn","description":"scikit-learn: machine learning in Python","fork":true,"url":"https://api.github.com/repos/IssamLaradji/scikit-learn","forks_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/forks","keys_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/teams","hooks_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/events","assignees_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/tags","blobs_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/subscription","commits_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/issues/comments/{number}","contents_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/merges","archive_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/downloads","issues_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/IssamLaradji/scikit-learn/releases{/id}","created_at":"2013-06-06T03:31:53Z","updated_at":"2014-06-24T12:42:31Z","pushed_at":"2014-07-20T00:55:54Z","git_url":"git://github.com/IssamLaradji/scikit-learn.git","ssh_url":"git@github.com:IssamLaradji/scikit-learn.git","clone_url":"https://github.com/IssamLaradji/scikit-learn.git","svn_url":"https://github.com/IssamLaradji/scikit-learn","homepage":"http://scikit-learn.org","size":54297,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"scikit-learn:master","ref":"master","sha":"f7e95277b95d2149603e31c4cdc1e80188b725ca","user":{"login":"scikit-learn","id":365630,"avatar_url":"https://avatars.githubusercontent.com/u/365630?","gravatar_id":"8a6ecb57b119c5a5b3d882d08a909278","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"repo":{"id":843222,"name":"scikit-learn","full_name":"scikit-learn/scikit-learn","owner":{"login":"scikit-learn","id":365630,"avatar_url":"https://avatars.githubusercontent.com/u/365630?","gravatar_id":"8a6ecb57b119c5a5b3d882d08a909278","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scikit-learn/scikit-learn","description":"scikit-learn: machine learning in Python","fork":false,"url":"https://api.github.com/repos/scikit-learn/scikit-learn","forks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/forks","keys_url":"https://api.github.com/repos/scikit-learn/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scikit-learn/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scikit-learn/scikit-learn/teams","hooks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/events","assignees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/scikit-learn/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/tags","blobs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/scikit-learn/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscription","commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/comments/{number}","contents_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/scikit-learn/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scikit-learn/scikit-learn/merges","archive_url":"https://api.github.com/repos/scikit-learn/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scikit-learn/scikit-learn/downloads","issues_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/scikit-learn/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/scikit-learn/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scikit-learn/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/scikit-learn/scikit-learn/releases{/id}","created_at":"2010-08-17T09:43:38Z","updated_at":"2014-07-22T19:52:06Z","pushed_at":"2014-07-22T14:10:40Z","git_url":"git://github.com/scikit-learn/scikit-learn.git","ssh_url":"git@github.com:scikit-learn/scikit-learn.git","clone_url":"https://github.com/scikit-learn/scikit-learn.git","svn_url":"https://github.com/scikit-learn/scikit-learn","homepage":"http://scikit-learn.org","size":112589,"stargazers_count":3663,"watchers_count":3663,"language":"C","has_issues":true,"has_downloads":true,"has_wiki":true,"forks_count":1935,"mirror_url":null,"open_issues_count":464,"forks":1935,"open_issues":464,"watchers":3663,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/3306"},"issue":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3306"},"comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3306/comments"},"review_comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306/comments"},"review_comment":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3306/commits"},"statuses":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/4cbf70943bd1ef5c7de5fb172ee15f457e54b255"}}}}