GitHub PullRequestReviewCommentEvent at 2014-08-04 03:56:19
Repository: spark
Actor: mengxr
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/15739934","id":15739934,"diff_hunk":"@@ -0,0 +1,77 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.mllib.feature\n+\n+import breeze.linalg.{DenseVector => BDV, SparseVector => BSV}\n+\n+import org.apache.spark.annotation.DeveloperApi\n+import org.apache.spark.mllib.linalg.{Vector, Vectors}\n+\n+/**\n+ * :: DeveloperApi ::\n+ * Normalizes samples individually to unit L^p^ norm\n+ *\n+ * For any 1 <= p < Double.PositiveInfinity, normalizes samples using\n+ * sum(abs(vector).^p^)^(1/p)^ as norm.\n+ *\n+ * For p = Double.PositiveInfinity, max(abs(vector)) will be used as norm for normalization.\n+ *\n+ * @param p Normalization in L^p^ space, p = 2 by default.\n+ */\n+@DeveloperApi\n+class Normalizer(p: Double) extends VectorTransformer {\n+\n+  def this() = this(2)\n+\n+  require(p >= 1.0)\n+\n+  /**\n+   * Applies unit length normalization on a vector.\n+   *\n+   * @param vector vector to be normalized.\n+   * @return normalized vector. If the norm of the input is zero, it will return the input vector.\n+   */\n+  override def transform(vector: Vector): Vector = {\n+    var norm = vector.toBreeze.norm(p)\n+\n+    if (norm != 0.0) {\n+      // For dense vector, we've to allocate new memory for new output vector.\n+      // However, for sparse vector, the `index` array will not be changed,\n+      // so we can re-use it to save memory.\n+      vector.toBreeze match {\n+        case dv: BDV[Double] => Vectors.fromBreeze(dv :/ norm)\n+        case sv: BSV[Double] =>\n+          val output = new BSV[Double](sv.index, sv.data.clone(), sv.length)\n+          var i = 0\n+          while (i < output.data.length) {\n+            output.data(i) /= norm\n+            i += 1\n+          }\n+          Vectors.fromBreeze(output)\n+        case v: Any =>","path":"mllib/src/main/scala/org/apache/spark/mllib/feature/Normalizer.scala","position":66,"original_position":66,"commit_id":"9bdc2b4971a5a5c566ad1bfcdf3d168c565bfa9f","original_commit_id":"9bdc2b4971a5a5c566ad1bfcdf3d168c565bfa9f","user":{"login":"mengxr","id":829644,"avatar_url":"https://avatars.githubusercontent.com/u/829644?v=1","gravatar_id":"0e16fce91950d1b5a312649751ff1b14","url":"https://api.github.com/users/mengxr","html_url":"https://github.com/mengxr","followers_url":"https://api.github.com/users/mengxr/followers","following_url":"https://api.github.com/users/mengxr/following{/other_user}","gists_url":"https://api.github.com/users/mengxr/gists{/gist_id}","starred_url":"https://api.github.com/users/mengxr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mengxr/subscriptions","organizations_url":"https://api.github.com/users/mengxr/orgs","repos_url":"https://api.github.com/users/mengxr/repos","events_url":"https://api.github.com/users/mengxr/events{/privacy}","received_events_url":"https://api.github.com/users/mengxr/received_events","type":"User","site_admin":false},"body":"minor: `: Any` can be removed","created_at":"2014-08-04T03:56:19Z","updated_at":"2014-08-04T03:56:19Z","html_url":"https://github.com/apache/spark/pull/1207#discussion_r15739934","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/1207","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/15739934"},"html":{"href":"https://github.com/apache/spark/pull/1207#discussion_r15739934"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/1207"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/1207","id":17570430,"html_url":"https://github.com/apache/spark/pull/1207","diff_url":"https://github.com/apache/spark/pull/1207.diff","patch_url":"https://github.com/apache/spark/pull/1207.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/1207","number":1207,"state":"open","title":"SPARK-2272 [MLlib] Feature scaling which standardizes the range of independent variables or features of data","user":{"login":"dbtsai","id":1134574,"avatar_url":"https://avatars.githubusercontent.com/u/1134574?v=1","gravatar_id":"39e915ddd0924cc3bde7225b4690ae6d","url":"https://api.github.com/users/dbtsai","html_url":"https://github.com/dbtsai","followers_url":"https://api.github.com/users/dbtsai/followers","following_url":"https://api.github.com/users/dbtsai/following{/other_user}","gists_url":"https://api.github.com/users/dbtsai/gists{/gist_id}","starred_url":"https://api.github.com/users/dbtsai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dbtsai/subscriptions","organizations_url":"https://api.github.com/users/dbtsai/orgs","repos_url":"https://api.github.com/users/dbtsai/repos","events_url":"https://api.github.com/users/dbtsai/events{/privacy}","received_events_url":"https://api.github.com/users/dbtsai/received_events","type":"User","site_admin":false},"body":"Feature scaling is a method used to standardize the range of independent variables or features of data. In data processing, it is generally performed during the data preprocessing step.\r\n\r\nIn this work, a trait called `VectorTransformer` is defined for generic transformation on a vector. It contains one method to be implemented, `transform` which applies transformation on a vector.\r\n\r\nThere are two implementations of `VectorTransformer` now, and they all can be easily extended with PMML transformation support.\r\n\r\n1) `StandardScaler` - Standardizes features by removing the mean and scaling to unit variance using column summary statistics on the samples in the training set.\r\n\r\n2) `Normalizer` - Normalizes samples individually to unit L^n norm","created_at":"2014-06-25T03:37:27Z","updated_at":"2014-08-04T03:56:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1e18c4f3335cd262f603adc9f9ffe4d458233568","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/1207/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/1207/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments/{number}","comments_url":"https://api.github.com/repos/apache/spark/issues/1207/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/9bdc2b4971a5a5c566ad1bfcdf3d168c565bfa9f","head":{"label":"dbtsai:dbtsai-feature-scaling","ref":"dbtsai-feature-scaling","sha":"9bdc2b4971a5a5c566ad1bfcdf3d168c565bfa9f","user":{"login":"dbtsai","id":1134574,"avatar_url":"https://avatars.githubusercontent.com/u/1134574?v=1","gravatar_id":"39e915ddd0924cc3bde7225b4690ae6d","url":"https://api.github.com/users/dbtsai","html_url":"https://github.com/dbtsai","followers_url":"https://api.github.com/users/dbtsai/followers","following_url":"https://api.github.com/users/dbtsai/following{/other_user}","gists_url":"https://api.github.com/users/dbtsai/gists{/gist_id}","starred_url":"https://api.github.com/users/dbtsai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dbtsai/subscriptions","organizations_url":"https://api.github.com/users/dbtsai/orgs","repos_url":"https://api.github.com/users/dbtsai/repos","events_url":"https://api.github.com/users/dbtsai/events{/privacy}","received_events_url":"https://api.github.com/users/dbtsai/received_events","type":"User","site_admin":false},"repo":{"id":18395072,"name":"spark","full_name":"dbtsai/spark","owner":{"login":"dbtsai","id":1134574,"avatar_url":"https://avatars.githubusercontent.com/u/1134574?v=1","gravatar_id":"39e915ddd0924cc3bde7225b4690ae6d","url":"https://api.github.com/users/dbtsai","html_url":"https://github.com/dbtsai","followers_url":"https://api.github.com/users/dbtsai/followers","following_url":"https://api.github.com/users/dbtsai/following{/other_user}","gists_url":"https://api.github.com/users/dbtsai/gists{/gist_id}","starred_url":"https://api.github.com/users/dbtsai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dbtsai/subscriptions","organizations_url":"https://api.github.com/users/dbtsai/orgs","repos_url":"https://api.github.com/users/dbtsai/repos","events_url":"https://api.github.com/users/dbtsai/events{/privacy}","received_events_url":"https://api.github.com/users/dbtsai/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dbtsai/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/dbtsai/spark","forks_url":"https://api.github.com/repos/dbtsai/spark/forks","keys_url":"https://api.github.com/repos/dbtsai/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dbtsai/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dbtsai/spark/teams","hooks_url":"https://api.github.com/repos/dbtsai/spark/hooks","issue_events_url":"https://api.github.com/repos/dbtsai/spark/issues/events{/number}","events_url":"https://api.github.com/repos/dbtsai/spark/events","assignees_url":"https://api.github.com/repos/dbtsai/spark/assignees{/user}","branches_url":"https://api.github.com/repos/dbtsai/spark/branches{/branch}","tags_url":"https://api.github.com/repos/dbtsai/spark/tags","blobs_url":"https://api.github.com/repos/dbtsai/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dbtsai/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dbtsai/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/dbtsai/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dbtsai/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/dbtsai/spark/languages","stargazers_url":"https://api.github.com/repos/dbtsai/spark/stargazers","contributors_url":"https://api.github.com/repos/dbtsai/spark/contributors","subscribers_url":"https://api.github.com/repos/dbtsai/spark/subscribers","subscription_url":"https://api.github.com/repos/dbtsai/spark/subscription","commits_url":"https://api.github.com/repos/dbtsai/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/dbtsai/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/dbtsai/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/dbtsai/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/dbtsai/spark/contents/{+path}","compare_url":"https://api.github.com/repos/dbtsai/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dbtsai/spark/merges","archive_url":"https://api.github.com/repos/dbtsai/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dbtsai/spark/downloads","issues_url":"https://api.github.com/repos/dbtsai/spark/issues{/number}","pulls_url":"https://api.github.com/repos/dbtsai/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/dbtsai/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/dbtsai/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dbtsai/spark/labels{/name}","releases_url":"https://api.github.com/repos/dbtsai/spark/releases{/id}","created_at":"2014-04-03T07:44:18Z","updated_at":"2014-06-25T20:10:11Z","pushed_at":"2014-08-04T03:24:39Z","git_url":"git://github.com/dbtsai/spark.git","ssh_url":"git@github.com:dbtsai/spark.git","clone_url":"https://github.com/dbtsai/spark.git","svn_url":"https://github.com/dbtsai/spark","homepage":null,"size":78187,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"5507dd8e18fbb52d5e0c64a767103b2418cb09c6","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=1","gravatar_id":"a676c0bf448fcd49f588249ead719b4c","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=1","gravatar_id":"a676c0bf448fcd49f588249ead719b4c","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2014-08-04T03:30:10Z","pushed_at":"2014-08-04T03:12:18Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":2461374,"stargazers_count":943,"watchers_count":943,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"forks_count":978,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":232,"forks":978,"open_issues":232,"watchers":943,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/1207"},"html":{"href":"https://github.com/apache/spark/pull/1207"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/1207"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/1207/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/1207/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/1207/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/9bdc2b4971a5a5c566ad1bfcdf3d168c565bfa9f"}}}}