GitHub PullRequestReviewCommentEvent at 2014-10-28 19:58:19
Repository: spark
Actor: mateiz
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/19498939","id":19498939,"diff_hunk":"@@ -209,25 +221,25 @@ private[sql] object ParquetFilters {\n           case _ => None\n         }\n       }\n-      case p @ EqualTo(left: Literal, right: NamedExpression) if !right.nullable =>\n+      case p @ EqualTo(left: Literal, right: NamedExpression) =>\n         Some(createEqualityFilter(right.name, left, p))\n-      case p @ EqualTo(left: NamedExpression, right: Literal) if !left.nullable =>\n+      case p @ EqualTo(left: NamedExpression, right: Literal) =>\n         Some(createEqualityFilter(left.name, right, p))\n-      case p @ LessThan(left: Literal, right: NamedExpression) if !right.nullable =>\n+      case p @ LessThan(left: Literal, right: NamedExpression) =>\n         Some(createLessThanFilter(right.name, left, p))\n-      case p @ LessThan(left: NamedExpression, right: Literal) if !left.nullable =>\n+      case p @ LessThan(left: NamedExpression, right: Literal) =>\n         Some(createLessThanFilter(left.name, right, p))\n-      case p @ LessThanOrEqual(left: Literal, right: NamedExpression) if !right.nullable =>\n+      case p @ LessThanOrEqual(left: Literal, right: NamedExpression) =>\n         Some(createLessThanOrEqualFilter(right.name, left, p))\n-      case p @ LessThanOrEqual(left: NamedExpression, right: Literal) if !left.nullable =>\n+      case p @ LessThanOrEqual(left: NamedExpression, right: Literal) =>\n         Some(createLessThanOrEqualFilter(left.name, right, p))\n-      case p @ GreaterThan(left: Literal, right: NamedExpression) if !right.nullable =>\n+      case p @ GreaterThan(left: Literal, right: NamedExpression) =>\n         Some(createGreaterThanFilter(right.name, left, p))\n-      case p @ GreaterThan(left: NamedExpression, right: Literal) if !left.nullable =>\n+      case p @ GreaterThan(left: NamedExpression, right: Literal) =>\n         Some(createGreaterThanFilter(left.name, right, p))\n-      case p @ GreaterThanOrEqual(left: Literal, right: NamedExpression) if !right.nullable =>\n+      case p @ GreaterThanOrEqual(left: Literal, right: NamedExpression) =>\n         Some(createGreaterThanOrEqualFilter(right.name, left, p))\n-      case p @ GreaterThanOrEqual(left: NamedExpression, right: Literal) if !left.nullable =>\n+      case p @ GreaterThanOrEqual(left: NamedExpression, right: Literal) =>","path":"sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetFilters.scala","position":243,"original_position":243,"commit_id":"515df1cc70fec520828f45fb4b52a2643a6e7dcb","original_commit_id":"515df1cc70fec520828f45fb4b52a2643a6e7dcb","user":{"login":"mateiz","id":228859,"avatar_url":"https://avatars.githubusercontent.com/u/228859?v=2","gravatar_id":"","url":"https://api.github.com/users/mateiz","html_url":"https://github.com/mateiz","followers_url":"https://api.github.com/users/mateiz/followers","following_url":"https://api.github.com/users/mateiz/following{/other_user}","gists_url":"https://api.github.com/users/mateiz/gists{/gist_id}","starred_url":"https://api.github.com/users/mateiz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mateiz/subscriptions","organizations_url":"https://api.github.com/users/mateiz/orgs","repos_url":"https://api.github.com/users/mateiz/repos","events_url":"https://api.github.com/users/mateiz/events{/privacy}","received_events_url":"https://api.github.com/users/mateiz/received_events","type":"User","site_admin":false},"body":"Would be nice to add some tests with == null or >= null as well to make sure these filters work","created_at":"2014-10-28T19:58:19Z","updated_at":"2014-10-28T19:58:19Z","html_url":"https://github.com/apache/spark/pull/2841#discussion_r19498939","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/2841","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/19498939"},"html":{"href":"https://github.com/apache/spark/pull/2841#discussion_r19498939"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/2841"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/2841","id":22967791,"html_url":"https://github.com/apache/spark/pull/2841","diff_url":"https://github.com/apache/spark/pull/2841.diff","patch_url":"https://github.com/apache/spark/pull/2841.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/2841","number":2841,"state":"open","locked":false,"title":"SPARK-3968 Use parquet-mr filter2 api in spark sql","user":{"login":"saucam","id":1253893,"avatar_url":"https://avatars.githubusercontent.com/u/1253893?v=2","gravatar_id":"","url":"https://api.github.com/users/saucam","html_url":"https://github.com/saucam","followers_url":"https://api.github.com/users/saucam/followers","following_url":"https://api.github.com/users/saucam/following{/other_user}","gists_url":"https://api.github.com/users/saucam/gists{/gist_id}","starred_url":"https://api.github.com/users/saucam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/saucam/subscriptions","organizations_url":"https://api.github.com/users/saucam/orgs","repos_url":"https://api.github.com/users/saucam/repos","events_url":"https://api.github.com/users/saucam/events{/privacy}","received_events_url":"https://api.github.com/users/saucam/received_events","type":"User","site_admin":false},"body":"The parquet-mr project has introduced a new filter api  (https://github.com/apache/incubator-parquet-mr/pull/4), along with several fixes . It can also eliminate entire RowGroups depending on certain statistics like min/max\r\nWe can leverage that to further improve performance of queries with filters.\r\nAlso filter2 api introduces ability to create custom filters. We can create a custom filter for the optimized In clause (InSet) , so that elimination happens in the ParquetRecordReader itself ","created_at":"2014-10-18T13:20:44Z","updated_at":"2014-10-28T19:58:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bc5c213ff0452bc7d4da4faadf18c6c106372758","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/2841/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/2841/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments/{number}","comments_url":"https://api.github.com/repos/apache/spark/issues/2841/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/515df1cc70fec520828f45fb4b52a2643a6e7dcb","head":{"label":"saucam:master","ref":"master","sha":"515df1cc70fec520828f45fb4b52a2643a6e7dcb","user":{"login":"saucam","id":1253893,"avatar_url":"https://avatars.githubusercontent.com/u/1253893?v=2","gravatar_id":"","url":"https://api.github.com/users/saucam","html_url":"https://github.com/saucam","followers_url":"https://api.github.com/users/saucam/followers","following_url":"https://api.github.com/users/saucam/following{/other_user}","gists_url":"https://api.github.com/users/saucam/gists{/gist_id}","starred_url":"https://api.github.com/users/saucam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/saucam/subscriptions","organizations_url":"https://api.github.com/users/saucam/orgs","repos_url":"https://api.github.com/users/saucam/repos","events_url":"https://api.github.com/users/saucam/events{/privacy}","received_events_url":"https://api.github.com/users/saucam/received_events","type":"User","site_admin":false},"repo":{"id":24550652,"name":"spark","full_name":"saucam/spark","owner":{"login":"saucam","id":1253893,"avatar_url":"https://avatars.githubusercontent.com/u/1253893?v=2","gravatar_id":"","url":"https://api.github.com/users/saucam","html_url":"https://github.com/saucam","followers_url":"https://api.github.com/users/saucam/followers","following_url":"https://api.github.com/users/saucam/following{/other_user}","gists_url":"https://api.github.com/users/saucam/gists{/gist_id}","starred_url":"https://api.github.com/users/saucam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/saucam/subscriptions","organizations_url":"https://api.github.com/users/saucam/orgs","repos_url":"https://api.github.com/users/saucam/repos","events_url":"https://api.github.com/users/saucam/events{/privacy}","received_events_url":"https://api.github.com/users/saucam/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/saucam/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/saucam/spark","forks_url":"https://api.github.com/repos/saucam/spark/forks","keys_url":"https://api.github.com/repos/saucam/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/saucam/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/saucam/spark/teams","hooks_url":"https://api.github.com/repos/saucam/spark/hooks","issue_events_url":"https://api.github.com/repos/saucam/spark/issues/events{/number}","events_url":"https://api.github.com/repos/saucam/spark/events","assignees_url":"https://api.github.com/repos/saucam/spark/assignees{/user}","branches_url":"https://api.github.com/repos/saucam/spark/branches{/branch}","tags_url":"https://api.github.com/repos/saucam/spark/tags","blobs_url":"https://api.github.com/repos/saucam/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/saucam/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/saucam/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/saucam/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/saucam/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/saucam/spark/languages","stargazers_url":"https://api.github.com/repos/saucam/spark/stargazers","contributors_url":"https://api.github.com/repos/saucam/spark/contributors","subscribers_url":"https://api.github.com/repos/saucam/spark/subscribers","subscription_url":"https://api.github.com/repos/saucam/spark/subscription","commits_url":"https://api.github.com/repos/saucam/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/saucam/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/saucam/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/saucam/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/saucam/spark/contents/{+path}","compare_url":"https://api.github.com/repos/saucam/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/saucam/spark/merges","archive_url":"https://api.github.com/repos/saucam/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/saucam/spark/downloads","issues_url":"https://api.github.com/repos/saucam/spark/issues{/number}","pulls_url":"https://api.github.com/repos/saucam/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/saucam/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/saucam/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/saucam/spark/labels{/name}","releases_url":"https://api.github.com/repos/saucam/spark/releases{/id}","created_at":"2014-09-28T05:01:31Z","updated_at":"2014-10-16T12:16:39Z","pushed_at":"2014-10-28T05:55:05Z","git_url":"git://github.com/saucam/spark.git","ssh_url":"git@github.com:saucam/spark.git","clone_url":"https://github.com/saucam/spark.git","svn_url":"https://github.com/saucam/spark","homepage":null,"size":80289,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"9530316887612dca060a128fca34dd5a6ab2a9a9","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=2","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=2","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2014-10-28T19:53:35Z","pushed_at":"2014-10-28T19:48:47Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":3809983,"stargazers_count":1864,"watchers_count":1864,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1637,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":285,"forks":1637,"open_issues":285,"watchers":1864,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/2841"},"html":{"href":"https://github.com/apache/spark/pull/2841"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/2841"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/2841/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/2841/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/2841/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/515df1cc70fec520828f45fb4b52a2643a6e7dcb"}}}}