GitHub PullRequestReviewCommentEvent at 2014-07-11 20:27:47
Repository: sparklingpandas
Actor: jhlch
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/14841949","id":14841949,"diff_hunk":"@@ -0,0 +1,306 @@\n+\"\"\"Provide wrapper around the grouped result from L{PRDD}\"\"\"\n+#\n+# Licensed to the Apache Software Foundation (ASF) under one or more\n+# contributor license agreements.  See the NOTICE file distributed with\n+# this work for additional information regarding copyright ownership.\n+# The ASF licenses this file to You under the Apache License, Version 2.0\n+# (the \"License\"); you may not use this file except in compliance with\n+# the License.  You may obtain a copy of the License at\n+#\n+#    http://www.apache.org/licenses/LICENSE-2.0\n+#\n+# Unless required by applicable law or agreed to in writing, software\n+# distributed under the License is distributed on an \"AS IS\" BASIS,\n+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+# See the License for the specific language governing permissions and\n+# limitations under the License.\n+#\n+\n+from sparklingpandas.utils import add_pyspark_path\n+from sparklingpandas.prdd import PRDD\n+add_pyspark_path()\n+import pandas\n+import numpy as np\n+\n+\n+class Groupby:\n+\n+    \"\"\"\n+    An RDD with key value pairs, where each value is a dataframe and the key is\n+    the result of the group.\n+    \"\"\"\n+    # TODO(HOLDEN): finish implement the rest of the non \"protected\" functions\n+    # in groupby\n+\n+    def __init__(self, rdd, *args, **kwargs):\n+        \"\"\"Construct a groupby object providing the functions on top of the\n+        provided RDD. We keep the base RDD so if someone calls aggregate we\n+        do things more intelligently.\n+        \"\"\"\n+        def extractKeys(groupedFrame):\n+            for key, group in groupedFrame:\n+                yield (key, group)\n+\n+        def group_and_extract(frame):\n+            return extractKeys(frame.groupby(*args, **kwargs))\n+        prereducedRDD = rdd.flatMap(group_and_extract)\n+        groupedRDD = self._groupRDD(prereducedRDD)\n+        self._sort = kwargs.get(\"sort\", True)\n+        if self._sort:\n+            groupedRDD = groupedRDD.sortByKey()\n+        self._baserdd = rdd\n+        self._prereducedrdd = prereducedRDD\n+        self._groupedrdd = groupedRDD\n+        self._myargs = args\n+        self._mykwargs = kwargs\n+\n+    def _sortIfNeeded(self, rdd):\n+        \"\"\"Sort by key if we need to.\"\"\"\n+        if self._sort:\n+            return rdd.sortByKey()\n+        else:\n+            return rdd\n+\n+    def _groupRDD(self, rdd):\n+        \"\"\"Group together the values with the same key.\"\"\"\n+        return rdd.reduceByKey(lambda x, y: x.append(y))\n+\n+    def _cache(self):\n+        \"\"\"Cache the grouped RDD. This is useful if you have multiple\n+        computations to run on the result. This is a SparklingPandas\n+        extension.\n+        \"\"\"\n+        self._groupedrdd.cache()\n+\n+    def __len__(self):\n+        \"\"\"Number of groups.\"\"\"\n+        return self._groupedrdd.count()\n+\n+    def get_group(self, name):\n+        \"\"\"Returns a concrete DataFrame for provided group name.\"\"\"\n+        self._groupedrdd.lookup(name)\n+\n+    def __iter__(self):\n+        \"\"\"Groupby iterator returns a sequence of (name, object) for each","path":"sparklingpandas/groupby.py","position":84,"original_position":84,"commit_id":"6962a2947f48e10c89990d7f87412981a3d873a7","original_commit_id":"6962a2947f48e10c89990d7f87412981a3d873a7","user":{"login":"jhlch","id":1074683,"avatar_url":"https://avatars.githubusercontent.com/u/1074683?","gravatar_id":"b17b0bd45989aae5106a4ef5e53ed38c","url":"https://api.github.com/users/jhlch","html_url":"https://github.com/jhlch","followers_url":"https://api.github.com/users/jhlch/followers","following_url":"https://api.github.com/users/jhlch/following{/other_user}","gists_url":"https://api.github.com/users/jhlch/gists{/gist_id}","starred_url":"https://api.github.com/users/jhlch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jhlch/subscriptions","organizations_url":"https://api.github.com/users/jhlch/orgs","repos_url":"https://api.github.com/users/jhlch/repos","events_url":"https://api.github.com/users/jhlch/events{/privacy}","received_events_url":"https://api.github.com/users/jhlch/received_events","type":"User","site_admin":false},"body":"I think a consistent phrasing for this documentation would be good. \"Returns an iterator of (name, dataframe)\" I think the group returned in the (name, group) iterator is of type DataFrame in pandas. Shouldn't this return a pRDD? what does it currently return? Perhaps we should test for this behavior and make it explicit.","created_at":"2014-07-11T20:27:47Z","updated_at":"2014-07-11T20:27:47Z","html_url":"https://github.com/holdenk/sparklingpandas/pull/29#discussion_r14841949","pull_request_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29","_links":{"self":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/14841949"},"html":{"href":"https://github.com/holdenk/sparklingpandas/pull/29#discussion_r14841949"},"pull_request":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29"}}},"pull_request":{"url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29","id":18246140,"html_url":"https://github.com/holdenk/sparklingpandas/pull/29","diff_url":"https://github.com/holdenk/sparklingpandas/pull/29.diff","patch_url":"https://github.com/holdenk/sparklingpandas/pull/29.patch","issue_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/29","number":29,"state":"open","title":"Add simple groupby","user":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"body":"Add a helper method to make a distributed dataframe from a local dataframe (this results in types being reguessed so it is less than ideal), add back in a run-tests script that can yell if SPARK_HOME isn't set (whereas directly using nose just resulted in no output). Add inital code coverage, although we might want to disable it since it only reports on things which run locally and most of our stuff runs remotetly. Remove some dead code from utils. Also update unittest library to unittest2.\r\n\r\nre-created this PR because I accidentally merged the other one. oops :p\r\n\r\n@jhlch let me know what you think (I'm especially not sure about the names I chose for the different RDDs inside of the groupby)","created_at":"2014-07-10T21:01:25Z","updated_at":"2014-07-11T20:27:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e17aba98ccea996f5c3e78874edf99fa10253081","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29/commits","review_comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29/comments","review_comment_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/{number}","comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/29/comments","statuses_url":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/6962a2947f48e10c89990d7f87412981a3d873a7","head":{"label":"holdenk:add-simple-groupby-r2","ref":"add-simple-groupby-r2","sha":"6962a2947f48e10c89990d7f87412981a3d873a7","user":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"repo":{"id":19795739,"name":"sparklingpandas","full_name":"holdenk/sparklingpandas","owner":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/holdenk/sparklingpandas","description":"Pandas On PySpark(POPS)","fork":false,"url":"https://api.github.com/repos/holdenk/sparklingpandas","forks_url":"https://api.github.com/repos/holdenk/sparklingpandas/forks","keys_url":"https://api.github.com/repos/holdenk/sparklingpandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/holdenk/sparklingpandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/holdenk/sparklingpandas/teams","hooks_url":"https://api.github.com/repos/holdenk/sparklingpandas/hooks","issue_events_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/events{/number}","events_url":"https://api.github.com/repos/holdenk/sparklingpandas/events","assignees_url":"https://api.github.com/repos/holdenk/sparklingpandas/assignees{/user}","branches_url":"https://api.github.com/repos/holdenk/sparklingpandas/branches{/branch}","tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/tags","blobs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/{sha}","languages_url":"https://api.github.com/repos/holdenk/sparklingpandas/languages","stargazers_url":"https://api.github.com/repos/holdenk/sparklingpandas/stargazers","contributors_url":"https://api.github.com/repos/holdenk/sparklingpandas/contributors","subscribers_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscribers","subscription_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscription","commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/comments/{number}","contents_url":"https://api.github.com/repos/holdenk/sparklingpandas/contents/{+path}","compare_url":"https://api.github.com/repos/holdenk/sparklingpandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/holdenk/sparklingpandas/merges","archive_url":"https://api.github.com/repos/holdenk/sparklingpandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/holdenk/sparklingpandas/downloads","issues_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues{/number}","pulls_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls{/number}","milestones_url":"https://api.github.com/repos/holdenk/sparklingpandas/milestones{/number}","notifications_url":"https://api.github.com/repos/holdenk/sparklingpandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/holdenk/sparklingpandas/labels{/name}","releases_url":"https://api.github.com/repos/holdenk/sparklingpandas/releases{/id}","created_at":"2014-05-14T20:49:18Z","updated_at":"2014-07-10T00:05:33Z","pushed_at":"2014-07-11T19:14:57Z","git_url":"git://github.com/holdenk/sparklingpandas.git","ssh_url":"git@github.com:holdenk/sparklingpandas.git","clone_url":"https://github.com/holdenk/sparklingpandas.git","svn_url":"https://github.com/holdenk/sparklingpandas","homepage":null,"size":2049,"stargazers_count":8,"watchers_count":8,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"forks_count":1,"mirror_url":null,"open_issues_count":16,"forks":1,"open_issues":16,"watchers":8,"default_branch":"master"}},"base":{"label":"holdenk:master","ref":"master","sha":"4a8daefd6bd6c51741b0b849d41d9189b7787023","user":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"repo":{"id":19795739,"name":"sparklingpandas","full_name":"holdenk/sparklingpandas","owner":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/holdenk/sparklingpandas","description":"Pandas On PySpark(POPS)","fork":false,"url":"https://api.github.com/repos/holdenk/sparklingpandas","forks_url":"https://api.github.com/repos/holdenk/sparklingpandas/forks","keys_url":"https://api.github.com/repos/holdenk/sparklingpandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/holdenk/sparklingpandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/holdenk/sparklingpandas/teams","hooks_url":"https://api.github.com/repos/holdenk/sparklingpandas/hooks","issue_events_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/events{/number}","events_url":"https://api.github.com/repos/holdenk/sparklingpandas/events","assignees_url":"https://api.github.com/repos/holdenk/sparklingpandas/assignees{/user}","branches_url":"https://api.github.com/repos/holdenk/sparklingpandas/branches{/branch}","tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/tags","blobs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/{sha}","languages_url":"https://api.github.com/repos/holdenk/sparklingpandas/languages","stargazers_url":"https://api.github.com/repos/holdenk/sparklingpandas/stargazers","contributors_url":"https://api.github.com/repos/holdenk/sparklingpandas/contributors","subscribers_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscribers","subscription_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscription","commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/comments/{number}","contents_url":"https://api.github.com/repos/holdenk/sparklingpandas/contents/{+path}","compare_url":"https://api.github.com/repos/holdenk/sparklingpandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/holdenk/sparklingpandas/merges","archive_url":"https://api.github.com/repos/holdenk/sparklingpandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/holdenk/sparklingpandas/downloads","issues_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues{/number}","pulls_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls{/number}","milestones_url":"https://api.github.com/repos/holdenk/sparklingpandas/milestones{/number}","notifications_url":"https://api.github.com/repos/holdenk/sparklingpandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/holdenk/sparklingpandas/labels{/name}","releases_url":"https://api.github.com/repos/holdenk/sparklingpandas/releases{/id}","created_at":"2014-05-14T20:49:18Z","updated_at":"2014-07-10T00:05:33Z","pushed_at":"2014-07-11T19:14:57Z","git_url":"git://github.com/holdenk/sparklingpandas.git","ssh_url":"git@github.com:holdenk/sparklingpandas.git","clone_url":"https://github.com/holdenk/sparklingpandas.git","svn_url":"https://github.com/holdenk/sparklingpandas","homepage":null,"size":2049,"stargazers_count":8,"watchers_count":8,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"forks_count":1,"mirror_url":null,"open_issues_count":16,"forks":1,"open_issues":16,"watchers":8,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29"},"html":{"href":"https://github.com/holdenk/sparklingpandas/pull/29"},"issue":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/issues/29"},"comments":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/issues/29/comments"},"review_comments":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29/comments"},"review_comment":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/29/commits"},"statuses":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/6962a2947f48e10c89990d7f87412981a3d873a7"}}}}