GitHub PullRequestReviewCommentEvent at 2014-08-03 01:33:01
Repository: sparklingpandas
Actor: holdenk
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/15732568","id":15732568,"diff_hunk":"@@ -43,29 +44,62 @@ def simple(cls, *args, **kwargs):\n         \"\"\"\n         return PSparkContext(SparkContext(*args, **kwargs))\n \n-    def csvfile(self, name, use_whole_file=True, *args, **kwargs):\n+    def read_csv(self, name, use_whole_file=False, names=None, skiprows=0,\n+                 *args, **kwargs):\n         \"\"\"\n-        Read a CSV file in and parse it into panda data frames. Note this uses\n-        wholeTextFiles by default underneath the hood so as to support\n-        multi-line CSV records so many small input files are preferred.\n+        Read a CSV file in and parse it into panda data frames.\n+        If no names is provided uses the first row for the names\n+        header=0 is the default unless names is provided in which case\n+        header=None is the default.\n+        No other values of header is supported.\n         All additional parameters are passed to the read_csv function\n         \"\"\"\n-        # TODO(holden): string IO stuff\n-\n-        def csv_file(contents, *args, **kwargs):\n-            return pandas.read_csv(StringIO(contents), *args, header=0,\n-                                   **kwargs)\n+        def csv_file(partitionNumber, files):\n+            file_count = 0\n+            for filename, contents in files:\n+                # Only skip lines on the first file\n+                if partitionNumber == 0 and file_count == 0 and _skiprows > 0:\n+                    yield pandas.read_csv(StringIO(contents), *args,\n+                                          header=None,\n+                                          names=mynames,\n+                                          skiprows=_skiprows, **kwargs)\n+                else:\n+                    file_count += 1\n+                    yield pandas.read_csv(StringIO(contents), *args,\n+                                          header=None,\n+                                          names=mynames,\n+                                          **kwargs)\n \n-        def csv_rows(rows, *args, **kwargs):\n+        def csv_rows(partitionNumber, rows):\n+            rc = 0\n             for row in rows:\n-                yield pandas.read_csv(StringIO(row), *args, header=0, **kwargs)\n+                # Skip the first rows from the first partition if requested\n+                if partitionNumber != 0 or rc >= _skiprows:\n+                    yield pandas.read_csv(StringIO(row), *args, header=None,\n+                                          names=mynames, **kwargs)\n+                else:\n+                    rc += 1\n+\n+        # If we need to peak at the first partition and determine the column\n+        # names\n+        mynames = None\n+        _skiprows = skiprows\n+        if names:\n+            mynames = names\n+        else:\n+            # In the future we could avoid this expensive call.\n+            first_line = self.sc.textFile(name).first()\n+            frame = pandas.read_csv(StringIO(first_line))\n+            mynames = list(frame.columns.values)\n+            _skiprows += 1","path":"sparklingpandas/pcontext.py","position":70,"original_position":70,"commit_id":"7bf9e0a343b84ca992c9e6884deda62972fa13bf","original_commit_id":"7bf9e0a343b84ca992c9e6884deda62972fa13bf","user":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?v=1","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"body":"I believe it does (or at least the documented behavior).","created_at":"2014-08-03T01:33:01Z","updated_at":"2014-08-03T01:33:01Z","html_url":"https://github.com/holdenk/sparklingpandas/pull/28#discussion_r15732568","pull_request_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28","_links":{"self":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/15732568"},"html":{"href":"https://github.com/holdenk/sparklingpandas/pull/28#discussion_r15732568"},"pull_request":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28"}}},"pull_request":{"url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28","id":18189392,"html_url":"https://github.com/holdenk/sparklingpandas/pull/28","diff_url":"https://github.com/holdenk/sparklingpandas/pull/28.diff","patch_url":"https://github.com/holdenk/sparklingpandas/pull/28.patch","issue_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/28","number":28,"state":"open","title":"Improve csv loading issue 21","user":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?v=1","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"body":"Add basic support for header to read_csv\r\nI'll clean up the tests and rebase this once the other PR is merged.","created_at":"2014-07-09T21:58:19Z","updated_at":"2014-08-03T01:33:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"26ff564642954809c4f0ad56ddefd1b4a52ba863","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28/commits","review_comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28/comments","review_comment_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/{number}","comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/28/comments","statuses_url":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/7bf9e0a343b84ca992c9e6884deda62972fa13bf","head":{"label":"holdenk:improve-csv-loading-issue-21","ref":"improve-csv-loading-issue-21","sha":"7bf9e0a343b84ca992c9e6884deda62972fa13bf","user":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?v=1","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"repo":{"id":19795739,"name":"sparklingpandas","full_name":"holdenk/sparklingpandas","owner":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?v=1","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/holdenk/sparklingpandas","description":"Pandas On PySpark(POPS)","fork":false,"url":"https://api.github.com/repos/holdenk/sparklingpandas","forks_url":"https://api.github.com/repos/holdenk/sparklingpandas/forks","keys_url":"https://api.github.com/repos/holdenk/sparklingpandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/holdenk/sparklingpandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/holdenk/sparklingpandas/teams","hooks_url":"https://api.github.com/repos/holdenk/sparklingpandas/hooks","issue_events_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/events{/number}","events_url":"https://api.github.com/repos/holdenk/sparklingpandas/events","assignees_url":"https://api.github.com/repos/holdenk/sparklingpandas/assignees{/user}","branches_url":"https://api.github.com/repos/holdenk/sparklingpandas/branches{/branch}","tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/tags","blobs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/{sha}","languages_url":"https://api.github.com/repos/holdenk/sparklingpandas/languages","stargazers_url":"https://api.github.com/repos/holdenk/sparklingpandas/stargazers","contributors_url":"https://api.github.com/repos/holdenk/sparklingpandas/contributors","subscribers_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscribers","subscription_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscription","commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/comments/{number}","contents_url":"https://api.github.com/repos/holdenk/sparklingpandas/contents/{+path}","compare_url":"https://api.github.com/repos/holdenk/sparklingpandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/holdenk/sparklingpandas/merges","archive_url":"https://api.github.com/repos/holdenk/sparklingpandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/holdenk/sparklingpandas/downloads","issues_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues{/number}","pulls_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls{/number}","milestones_url":"https://api.github.com/repos/holdenk/sparklingpandas/milestones{/number}","notifications_url":"https://api.github.com/repos/holdenk/sparklingpandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/holdenk/sparklingpandas/labels{/name}","releases_url":"https://api.github.com/repos/holdenk/sparklingpandas/releases{/id}","created_at":"2014-05-14T20:49:18Z","updated_at":"2014-08-02T21:54:03Z","pushed_at":"2014-08-03T00:35:56Z","git_url":"git://github.com/holdenk/sparklingpandas.git","ssh_url":"git@github.com:holdenk/sparklingpandas.git","clone_url":"https://github.com/holdenk/sparklingpandas.git","svn_url":"https://github.com/holdenk/sparklingpandas","homepage":null,"size":2402,"stargazers_count":56,"watchers_count":56,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"forks_count":4,"mirror_url":null,"open_issues_count":16,"forks":4,"open_issues":16,"watchers":56,"default_branch":"master"}},"base":{"label":"holdenk:master","ref":"master","sha":"4a8daefd6bd6c51741b0b849d41d9189b7787023","user":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?v=1","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"repo":{"id":19795739,"name":"sparklingpandas","full_name":"holdenk/sparklingpandas","owner":{"login":"holdenk","id":59893,"avatar_url":"https://avatars.githubusercontent.com/u/59893?v=1","gravatar_id":"a694ad18d60a9def9a25550f88daab0e","url":"https://api.github.com/users/holdenk","html_url":"https://github.com/holdenk","followers_url":"https://api.github.com/users/holdenk/followers","following_url":"https://api.github.com/users/holdenk/following{/other_user}","gists_url":"https://api.github.com/users/holdenk/gists{/gist_id}","starred_url":"https://api.github.com/users/holdenk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/holdenk/subscriptions","organizations_url":"https://api.github.com/users/holdenk/orgs","repos_url":"https://api.github.com/users/holdenk/repos","events_url":"https://api.github.com/users/holdenk/events{/privacy}","received_events_url":"https://api.github.com/users/holdenk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/holdenk/sparklingpandas","description":"Pandas On PySpark(POPS)","fork":false,"url":"https://api.github.com/repos/holdenk/sparklingpandas","forks_url":"https://api.github.com/repos/holdenk/sparklingpandas/forks","keys_url":"https://api.github.com/repos/holdenk/sparklingpandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/holdenk/sparklingpandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/holdenk/sparklingpandas/teams","hooks_url":"https://api.github.com/repos/holdenk/sparklingpandas/hooks","issue_events_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/events{/number}","events_url":"https://api.github.com/repos/holdenk/sparklingpandas/events","assignees_url":"https://api.github.com/repos/holdenk/sparklingpandas/assignees{/user}","branches_url":"https://api.github.com/repos/holdenk/sparklingpandas/branches{/branch}","tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/tags","blobs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/{sha}","languages_url":"https://api.github.com/repos/holdenk/sparklingpandas/languages","stargazers_url":"https://api.github.com/repos/holdenk/sparklingpandas/stargazers","contributors_url":"https://api.github.com/repos/holdenk/sparklingpandas/contributors","subscribers_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscribers","subscription_url":"https://api.github.com/repos/holdenk/sparklingpandas/subscription","commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/holdenk/sparklingpandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/holdenk/sparklingpandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues/comments/{number}","contents_url":"https://api.github.com/repos/holdenk/sparklingpandas/contents/{+path}","compare_url":"https://api.github.com/repos/holdenk/sparklingpandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/holdenk/sparklingpandas/merges","archive_url":"https://api.github.com/repos/holdenk/sparklingpandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/holdenk/sparklingpandas/downloads","issues_url":"https://api.github.com/repos/holdenk/sparklingpandas/issues{/number}","pulls_url":"https://api.github.com/repos/holdenk/sparklingpandas/pulls{/number}","milestones_url":"https://api.github.com/repos/holdenk/sparklingpandas/milestones{/number}","notifications_url":"https://api.github.com/repos/holdenk/sparklingpandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/holdenk/sparklingpandas/labels{/name}","releases_url":"https://api.github.com/repos/holdenk/sparklingpandas/releases{/id}","created_at":"2014-05-14T20:49:18Z","updated_at":"2014-08-02T21:54:03Z","pushed_at":"2014-08-03T00:35:56Z","git_url":"git://github.com/holdenk/sparklingpandas.git","ssh_url":"git@github.com:holdenk/sparklingpandas.git","clone_url":"https://github.com/holdenk/sparklingpandas.git","svn_url":"https://github.com/holdenk/sparklingpandas","homepage":null,"size":2402,"stargazers_count":56,"watchers_count":56,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"forks_count":4,"mirror_url":null,"open_issues_count":16,"forks":4,"open_issues":16,"watchers":56,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28"},"html":{"href":"https://github.com/holdenk/sparklingpandas/pull/28"},"issue":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/issues/28"},"comments":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/issues/28/comments"},"review_comments":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28/comments"},"review_comment":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/pulls/28/commits"},"statuses":{"href":"https://api.github.com/repos/holdenk/sparklingpandas/statuses/7bf9e0a343b84ca992c9e6884deda62972fa13bf"}}}}