GitHub PullRequestReviewCommentEvent at 2014-04-10 05:26:29
Repository: spark
Actor: yinxusen
Payload: {"comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/11470220","id":11470220,"diff_hunk":"@@ -44,4 +47,15 @@ private[spark] class WholeTextFileInputFormat extends CombineFileInputFormat[Str\n       context,\n       classOf[WholeTextFileRecordReader])\n   }\n+\n+  /**\n+   * Allow minSplits set by end-user in order to keep compatibility with old Hadoop API.\n+   */\n+  def setMaxSplitSize(context: JobContext, minSplits: Int) {\n+    val files = listStatus(context)\n+    val totalLen = files.map { file =>\n+      if (file.isDir) 0L else file.getLen\n+    }.sum\n+    super.setMaxSplitSize(totalLen / (if (minSplits == 0) 1 else minSplits))","path":"core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala","position":24,"original_position":24,"commit_id":"85975a047eb25b810794cf32dee040fe3ae3b8e7","original_commit_id":"85975a047eb25b810794cf32dee040fe3ae3b8e7","user":{"login":"yinxusen","id":2637239,"avatar_url":"https://avatars.githubusercontent.com/u/2637239?","gravatar_id":"fa6253ec8cf91224f056a1b50778f9de","url":"https://api.github.com/users/yinxusen","html_url":"https://github.com/yinxusen","followers_url":"https://api.github.com/users/yinxusen/followers","following_url":"https://api.github.com/users/yinxusen/following{/other_user}","gists_url":"https://api.github.com/users/yinxusen/gists{/gist_id}","starred_url":"https://api.github.com/users/yinxusen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yinxusen/subscriptions","organizations_url":"https://api.github.com/users/yinxusen/orgs","repos_url":"https://api.github.com/users/yinxusen/repos","events_url":"https://api.github.com/users/yinxusen/events{/privacy}","received_events_url":"https://api.github.com/users/yinxusen/received_events","type":"User","site_admin":false},"body":"For example, `FileInputForrmat` uses `mapred.max.split.szie` and `getMaxSplitSize()`, while `CombineFileInputFormat` use `mapreduce.input.fileinputformat.split.max.size` and `setMaxSplitSize()`.","created_at":"2014-04-10T05:26:29Z","updated_at":"2014-04-10T05:26:29Z","html_url":"https://github.com/apache/spark/pull/376#discussion_r11470220","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/376","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/11470220"},"html":{"href":"https://github.com/apache/spark/pull/376#discussion_r11470220"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/376"}}}}