GitHub PullRequestReviewCommentEvent at 2014-08-14 18:22:14
Repository: tachyon
Actor: dcapwell
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/amplab/tachyon/pulls/comments/16256124","id":16256124,"diff_hunk":"@@ -0,0 +1,582 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements. See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License. You may obtain a copy of the License at\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package tachyon.worker.hierarchy;\n+\n+import java.io.IOException;\n+import java.nio.ByteBuffer;\n+import java.util.ArrayList;\n+import java.util.HashMap;\n+import java.util.HashSet;\n+import java.util.List;\n+import java.util.Map;\n+import java.util.Set;\n+import java.util.concurrent.ArrayBlockingQueue;\n+import java.util.concurrent.BlockingQueue;\n+\n+import org.apache.log4j.Logger;\n+\n+import tachyon.Constants;\n+import tachyon.UnderFileSystem;\n+import tachyon.client.BlockHandler;\n+import tachyon.util.CommonUtils;\n+import tachyon.worker.WorkerSpaceCounter;\n+\n+/**\n+ * It is used to store and manage block files in storage directory on different\n+ * under file systems.\n+ */\n+public class StorageDir {\n+  protected final Logger LOG = Logger.getLogger(Constants.LOGGER_TYPE);\n+  private final Set<Long> mBlockIds = new HashSet<Long>();\n+  private final Map<Long, Long> mBlockSizes = new HashMap<Long, Long>();\n+  private final String mDataPath;\n+  private final Map<Long, Long> mLatestBlockAccessTimeMs = new HashMap<Long, Long>();\n+\n+  private final Map<Long, Set<Long>> mLockedBlocksPerUser = new HashMap<Long, Set<Long>>();\n+  private final BlockingQueue<Long> mRemovedBlockList = new ArrayBlockingQueue<Long>(\n+      Constants.WORKER_BLOCKS_QUEUE_SIZE);\n+  private final WorkerSpaceCounter mSpaceCounter;\n+  private final String mDirPath;\n+  private final long mStorageId;\n+  private final UnderFileSystem mUFS;\n+  private final Object mUFSConf;\n+  private final Map<Long, Long> mUserAllocatedSpace = new HashMap<Long, Long>();\n+  private final String mUserTempPath;\n+  private final Map<Long, Set<Long>> mUsersPerLockedBlock = new HashMap<Long, Set<Long>>();\n+\n+  StorageDir(long storageId, String dirPath, long capacity, String dataFolder,\n+      String userTempFolder, Object conf) {\n+    mDirPath = dirPath;\n+    mUFSConf = conf;\n+    mUFS = UnderFileSystem.get(mDirPath, conf);\n+    mSpaceCounter = new WorkerSpaceCounter(capacity);\n+    mStorageId = storageId;\n+    mDataPath = mDirPath + Constants.PATH_SEPARATOR + dataFolder;\n+    mUserTempPath = mDirPath + Constants.PATH_SEPARATOR + userTempFolder;\n+  }\n+\n+  /**\n+   * Update the access time of the block\n+   * \n+   * @param blockId\n+   *          id of the block\n+   */\n+  public void accessBlock(long blockId) {\n+    synchronized (mLatestBlockAccessTimeMs) {\n+      mLatestBlockAccessTimeMs.put(blockId, System.currentTimeMillis());\n+    }\n+  }\n+\n+  /**\n+   * Add a block id in current storage dir\n+   * \n+   * @param blockId\n+   *          id of the block\n+   * @param size\n+   *          size of the block file\n+   */\n+  private void addBlockId(long blockId, long size) {\n+    synchronized (mLatestBlockAccessTimeMs) {\n+      mLatestBlockAccessTimeMs.put(blockId, System.currentTimeMillis());\n+      mBlockSizes.put(blockId, size);\n+      mBlockIds.add(blockId);\n+    }\n+  }\n+\n+  /**\n+   * Move the cached file from user temp directory to data directory\n+   * \n+   * @param userId\n+   *          id of the user\n+   * @param blockId\n+   *          id of the block\n+   * @return true if success, false otherwise\n+   * @throws IOException\n+   */\n+  public boolean cacheBlock(long userId, long blockId) throws IOException {\n+    String srcPath = getUserTempFilePath(userId, blockId);\n+    String destPath = getBlockFilePath(blockId);\n+    boolean result = mUFS.rename(srcPath, destPath);\n+    if (result) {\n+      addBlockId(blockId, getFileSize(blockId));\n+    }\n+    return result;\n+  }\n+\n+  /**\n+   * Check status of the users\n+   * \n+   * @param removedUsers\n+   *          id of the removed users\n+   */\n+  public void checkStatus(List<Long> removedUsers) {\n+    for (long userId : removedUsers) {","path":"core/src/main/java/tachyon/worker/hierarchy/StorageDir.java","position":125,"original_position":125,"commit_id":"e4d5f0f0aed3990dd9f6a8f29d64d1a337f05960","original_commit_id":"e4d5f0f0aed3990dd9f6a8f29d64d1a337f05960","user":{"login":"dcapwell","id":1852601,"avatar_url":"https://avatars.githubusercontent.com/u/1852601?v=2","gravatar_id":"136010620e86ea76cd82ee1f865b5a2e","url":"https://api.github.com/users/dcapwell","html_url":"https://github.com/dcapwell","followers_url":"https://api.github.com/users/dcapwell/followers","following_url":"https://api.github.com/users/dcapwell/following{/other_user}","gists_url":"https://api.github.com/users/dcapwell/gists{/gist_id}","starred_url":"https://api.github.com/users/dcapwell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dcapwell/subscriptions","organizations_url":"https://api.github.com/users/dcapwell/orgs","repos_url":"https://api.github.com/users/dcapwell/repos","events_url":"https://api.github.com/users/dcapwell/events{/privacy}","received_events_url":"https://api.github.com/users/dcapwell/received_events","type":"User","site_admin":false},"body":"Ask because with List, there are different ways to iterate in a thread-safe matter, but that differs from implementations.  Since we are only working with List, we don't know if the caller can update, which makes this loop unsafe.\r\n\r\nMight be good to switch to guava's `ImmutableList` as a means to product this code.","created_at":"2014-08-14T18:22:14Z","updated_at":"2014-08-14T18:22:14Z","html_url":"https://github.com/amplab/tachyon/pull/361#discussion_r16256124","pull_request_url":"https://api.github.com/repos/amplab/tachyon/pulls/361","_links":{"self":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/comments/16256124"},"html":{"href":"https://github.com/amplab/tachyon/pull/361#discussion_r16256124"},"pull_request":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/361"}}},"pull_request":{"url":"https://api.github.com/repos/amplab/tachyon/pulls/361","id":19768384,"html_url":"https://github.com/amplab/tachyon/pull/361","diff_url":"https://github.com/amplab/tachyon/pull/361.diff","patch_url":"https://github.com/amplab/tachyon/pull/361.patch","issue_url":"https://api.github.com/repos/amplab/tachyon/issues/361","number":361,"state":"open","locked":false,"title":"Add storage dir and  eviction strategy [TACHYON-33][SubTask 2 & 3]","user":{"login":"shimingfei","id":2638130,"avatar_url":"https://avatars.githubusercontent.com/u/2638130?v=2","gravatar_id":null,"url":"https://api.github.com/users/shimingfei","html_url":"https://github.com/shimingfei","followers_url":"https://api.github.com/users/shimingfei/followers","following_url":"https://api.github.com/users/shimingfei/following{/other_user}","gists_url":"https://api.github.com/users/shimingfei/gists{/gist_id}","starred_url":"https://api.github.com/users/shimingfei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shimingfei/subscriptions","organizations_url":"https://api.github.com/users/shimingfei/orgs","repos_url":"https://api.github.com/users/shimingfei/repos","events_url":"https://api.github.com/users/shimingfei/events{/privacy}","received_events_url":"https://api.github.com/users/shimingfei/received_events","type":"User","site_admin":false},"body":"details are on Tachyon's JIRA\r\nhttps://tachyon.atlassian.net/browse/TACHYON-33\r\nstorage dir is the container which manages and stores the data on different storage tiers. it also maintains the data migration between different storage dirs(on same or different storage tiers).\r\nif current storage tier has no enough space for new coming block, some blocks needs to be swapped out to its successor storage tier, to get enough space. Evict strategy defines how to choose the blocks to be evicted. currently LRU based eviction strategies are implemented. an interface is also exposed for other eviction strategy implementation","created_at":"2014-08-14T05:04:37Z","updated_at":"2014-08-14T18:22:14Z","closed_at":null,"merged_at":null,"merge_commit_sha":"fe88b866e772e0f808d741f66d90dfb46bd2b6cf","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/amplab/tachyon/pulls/361/commits","review_comments_url":"https://api.github.com/repos/amplab/tachyon/pulls/361/comments","review_comment_url":"https://api.github.com/repos/amplab/tachyon/pulls/comments/{number}","comments_url":"https://api.github.com/repos/amplab/tachyon/issues/361/comments","statuses_url":"https://api.github.com/repos/amplab/tachyon/statuses/e4d5f0f0aed3990dd9f6a8f29d64d1a337f05960","head":{"label":"shimingfei:AddEvictionStrategy","ref":"AddEvictionStrategy","sha":"e4d5f0f0aed3990dd9f6a8f29d64d1a337f05960","user":{"login":"shimingfei","id":2638130,"avatar_url":"https://avatars.githubusercontent.com/u/2638130?v=2","gravatar_id":null,"url":"https://api.github.com/users/shimingfei","html_url":"https://github.com/shimingfei","followers_url":"https://api.github.com/users/shimingfei/followers","following_url":"https://api.github.com/users/shimingfei/following{/other_user}","gists_url":"https://api.github.com/users/shimingfei/gists{/gist_id}","starred_url":"https://api.github.com/users/shimingfei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shimingfei/subscriptions","organizations_url":"https://api.github.com/users/shimingfei/orgs","repos_url":"https://api.github.com/users/shimingfei/repos","events_url":"https://api.github.com/users/shimingfei/events{/privacy}","received_events_url":"https://api.github.com/users/shimingfei/received_events","type":"User","site_admin":false},"repo":{"id":20919861,"name":"tachyon","full_name":"shimingfei/tachyon","owner":{"login":"shimingfei","id":2638130,"avatar_url":"https://avatars.githubusercontent.com/u/2638130?v=2","gravatar_id":null,"url":"https://api.github.com/users/shimingfei","html_url":"https://github.com/shimingfei","followers_url":"https://api.github.com/users/shimingfei/followers","following_url":"https://api.github.com/users/shimingfei/following{/other_user}","gists_url":"https://api.github.com/users/shimingfei/gists{/gist_id}","starred_url":"https://api.github.com/users/shimingfei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shimingfei/subscriptions","organizations_url":"https://api.github.com/users/shimingfei/orgs","repos_url":"https://api.github.com/users/shimingfei/repos","events_url":"https://api.github.com/users/shimingfei/events{/privacy}","received_events_url":"https://api.github.com/users/shimingfei/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/shimingfei/tachyon","description":"Memory-centric Storage System for Big Data Analytics","fork":true,"url":"https://api.github.com/repos/shimingfei/tachyon","forks_url":"https://api.github.com/repos/shimingfei/tachyon/forks","keys_url":"https://api.github.com/repos/shimingfei/tachyon/keys{/key_id}","collaborators_url":"https://api.github.com/repos/shimingfei/tachyon/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/shimingfei/tachyon/teams","hooks_url":"https://api.github.com/repos/shimingfei/tachyon/hooks","issue_events_url":"https://api.github.com/repos/shimingfei/tachyon/issues/events{/number}","events_url":"https://api.github.com/repos/shimingfei/tachyon/events","assignees_url":"https://api.github.com/repos/shimingfei/tachyon/assignees{/user}","branches_url":"https://api.github.com/repos/shimingfei/tachyon/branches{/branch}","tags_url":"https://api.github.com/repos/shimingfei/tachyon/tags","blobs_url":"https://api.github.com/repos/shimingfei/tachyon/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/shimingfei/tachyon/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/shimingfei/tachyon/git/refs{/sha}","trees_url":"https://api.github.com/repos/shimingfei/tachyon/git/trees{/sha}","statuses_url":"https://api.github.com/repos/shimingfei/tachyon/statuses/{sha}","languages_url":"https://api.github.com/repos/shimingfei/tachyon/languages","stargazers_url":"https://api.github.com/repos/shimingfei/tachyon/stargazers","contributors_url":"https://api.github.com/repos/shimingfei/tachyon/contributors","subscribers_url":"https://api.github.com/repos/shimingfei/tachyon/subscribers","subscription_url":"https://api.github.com/repos/shimingfei/tachyon/subscription","commits_url":"https://api.github.com/repos/shimingfei/tachyon/commits{/sha}","git_commits_url":"https://api.github.com/repos/shimingfei/tachyon/git/commits{/sha}","comments_url":"https://api.github.com/repos/shimingfei/tachyon/comments{/number}","issue_comment_url":"https://api.github.com/repos/shimingfei/tachyon/issues/comments/{number}","contents_url":"https://api.github.com/repos/shimingfei/tachyon/contents/{+path}","compare_url":"https://api.github.com/repos/shimingfei/tachyon/compare/{base}...{head}","merges_url":"https://api.github.com/repos/shimingfei/tachyon/merges","archive_url":"https://api.github.com/repos/shimingfei/tachyon/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/shimingfei/tachyon/downloads","issues_url":"https://api.github.com/repos/shimingfei/tachyon/issues{/number}","pulls_url":"https://api.github.com/repos/shimingfei/tachyon/pulls{/number}","milestones_url":"https://api.github.com/repos/shimingfei/tachyon/milestones{/number}","notifications_url":"https://api.github.com/repos/shimingfei/tachyon/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/shimingfei/tachyon/labels{/name}","releases_url":"https://api.github.com/repos/shimingfei/tachyon/releases{/id}","created_at":"2014-06-17T10:50:29Z","updated_at":"2014-06-24T23:54:16Z","pushed_at":"2014-08-14T05:02:47Z","git_url":"git://github.com/shimingfei/tachyon.git","ssh_url":"git@github.com:shimingfei/tachyon.git","clone_url":"https://github.com/shimingfei/tachyon.git","svn_url":"https://github.com/shimingfei/tachyon","homepage":"http://tachyon-project.org","size":6860,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"amplab:master","ref":"master","sha":"954be4f66e3319cd1068f72a87c3a3dfa96e2c5c","user":{"login":"amplab","id":1438454,"avatar_url":"https://avatars.githubusercontent.com/u/1438454?v=2","gravatar_id":"c0783723b006eb46526c04fa47ba2cd7","url":"https://api.github.com/users/amplab","html_url":"https://github.com/amplab","followers_url":"https://api.github.com/users/amplab/followers","following_url":"https://api.github.com/users/amplab/following{/other_user}","gists_url":"https://api.github.com/users/amplab/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab/subscriptions","organizations_url":"https://api.github.com/users/amplab/orgs","repos_url":"https://api.github.com/users/amplab/repos","events_url":"https://api.github.com/users/amplab/events{/privacy}","received_events_url":"https://api.github.com/users/amplab/received_events","type":"Organization","site_admin":false},"repo":{"id":7276954,"name":"tachyon","full_name":"amplab/tachyon","owner":{"login":"amplab","id":1438454,"avatar_url":"https://avatars.githubusercontent.com/u/1438454?v=2","gravatar_id":"c0783723b006eb46526c04fa47ba2cd7","url":"https://api.github.com/users/amplab","html_url":"https://github.com/amplab","followers_url":"https://api.github.com/users/amplab/followers","following_url":"https://api.github.com/users/amplab/following{/other_user}","gists_url":"https://api.github.com/users/amplab/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab/subscriptions","organizations_url":"https://api.github.com/users/amplab/orgs","repos_url":"https://api.github.com/users/amplab/repos","events_url":"https://api.github.com/users/amplab/events{/privacy}","received_events_url":"https://api.github.com/users/amplab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/amplab/tachyon","description":" A Reliable Memory Centric Storage for Big Data Analytics","fork":false,"url":"https://api.github.com/repos/amplab/tachyon","forks_url":"https://api.github.com/repos/amplab/tachyon/forks","keys_url":"https://api.github.com/repos/amplab/tachyon/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amplab/tachyon/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amplab/tachyon/teams","hooks_url":"https://api.github.com/repos/amplab/tachyon/hooks","issue_events_url":"https://api.github.com/repos/amplab/tachyon/issues/events{/number}","events_url":"https://api.github.com/repos/amplab/tachyon/events","assignees_url":"https://api.github.com/repos/amplab/tachyon/assignees{/user}","branches_url":"https://api.github.com/repos/amplab/tachyon/branches{/branch}","tags_url":"https://api.github.com/repos/amplab/tachyon/tags","blobs_url":"https://api.github.com/repos/amplab/tachyon/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amplab/tachyon/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amplab/tachyon/git/refs{/sha}","trees_url":"https://api.github.com/repos/amplab/tachyon/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amplab/tachyon/statuses/{sha}","languages_url":"https://api.github.com/repos/amplab/tachyon/languages","stargazers_url":"https://api.github.com/repos/amplab/tachyon/stargazers","contributors_url":"https://api.github.com/repos/amplab/tachyon/contributors","subscribers_url":"https://api.github.com/repos/amplab/tachyon/subscribers","subscription_url":"https://api.github.com/repos/amplab/tachyon/subscription","commits_url":"https://api.github.com/repos/amplab/tachyon/commits{/sha}","git_commits_url":"https://api.github.com/repos/amplab/tachyon/git/commits{/sha}","comments_url":"https://api.github.com/repos/amplab/tachyon/comments{/number}","issue_comment_url":"https://api.github.com/repos/amplab/tachyon/issues/comments/{number}","contents_url":"https://api.github.com/repos/amplab/tachyon/contents/{+path}","compare_url":"https://api.github.com/repos/amplab/tachyon/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amplab/tachyon/merges","archive_url":"https://api.github.com/repos/amplab/tachyon/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amplab/tachyon/downloads","issues_url":"https://api.github.com/repos/amplab/tachyon/issues{/number}","pulls_url":"https://api.github.com/repos/amplab/tachyon/pulls{/number}","milestones_url":"https://api.github.com/repos/amplab/tachyon/milestones{/number}","notifications_url":"https://api.github.com/repos/amplab/tachyon/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amplab/tachyon/labels{/name}","releases_url":"https://api.github.com/repos/amplab/tachyon/releases{/id}","created_at":"2012-12-21T17:43:46Z","updated_at":"2014-08-13T13:12:13Z","pushed_at":"2014-08-14T17:33:31Z","git_url":"git://github.com/amplab/tachyon.git","ssh_url":"git@github.com:amplab/tachyon.git","clone_url":"https://github.com/amplab/tachyon.git","svn_url":"https://github.com/amplab/tachyon","homepage":"http://tachyon-project.org","size":26106,"stargazers_count":894,"watchers_count":894,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"forks_count":244,"mirror_url":null,"open_issues_count":18,"forks":244,"open_issues":18,"watchers":894,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/361"},"html":{"href":"https://github.com/amplab/tachyon/pull/361"},"issue":{"href":"https://api.github.com/repos/amplab/tachyon/issues/361"},"comments":{"href":"https://api.github.com/repos/amplab/tachyon/issues/361/comments"},"review_comments":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/361/comments"},"review_comment":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/361/commits"},"statuses":{"href":"https://api.github.com/repos/amplab/tachyon/statuses/e4d5f0f0aed3990dd9f6a8f29d64d1a337f05960"}}}}