Repository: pantsbuild/pants

Issue:
Make ivy and coursier cache links snapshottable
This code for ivy:
https://github.com/pantsbuild/pants/blob/5f36f1548023b28a8c6ecbf330b2612aefa22fef/src/python/pants/backend/jvm/ivy_utils.py#L829-L871

And this code for coursier: https://github.com/pantsbuild/pants/blob/5f36f1548023b28a8c6ecbf330b2612aefa22fef/src/python/pants/backend/jvm/tasks/coursier_resolve.py#L612-L614

Make trees of symlinks pointing at jar files in central cache directories.

Unfortunately, symlinks don't play nicely with v2 snapshotting and remote execution.

Long-term, we're going to want to put these fetches natively into v2, but short term, we need to be able to get at these files using the existing codepaths.

Let's change the behavior here so that we don't have symlinks. The desired end-state is:
If we can hardlink where we currently symlink, do so.
If not, copy the file.

Probably the easiest way to do this is to attempt to hardlink, and if that throws, instead copy.

Let's not worry about replacing existing symlinks; I think it's reasonable for people to have to `clean-all` one time if they're starting to use remote execution.


Discussion:
How can I test it once fixed?
To test snapshotting explicitly:

`cd src/rust/engine && ../../../build-support/bin/native/cargo build -p fs_util` that will produce `target/debug/fs_util`. If you then: `target/debug/fs_util --local-store-path="${HOME}/.cache/pants/lmdb_store" directory save --root=/path/to/.pants.d 'ivy/jars/**'` it should work and not error. I expect this to error right now, but if you `rm -rf .pants.d` and build something which has 3rdparty deps, it should then succeed.

To test the general effect, rather than whether snapshotting works:
`rm -rf .pants.d` followed by building something with deps, followed by `find .pants.d/ivy/jars -type f` should show results, but `find .pants.d/ivy/jars -type l` should show no results.

Resolving patch:
diff --git a/src/python/pants/backend/jvm/ivy_utils.py b/src/python/pants/backend/jvm/ivy_utils.py
--- a/src/python/pants/backend/jvm/ivy_utils.py
+++ b/src/python/pants/backend/jvm/ivy_utils.py
@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import errno
 import json
 import logging
 import os
@@ -32,7 +31,7 @@
 from pants.java.jar.jar_dependency_utils import M2Coordinate, ResolvedJar
 from pants.java.util import execute_runner
 from pants.util.dirutil import safe_concurrent_creation, safe_mkdir, safe_open
-from pants.util.fileutil import atomic_copy
+from pants.util.fileutil import atomic_copy, safe_hardlink_or_copy
 
 
 class IvyResolutionStep(object):
@@ -86,19 +85,19 @@ def workdir(self):
     return os.path.join(self.global_ivy_workdir, self.hash_name)
 
   @property
-  def symlink_classpath_filename(self):
+  def hardlink_classpath_filename(self):
     return os.path.join(self.workdir, 'classpath')
 
   @property
   def ivy_cache_classpath_filename(self):
-    return '{}.raw'.format(self.symlink_classpath_filename)
+    return '{}.raw'.format(self.hardlink_classpath_filename)
 
   @property
   def frozen_resolve_file(self):
     return os.path.join(self.workdir, 'resolution.json')
 
   @property
-  def symlink_dir(self):
+  def hardlink_dir(self):
     return os.path.join(self.global_ivy_workdir, 'jars')
 
   @abstractmethod
@@ -109,13 +108,13 @@ def ivy_xml_path(self):
   def resolve_report_path(self, conf):
     """Location of the resolve report in the workdir."""
 
-  def _construct_and_load_symlink_map(self):
-    artifact_paths, symlink_map = IvyUtils.construct_and_load_symlink_map(
-      self.symlink_dir,
+  def _construct_and_load_hardlink_map(self):
+    artifact_paths, hardlink_map = IvyUtils.construct_and_load_hardlink_map(
+      self.hardlink_dir,
       self.ivy_repository_cache_dir,
       self.ivy_cache_classpath_filename,
-      self.symlink_classpath_filename)
-    return artifact_paths, symlink_map
+      self.hardlink_classpath_filename)
+    return artifact_paths, hardlink_map
 
   def _call_ivy(self, executor, extra_args, ivyxml, jvm_options, hash_name_for_report,
                 workunit_factory, workunit_name):
@@ -179,9 +178,9 @@ def exec_and_load(self, executor, extra_args, targets, jvm_options, workunit_nam
     return result
 
   def _load_from_fetch(self, frozen_resolutions):
-    artifact_paths, symlink_map = self._construct_and_load_symlink_map()
+    artifact_paths, hardlink_map = self._construct_and_load_hardlink_map()
     return IvyFetchResolveResult(artifact_paths,
-                                 symlink_map,
+                                 hardlink_map,
                                  self.hash_name,
                                  self.workdir_reports_by_conf,
                                  frozen_resolutions)
@@ -227,9 +226,9 @@ def ivy_xml_path(self):
     return os.path.join(self.workdir, 'resolve-ivy.xml')
 
   def load(self, targets):
-    artifact_paths, symlink_map = self._construct_and_load_symlink_map()
+    artifact_paths, hardlink_map = self._construct_and_load_hardlink_map()
     return IvyResolveResult(artifact_paths,
-                            symlink_map,
+                            hardlink_map,
                             self.hash_name,
                             self.workdir_reports_by_conf)
 
@@ -381,11 +380,11 @@ class IvyResolveResult(object):
   and the targets that requested them and the hash name of the resolve.
   """
 
-  def __init__(self, resolved_artifact_paths, symlink_map, resolve_hash_name, reports_by_conf):
+  def __init__(self, resolved_artifact_paths, hardlink_map, resolve_hash_name, reports_by_conf):
     self._reports_by_conf = reports_by_conf
     self.resolved_artifact_paths = resolved_artifact_paths
     self.resolve_hash_name = resolve_hash_name
-    self._symlink_map = symlink_map
+    self._hardlink_map = hardlink_map
 
   @property
   def has_resolved_artifacts(self):
@@ -437,7 +436,7 @@ def resolved_jars_for_each_target(self, conf, targets):
     ivy_jar_memo = {}
     for target in jar_library_targets:
       # Add the artifacts from each dependency module.
-      resolved_jars = self._resolved_jars_with_symlinks(conf, ivy_info, ivy_jar_memo,
+      resolved_jars = self._resolved_jars_with_hardlinks(conf, ivy_info, ivy_jar_memo,
                                                         self._jar_dependencies_for_target(conf,
                                                                                           target),
                                                         target)
@@ -450,33 +449,33 @@ def _ivy_info_for(self, conf):
     report_path = self._reports_by_conf.get(conf)
     return IvyUtils.parse_xml_report(conf, report_path)
 
-  def _new_resolved_jar_with_symlink_path(self, conf, target, resolved_jar_without_symlink):
+  def _new_resolved_jar_with_hardlink_path(self, conf, target, resolved_jar_without_hardlink):
     def candidate_cache_paths():
       # There is a focus on being lazy here to avoid `os.path.realpath` when we can.
-      yield resolved_jar_without_symlink.cache_path
-      yield os.path.realpath(resolved_jar_without_symlink.cache_path)
+      yield resolved_jar_without_hardlink.cache_path
+      yield os.path.realpath(resolved_jar_without_hardlink.cache_path)
 
     for cache_path in candidate_cache_paths():
-      pants_path = self._symlink_map.get(cache_path)
+      pants_path = self._hardlink_map.get(cache_path)
       if pants_path:
         break
     else:
 
       raise IvyResolveMappingError(
         'Jar {resolved_jar} in {spec} not resolved to the ivy '
-        'symlink map in conf {conf}.'
+        'hardlink map in conf {conf}.'
           .format(spec=target.address.spec,
-                  resolved_jar=resolved_jar_without_symlink.cache_path,
+                  resolved_jar=resolved_jar_without_hardlink.cache_path,
                   conf=conf))
 
-    return ResolvedJar(coordinate=resolved_jar_without_symlink.coordinate,
+    return ResolvedJar(coordinate=resolved_jar_without_hardlink.coordinate,
                        pants_path=pants_path,
-                       cache_path=resolved_jar_without_symlink.cache_path)
+                       cache_path=resolved_jar_without_hardlink.cache_path)
 
-  def _resolved_jars_with_symlinks(self, conf, ivy_info, ivy_jar_memo, coordinates, target):
+  def _resolved_jars_with_hardlinks(self, conf, ivy_info, ivy_jar_memo, coordinates, target):
     raw_resolved_jars = ivy_info.get_resolved_jars_for_coordinates(coordinates,
                                                                    memo=ivy_jar_memo)
-    resolved_jars = [self._new_resolved_jar_with_symlink_path(conf, target, raw_resolved_jar)
+    resolved_jars = [self._new_resolved_jar_with_hardlink_path(conf, target, raw_resolved_jar)
                      for raw_resolved_jar in raw_resolved_jars]
     return resolved_jars
 
@@ -484,9 +483,9 @@ def _resolved_jars_with_symlinks(self, conf, ivy_info, ivy_jar_memo, coordinates
 class IvyFetchResolveResult(IvyResolveResult):
   """A resolve result that uses the frozen resolution to look up dependencies."""
 
-  def __init__(self, resolved_artifact_paths, symlink_map, resolve_hash_name, reports_by_conf,
+  def __init__(self, resolved_artifact_paths, hardlink_map, resolve_hash_name, reports_by_conf,
                frozen_resolutions):
-    super(IvyFetchResolveResult, self).__init__(resolved_artifact_paths, symlink_map,
+    super(IvyFetchResolveResult, self).__init__(resolved_artifact_paths, hardlink_map,
                                                 resolve_hash_name, reports_by_conf)
     self._frozen_resolutions = frozen_resolutions
 
@@ -693,8 +692,8 @@ class IvyUtils(object):
   # Protects ivy executions.
   _ivy_lock = threading.RLock()
 
-  # Protect writes to the global map of jar path -> symlinks to that jar.
-  _symlink_map_lock = threading.Lock()
+  # Protect writes to the global map of jar path -> hardlinks to that jar.
+  _hardlink_map_lock = threading.Lock()
 
   INTERNAL_ORG_NAME = 'internal'
 
@@ -810,66 +809,61 @@ def _exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor,
       raise IvyUtils.IvyError(e)
 
   @classmethod
-  def construct_and_load_symlink_map(cls, symlink_dir, ivy_repository_cache_dir,
-                                     ivy_cache_classpath_filename, symlink_classpath_filename):
-    # Make our actual classpath be symlinks, so that the paths are uniform across systems.
+  def construct_and_load_hardlink_map(cls, hardlink_dir, ivy_repository_cache_dir,
+                                     ivy_cache_classpath_filename, hardlink_classpath_filename):
+    # Make our actual classpath be hardlinks, so that the paths are uniform across systems.
     # Note that we must do this even if we read the raw_target_classpath_file from the artifact
-    # cache. If we cache the target_classpath_file we won't know how to create the symlinks.
-    with IvyUtils._symlink_map_lock:
-      # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
+    # cache. If we cache the target_classpath_file we won't know how to create the hardlinks.
+    with IvyUtils._hardlink_map_lock:
+      # A common dir for hardlinks into the ivy2 cache. This ensures that paths to jars
       # in artifact-cached analysis files are consistent across systems.
-      # Note that we have one global, well-known symlink dir, again so that paths are
+      # Note that we have one global, well-known hardlink dir, again so that paths are
       # consistent across builds.
-      symlink_map = cls._symlink_cachepath(ivy_repository_cache_dir,
+      hardlink_map = cls._hardlink_cachepath(ivy_repository_cache_dir,
                                            ivy_cache_classpath_filename,
-                                           symlink_dir,
-                                           symlink_classpath_filename)
-    classpath = cls._load_classpath_from_cachepath(symlink_classpath_filename)
-    return classpath, symlink_map
+                                           hardlink_dir,
+                                           hardlink_classpath_filename)
+    classpath = cls._load_classpath_from_cachepath(hardlink_classpath_filename)
+    return classpath, hardlink_map
 
   @classmethod
-  def _symlink_cachepath(cls, ivy_repository_cache_dir, inpath, symlink_dir, outpath):
-    """Symlinks all paths listed in inpath that are under ivy_repository_cache_dir into symlink_dir.
+  def _hardlink_cachepath(cls, ivy_repository_cache_dir, inpath, hardlink_dir, outpath):
+    """hardlinks all paths listed in inpath that are under ivy_repository_cache_dir into hardlink_dir.
 
-    If there is an existing symlink for a file under inpath, it is used rather than creating
-    a new symlink. Preserves all other paths. Writes the resulting paths to outpath.
-    Returns a map of path -> symlink to that path.
+    If there is an existing hardlink for a file under inpath, it is used rather than creating
+    a new hardlink. Preserves all other paths. Writes the resulting paths to outpath.
+    Returns a map of path -> hardlink to that path.
     """
-    safe_mkdir(symlink_dir)
-    # The ivy_repository_cache_dir might itself be a symlink. In this case, ivy may return paths that
+    safe_mkdir(hardlink_dir)
+    # The ivy_repository_cache_dir might itself be a hardlink. In this case, ivy may return paths that
     # reference the realpath of the .jar file after it is resolved in the cache dir. To handle
-    # this case, add both the symlink'ed path and the realpath to the jar to the symlink map.
+    # this case, add both the hardlink'ed path and the realpath to the jar to the hardlink map.
     real_ivy_cache_dir = os.path.realpath(ivy_repository_cache_dir)
-    symlink_map = OrderedDict()
+    hardlink_map = OrderedDict()
 
     inpaths = cls._load_classpath_from_cachepath(inpath)
     paths = OrderedSet([os.path.realpath(path) for path in inpaths])
 
     for path in paths:
       if path.startswith(real_ivy_cache_dir):
-        symlink_map[path] = os.path.join(symlink_dir, os.path.relpath(path, real_ivy_cache_dir))
+        hardlink_map[path] = os.path.join(hardlink_dir, os.path.relpath(path, real_ivy_cache_dir))
       else:
-        # This path is outside the cache. We won't symlink it.
-        symlink_map[path] = path
+        # This path is outside the cache. We won't hardlink it.
+        hardlink_map[path] = path
 
-    # Create symlinks for paths in the ivy cache dir.
-    for path, symlink in six.iteritems(symlink_map):
-      if path == symlink:
-        # Skip paths that aren't going to be symlinked.
+    # Create hardlinks for paths in the ivy cache dir.
+    for path, hardlink in six.iteritems(hardlink_map):
+      if path == hardlink:
+        # Skip paths that aren't going to be hardlinked.
         continue
-      safe_mkdir(os.path.dirname(symlink))
-      try:
-        os.symlink(path, symlink)
-      except OSError as e:
-        # We don't delete and recreate the symlink, as this may break concurrently executing code.
-        if e.errno != errno.EEXIST:
-          raise
+      safe_mkdir(os.path.dirname(hardlink))
+      safe_hardlink_or_copy(path, hardlink)
 
     # (re)create the classpath with all of the paths
     with safe_open(outpath, 'w') as outfile:
-      outfile.write(':'.join(OrderedSet(symlink_map.values())))
+      outfile.write(':'.join(OrderedSet(hardlink_map.values())))
 
-    return dict(symlink_map)
+    return dict(hardlink_map)
 
   @classmethod
   def xml_report_path(cls, resolution_cache_dir, resolve_hash_name, conf):
diff --git a/src/python/pants/backend/jvm/tasks/coursier_resolve.py b/src/python/pants/backend/jvm/tasks/coursier_resolve.py
--- a/src/python/pants/backend/jvm/tasks/coursier_resolve.py
+++ b/src/python/pants/backend/jvm/tasks/coursier_resolve.py
@@ -29,6 +29,7 @@
 from pants.java.jar.jar_dependency_utils import M2Coordinate, ResolvedJar
 from pants.util.contextutil import temporary_file
 from pants.util.dirutil import safe_mkdir
+from pants.util.fileutil import safe_hardlink_or_copy
 
 
 class CoursierResultNotFound(Exception):
@@ -191,7 +192,7 @@ def _prepare_vts_results_dir(self, pants_workdir, vts):
 
   def _prepare_workdir(self, pants_workdir):
     """
-    Given pants workdir, prepare the location in pants workdir to store all the symlinks
+    Given pants workdir, prepare the location in pants workdir to store all the hardlinks
     to coursier cache dir.
     """
     pants_jar_base_dir = os.path.join(pants_workdir, 'coursier', 'cache')
@@ -424,7 +425,7 @@ def _load_json_result(self, conf, compile_classpath, coursier_cache_path, invali
     :param compile_classpath: `ClasspathProducts` that will be modified
     :param coursier_cache_path: cache location that is managed by coursier
     :param invalidation_check: InvalidationCheck
-    :param pants_jar_path_base: location under pants workdir that contains all the symlinks to coursier cache
+    :param pants_jar_path_base: location under pants workdir that contains all the hardlinks to coursier cache
     :param result: result dict converted from the json produced by one coursier run
     :return: n/a
     """
@@ -592,7 +593,7 @@ def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path
 
     :param result: coursier json output
     :param coursier_cache_path: coursier cache location
-    :param pants_jar_path_base: location under pants workdir to store the symlink to the coursier cache
+    :param pants_jar_path_base: location under pants workdir to store the hardlink to the coursier cache
     :return: a map from maven coordinate to a resolved jar.
     """
 
@@ -613,7 +614,7 @@ def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path
 
       if not os.path.exists(pants_path):
         safe_mkdir(os.path.dirname(pants_path))
-        os.symlink(jar_path, pants_path)
+        safe_hardlink_or_copy(jar_path, pants_path)
 
       coord = cls.to_m2_coord(coord)
       resolved_jar = ResolvedJar(coord,
@@ -632,7 +633,7 @@ def _get_path_to_jar(cls, coursier_cache_path, pants_jar_path_base, jar_path):
     Create the path to the jar that will live in .pants.d
     
     :param coursier_cache_path: coursier cache location
-    :param pants_jar_path_base: location under pants workdir to store the symlink to the coursier cache
+    :param pants_jar_path_base: location under pants workdir to store the hardlink to the coursier cache
     :param jar_path: path of the jar
     :return:
     """
diff --git a/src/python/pants/util/fileutil.py b/src/python/pants/util/fileutil.py
--- a/src/python/pants/util/fileutil.py
+++ b/src/python/pants/util/fileutil.py
@@ -4,9 +4,11 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import errno
 import os
 import random
 import shutil
+from uuid import uuid4
 
 from pants.util.contextutil import temporary_file
 
@@ -37,3 +39,23 @@ def line_count(filename):
     'nosize': lambda srcs: 0,
     'random': lambda srcs: random.randint(0, 10000),
   }
+
+
+def safe_hardlink_or_copy(source, dest, overwrite=False):
+  def do_copy():
+    temp_dest = dest + uuid4().hex
+    shutil.copyfile(source, temp_dest)
+    os.rename(temp_dest, dest)
+
+  try:
+    os.link(source, dest)
+  except OSError as e:
+    if e.errno == errno.EEXIST:
+      # File already exists.  If overwrite=True, write otherwise skip.
+      if overwrite:
+        do_copy()
+    elif e.errno == errno.EXDEV:
+      # Hard link across devices, fall back on copying
+      do_copy()
+    else:
+      raise


Test patch:
diff --git a/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py b/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py
--- a/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py
+++ b/tests/python/pants_test/backend/jvm/tasks/test_coursier_resolve.py
@@ -25,6 +25,7 @@
 from pants.java.jar.jar_dependency import JarDependency
 from pants.task.task import Task
 from pants.util.contextutil import temporary_dir, temporary_file_path
+from pants.util.dirutil import safe_rmtree
 from pants_test.jvm.jvm_tool_task_test_base import JvmToolTaskTestBase
 from pants_test.subsystem.subsystem_util import init_subsystem
 from pants_test.task_test_base import TaskTestBase
@@ -216,65 +217,41 @@ def test_second_noop_does_not_invoke_coursier(self):
       task.execute()
       task.runjava.assert_not_called()
 
-  def test_when_invalid_artifact_symlink_should_trigger_resolve(self):
+  def test_when_invalid_hardlink_and_coursier_cache_should_trigger_resolve(self):
     jar_lib = self._make_junit_target()
     with self._temp_workdir():
+      with temporary_dir() as couriser_cache_dir:
+        self.set_options_for_scope('coursier', cache_dir=couriser_cache_dir)
 
-      context = self.context(target_roots=[jar_lib])
-      task = self.execute(context)
-      compile_classpath = context.products.get_data('compile_classpath')
+        context = self.context(target_roots=[jar_lib])
+        task = self.execute(context)
+        compile_classpath = context.products.get_data('compile_classpath')
 
-      jar_cp = compile_classpath.get_for_target(jar_lib)
+        jar_cp = compile_classpath.get_for_target(jar_lib)
 
-      # └─ junit:junit:4.12
-      #    └─ org.hamcrest:hamcrest-core:1.3
-      self.assertEquals(2, len(jar_cp))
+        # └─ junit:junit:4.12
+        #    └─ org.hamcrest:hamcrest-core:1.3
+        self.assertEquals(2, len(jar_cp))
 
-      # Take a sample jar path, remove it, then call the task again, it should invoke coursier again
-      conf, path = jar_cp[0]
-      safe_remove(os.path.realpath(path))
 
-      task.runjava = MagicMock()
-
-      # Ignore any error because runjava may fail due to undefined behavior
-      try:
-        task.execute()
-      except TaskError:
-        pass
-
-      task.runjava.assert_called()
-
-  def test_when_invalid_coursier_cache_should_trigger_resolve(self):
-    jar_lib = self._make_junit_target()
-    with self._temp_workdir():
+        # Take a sample jar path, remove it, then call the task again, it should invoke coursier again
+        conf, path = jar_cp[0]
 
-      context = self.context(target_roots=[jar_lib])
-      task = self.execute(context)
-      compile_classpath = context.products.get_data('compile_classpath')
+        # Remove the hard link under .pants.d/
+        safe_remove(path)
 
-      jar_cp = compile_classpath.get_for_target(jar_lib)
+        # Remove coursier's cache
+        safe_rmtree(couriser_cache_dir)
 
-      # └─ junit:junit:4.12
-      #    └─ org.hamcrest:hamcrest-core:1.3
-      self.assertEquals(2, len(jar_cp))
-
-
-      # Take a sample jar path, remove it, then call the task again, it should invoke coursier again
-      conf, path = jar_cp[0]
-
-      self.assertTrue(os.path.islink(path))
-
-      safe_remove(os.path.realpath(path))
-
-      task.runjava = MagicMock()
+        task.runjava = MagicMock()
 
-      # Ignore any error because runjava may fail due to undefined behavior
-      try:
-        task.execute()
-      except TaskError:
-        pass
+        # Ignore any error because runjava may fail due to undefined behavior
+        try:
+          task.execute()
+        except TaskError:
+          pass
 
-      task.runjava.assert_called()
+        task.runjava.assert_called()
 
   def test_resolve_jarless_pom(self):
     jar = JarDependency('org.apache.commons', 'commons-weaver-privilizer-parent', '1.3')
diff --git a/tests/python/pants_test/backend/jvm/tasks/test_ivy_imports.py b/tests/python/pants_test/backend/jvm/tasks/test_ivy_imports.py
--- a/tests/python/pants_test/backend/jvm/tasks/test_ivy_imports.py
+++ b/tests/python/pants_test/backend/jvm/tasks/test_ivy_imports.py
@@ -65,6 +65,6 @@ def verify_product_mapping(self, jar_import_products, target, expected_coordinat
     jars_by_coordinate = dict(jar_import_products.imports(target))
     self.assertIn(expected_coordinate, jars_by_coordinate)
     jar_filename = jars_by_coordinate[expected_coordinate]
-    self.assertTrue(os.path.islink(jar_filename))
+    self.assertFalse(os.path.islink(jar_filename))
     # Make sure there is a real .jar there
     self.assertTrue(zipfile.is_zipfile(jar_filename))
diff --git a/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py b/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py
--- a/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py
+++ b/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py
@@ -332,10 +332,10 @@ def find_single(self, elem, xpath):
   def assert_attributes(self, elem, **kwargs):
     self.assertEqual(dict(**kwargs), dict(elem.attrib))
 
-  def test_construct_and_load_symlink_map(self):
+  def test_construct_and_load_hardlink_map(self):
     self.maxDiff = None
     with temporary_dir() as mock_cache_dir:
-      with temporary_dir() as symlink_dir:
+      with temporary_dir() as hardlink_dir:
         with temporary_dir() as classpath_dir:
           input_path = os.path.join(classpath_dir, 'inpath')
           output_path = os.path.join(classpath_dir, 'classpath')
@@ -345,21 +345,21 @@ def test_construct_and_load_symlink_map(self):
 
           with open(input_path, 'w') as inpath:
             inpath.write(foo_path)
-          result_classpath, result_map = IvyUtils.construct_and_load_symlink_map(symlink_dir,
+          result_classpath, result_map = IvyUtils.construct_and_load_hardlink_map(hardlink_dir,
                                                                                  mock_cache_dir,
                                                                                  input_path,
                                                                                  output_path)
-          symlink_foo_path = os.path.join(symlink_dir, 'foo.jar')
-          self.assertEquals([symlink_foo_path], result_classpath)
+          hardlink_foo_path = os.path.join(hardlink_dir, 'foo.jar')
+          self.assertEquals([hardlink_foo_path], result_classpath)
           self.assertEquals(
             {
-              os.path.realpath(foo_path): symlink_foo_path
+              os.path.realpath(foo_path): hardlink_foo_path
             },
             result_map)
           with open(output_path, 'r') as outpath:
-            self.assertEquals(symlink_foo_path, outpath.readline())
-          self.assertTrue(os.path.islink(symlink_foo_path))
-          self.assertTrue(os.path.exists(symlink_foo_path))
+            self.assertEquals(hardlink_foo_path, outpath.readline())
+          self.assertFalse(os.path.islink(hardlink_foo_path))
+          self.assertTrue(os.path.exists(hardlink_foo_path))
 
           # Now add an additional path to the existing map
           bar_path = os.path.join(mock_cache_dir, 'bar.jar')
@@ -367,35 +367,35 @@ def test_construct_and_load_symlink_map(self):
             bar.write("test jar contents2")
           with open(input_path, 'w') as inpath:
             inpath.write(os.pathsep.join([foo_path, bar_path]))
-          result_classpath, result_map = IvyUtils.construct_and_load_symlink_map(symlink_dir,
+          result_classpath, result_map = IvyUtils.construct_and_load_hardlink_map(hardlink_dir,
                                                                                  mock_cache_dir,
                                                                                  input_path,
                                                                                  output_path)
-          symlink_bar_path = os.path.join(symlink_dir, 'bar.jar')
+          hardlink_bar_path = os.path.join(hardlink_dir, 'bar.jar')
           self.assertEquals(
             {
-              os.path.realpath(foo_path): symlink_foo_path,
-              os.path.realpath(bar_path): symlink_bar_path,
+              os.path.realpath(foo_path): hardlink_foo_path,
+              os.path.realpath(bar_path): hardlink_bar_path,
             },
             result_map)
-          self.assertEquals([symlink_foo_path, symlink_bar_path], result_classpath)
+          self.assertEquals([hardlink_foo_path, hardlink_bar_path], result_classpath)
 
           with open(output_path, 'r') as outpath:
-            self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline())
-          self.assertTrue(os.path.islink(symlink_foo_path))
-          self.assertTrue(os.path.exists(symlink_foo_path))
-          self.assertTrue(os.path.islink(symlink_bar_path))
-          self.assertTrue(os.path.exists(symlink_bar_path))
+            self.assertEquals(hardlink_foo_path + os.pathsep + hardlink_bar_path, outpath.readline())
+          self.assertFalse(os.path.islink(hardlink_foo_path))
+          self.assertTrue(os.path.exists(hardlink_foo_path))
+          self.assertFalse(os.path.islink(hardlink_bar_path))
+          self.assertTrue(os.path.exists(hardlink_bar_path))
 
           # Reverse the ordering and make sure order is preserved in the output path
           with open(input_path, 'w') as inpath:
             inpath.write(os.pathsep.join([bar_path, foo_path]))
-          IvyUtils.construct_and_load_symlink_map(symlink_dir,
+          IvyUtils.construct_and_load_hardlink_map(hardlink_dir,
                                                   mock_cache_dir,
                                                   input_path,
                                                   output_path)
           with open(output_path, 'r') as outpath:
-            self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())
+            self.assertEquals(hardlink_bar_path + os.pathsep + hardlink_foo_path, outpath.readline())
 
   def test_missing_ivy_report(self):
     self.set_options_for_scope(IvySubsystem.options_scope,
@@ -587,7 +587,7 @@ def test_ivy_resolve_report_copying_fails_when_report_is_missing(self):
 
 
 class IvyUtilsResolveStepsTest(TestBase):
-  def test_if_not_all_symlinked_files_exist_after_successful_resolve_fail(self):
+  def test_if_not_all_hardlinked_files_exist_after_successful_resolve_fail(self):
     resolve = IvyResolveStep(
       ['default'],
       'hash_name',
@@ -604,7 +604,7 @@ def test_if_not_all_symlinked_files_exist_after_successful_resolve_fail(self):
     with self.assertRaises(IvyResolveMappingError):
       resolve.exec_and_load(None, None, [], None, None, None)
 
-  def test_if_not_all_symlinked_files_exist_after_successful_fetch_fail(self):
+  def test_if_not_all_hardlinked_files_exist_after_successful_fetch_fail(self):
     fetch = IvyFetchStep(['default'],
                          'hash_name',
                          False,
@@ -620,9 +620,9 @@ def test_if_not_all_symlinked_files_exist_after_successful_fetch_fail(self):
     with self.assertRaises(IvyResolveMappingError):
       fetch.exec_and_load(None, None, [], None, None, None)
 
-  def test_missing_symlinked_jar_in_candidates(self):
-    empty_symlink_map = {}
-    result = IvyResolveResult(['non-existent-file-location'], empty_symlink_map, 'hash-name',
+  def test_missing_hardlinked_jar_in_candidates(self):
+    empty_hardlink_map = {}
+    result = IvyResolveResult(['non-existent-file-location'], empty_hardlink_map, 'hash-name',
                               {'default':
                                  self.ivy_report_path('ivy_utils_resources/report_with_diamond.xml')
                                })
diff --git a/tests/python/pants_test/util/BUILD b/tests/python/pants_test/util/BUILD
--- a/tests/python/pants_test/util/BUILD
+++ b/tests/python/pants_test/util/BUILD
@@ -58,6 +58,7 @@ python_tests(
   name = 'fileutil',
   sources = ['test_fileutil.py'],
   dependencies = [
+    '3rdparty/python:mock',
     'src/python/pants/util:contextutil',
     'src/python/pants/util:fileutil',
   ]
diff --git a/tests/python/pants_test/util/test_fileutil.py b/tests/python/pants_test/util/test_fileutil.py
--- a/tests/python/pants_test/util/test_fileutil.py
+++ b/tests/python/pants_test/util/test_fileutil.py
@@ -4,12 +4,16 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import errno
 import os
 import random
+import stat
 import unittest
 
-from pants.util.contextutil import temporary_file, temporary_file_path
-from pants.util.fileutil import atomic_copy, create_size_estimators
+from mock import mock
+
+from pants.util.contextutil import temporary_dir, temporary_file, temporary_file_path
+from pants.util.fileutil import atomic_copy, create_size_estimators, safe_hardlink_or_copy
 
 
 class FileutilTest(unittest.TestCase):
@@ -37,3 +41,55 @@ def test_random_estimator(self):
     random.seed(seedValue)
     with temporary_file_path() as src:
       self.assertEqual(create_size_estimators()['random']([src]), rand)
+
+  @classmethod
+  def _is_hard_link(cls, filename, other):
+    s1 = os.stat(filename)
+    s2 = os.stat(other)
+    return (s1[stat.ST_INO], s1[stat.ST_DEV]) == \
+           (s2[stat.ST_INO], s2[stat.ST_DEV])
+
+  def test_hardlink_or_copy(self):
+    content = b'hello'
+
+    with temporary_dir() as src_dir, temporary_file() as dst:
+      dst.write(content)
+      dst.close()
+
+      src_path = os.path.join(src_dir, 'src')
+      safe_hardlink_or_copy(dst.name, src_path)
+
+      with open(src_path, 'rb') as f:
+        self.assertEqual(content, f.read())
+
+      # Make sure it's not symlink
+      self.assertFalse(os.path.islink(dst.name))
+
+      # Make sure they point to the same node
+      self.assertTrue(self._is_hard_link(dst.name, src_path))
+
+  def test_hardlink_or_copy_cross_device_should_copy(self):
+    content = b'hello'
+
+    # Mock os.link to throw an CROSS-DEVICE error
+    with mock.patch('os.link') as os_mock:
+      err = OSError()
+      err.errno = errno.EXDEV
+      os_mock.side_effect = err
+
+      with temporary_dir() as src_dir, temporary_file() as dst:
+        dst.write(content)
+        dst.close()
+
+        src_path = os.path.join(src_dir, 'src')
+
+        safe_hardlink_or_copy(dst.name, src_path)
+
+        with open(src_path, 'rb') as f:
+          self.assertEqual(content, f.read())
+
+        # Make sure it's not symlink
+        self.assertFalse(os.path.islink(dst.name))
+
+        # Make sure they are separate copies
+        self.assertFalse(self._is_hard_link(dst.name, src_path))