Repository: airspeed-velocity/asv

Issue:
Help comparing benchmarks across virtualenvs
Sorry if this is the wrong place to ask.  Sorry too that I am sure I am missing something obvious.

I am using asv on Windows : https://github.com/numpy/numpy/issues/5479

I'm comparing performance of numpy in two virtualenvs, one with MKL linking, the other with ATLAS linking.

I told `asv machine` that I have two machines one called 'mike-mkl' and the other 'mike-atlas', identical other than the names.

Then I ran the benchmarks in the MKL virtualenv as:

```
cd numpy/benchmarks
asv run --python=same --machine=mike-mkl
```

In the ATLAS virtualenv:

```
cd numpy/benchmarks
asv run --python=same --machine=mike-atlas
```

I was expecting this to store the two sets of benchmarks, but it appears from the `results/benchmarks.json` file size and the `asv preview` output, that they overwrite each other.

What is the right way to record / show these benchmarks relative to one another?


Discussion:
pv: Looks ok. Needs test, however.
jeremiedbb: It was a little bit less straightforward because the commit hash is truncated to it's 8 first characters for the results file name. However, with provided commit labels I think we want to keep their full length, because the risk of collision is quite high (for instance 'test_label_1' and 'test_label_2' would be cut to 'test_lab').

I chosed to add a prefix for the commit label and check for this prefix when dealing with the result file name. Do you think it's reasonable ?
jeremiedbb: I added a test. I wasn't sure where to put it so I made a new test file for the run command. Is there a better place for it ?
jeremiedbb: The failing tests seems unrelated and are already failing on master [https://travis-ci.org/airspeed-velocity/asv/jobs/506823174](https://travis-ci.org/airspeed-velocity/asv/jobs/506823174)

Patch:
diff --git a/asv/commands/run.py b/asv/commands/run.py
index cf1a52e..74d88d0 100644
--- a/asv/commands/run.py
+++ b/asv/commands/run.py
@@ -92,6 +92,11 @@ class Run(Command):
             benchmark functions faster.  The results are unlikely to
             be useful, and thus are not saved.""")
         common_args.add_environment(parser)
+        parser.add_argument(
+            "--set-commit-hash", default=None,
+            help="""Set the commit hash to use when recording benchmark
+            results. This makes results to be saved also when using an
+            existing environment.""")
         common_args.add_launch_method(parser)
         parser.add_argument(
             "--dry-run", "-n", action="store_true",
@@ -136,7 +141,7 @@ class Run(Command):
             conf=conf, range_spec=args.range, steps=args.steps,
             bench=args.bench, attribute=args.attribute, parallel=args.parallel,
             show_stderr=args.show_stderr, quick=args.quick,
-            profile=args.profile, env_spec=args.env_spec,
+            profile=args.profile, env_spec=args.env_spec, set_commit_hash=args.set_commit_hash,
             dry_run=args.dry_run, machine=args.machine,
             skip_successful=args.skip_existing_successful or args.skip_existing,
             skip_failed=args.skip_existing_failed or args.skip_existing,
@@ -149,7 +154,7 @@ class Run(Command):
 
     @classmethod
     def run(cls, conf, range_spec=None, steps=None, bench=None, attribute=None, parallel=1,
-            show_stderr=False, quick=False, profile=False, env_spec=None,
+            show_stderr=False, quick=False, profile=False, env_spec=None, set_commit_hash=None,
             dry_run=False, machine=None, _machine_file=None, skip_successful=False,
             skip_failed=False, skip_existing_commits=False, record_samples=False,
             append_samples=False, pull=True, interleave_processes=False,
@@ -161,7 +166,7 @@ class Run(Command):
 
         environments = list(environment.get_environments(conf, env_spec))
 
-        if environment.is_existing_only(environments):
+        if environment.is_existing_only(environments) and set_commit_hash is None:
             # No repository required, so skip using it
             conf.dvcs = "none"
 
@@ -386,11 +391,15 @@ class Run(Command):
                         params['python'] = env.python
                         params.update(env.requirements)
 
-                        skip_save = (dry_run or isinstance(env, environment.ExistingEnvironment))
+                        skip_save = dry_run or (isinstance(env, environment.ExistingEnvironment)
+                                                and set_commit_hash is None)
 
                         skip_list = skipped_benchmarks[(commit_hash, env.name)]
                         benchmark_set = benchmarks.filter_out(skip_list)
 
+                        if set_commit_hash is not None:
+                            commit_hash = set_commit_hash
+
                         result = Results(
                             params,
                             env.requirements,


Test patch:
diff --git a/test/test_run.py b/test/test_run.py
new file mode 100644
index 0000000..814ae2c
--- /dev/null
+++ b/test/test_run.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import os
+from os.path import abspath, dirname, join, relpath
+
+import six
+import pytest
+import shutil
+
+from asv import config
+from asv import results
+from asv import environment
+
+from . import tools
+
+
+@pytest.fixture
+def basic_conf(tmpdir):
+    tmpdir = six.text_type(tmpdir)
+    local = abspath(dirname(__file__))
+    os.chdir(tmpdir)
+
+    shutil.copytree(os.path.join(local, 'benchmark'), 'benchmark')
+
+    shutil.copyfile(join(local, 'asv-machine.json'),
+                    join(tmpdir, 'asv-machine.json'))
+
+    repo = tools.generate_test_repo(tmpdir)
+    repo_path = relpath(repo.path)
+
+    conf_dict = {
+        'benchmark_dir': 'benchmark',
+        'results_dir': 'results_workflow',
+        'repo': repo_path,
+        'project': 'asv',
+        'environment_type': "existing",
+        'python': "same"
+    }
+
+    conf = config.Config.from_json(conf_dict)
+
+    return tmpdir, conf, repo
+
+
+def test_set_commit_hash(capsys, basic_conf):
+    tmpdir, conf, repo = basic_conf
+    commit_hash = repo.get_branch_hashes()[0]
+
+    tools.run_asv_with_conf(conf, 'run', '--set-commit-hash=' + commit_hash, _machine_file=join(tmpdir, 'asv-machine.json'))
+
+    env_name = list(environment.get_environments(conf, None))[0].name
+    result_filename = commit_hash[:conf.hash_length] + '-' + env_name + '.json'
+    assert result_filename in os.listdir(join('results_workflow', 'orangutan'))
+
+    result_path = join('results_workflow', 'orangutan', result_filename)
+    times = results.Results.load(result_path)
+    assert times.commit_hash == commit_hash