Repository: GoogleCloudPlatform/PerfKitBenchmarker

Issue:
Cassandra stress advertises that it doesn't require disks
But it does:

https://github.com/GoogleCloudPlatform/PerfKitBenchmarker/blob/master/perfkitbenchmarker/benchmarks/cassandra_stress_benchmark.py#L97-L108

See #475.



Discussion:
I had a chance to debug the issue further. It's not so much about advertising if it requires scratch disk or not (it's P2). In Kubernetes (and all the Docker based clouds like Mesos+Marathon) you cannot attach a volume to already running instance. Cassandra_stress benchmark is a specific benchmark because it (as the only one) calls  CreateScratchDisk() method (see cassandra_stress_benchmark.py:114). And it does this _after_ instance creation what makes it impossible for Kubernetes to use this benchmark. And that's P0 or P1 IMO - taking into consideration that we plan to add support for other Docker based clouds. Maybe with the configs functionality we could somehow create disk_specs for some of the instances _before_ the instances are created. This way disks for Kubernetes would be automatically created in _CreateDependencies phase. 

I'm attaching PKB output from cassandra_stress benchmark:

```
2015-10-15 10:52:40,918 1cf4c0b0 Thread-91 cassandra_stress(1/1) INFO     Running: ssh -A -p 30505 root@10.102.17.41 -2 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -o PreferredAuthentications=publickey -o PasswordAuthentication=no -o ConnectTimeout=5 -o GSSAPIAuthentication=no -o ServerAliveInterval=30 -o ServerAliveCountMax=10 -i /tmp/perfkitbenchmarker/run_1cf4c0b0/perfkitbenchmarker_keyfile cd /tmp/pkb/apache-cassandra/lib && curl -LJO https://maven.java.net/content/repositories/releases/net/java/dev/jna/jna/4.1.0/jna-4.1.0.jar
2015-10-15 10:52:53,063 1cf4c0b0 MainThread cassandra_stress(1/1) ERROR    Exception occurred while calling Configure(10.102.17.41, seed_vms=[<BaseVirtualMachine [ip=10.102.17.41, internal_ip=10.3.204.31]>]):
Traceback (most recent call last):
  File "/root/mateuszb/perfkit_ww42_4/PerfKitBenchmarker/perfkitbenchmarker/vm_util.py", line 227, in _ExecuteThreadCall
    queue.put(ThreadCallResult(call_id, target(*args, **kwargs), None))
  File "/root/mateuszb/perfkit_ww42_4/PerfKitBenchmarker/perfkitbenchmarker/packages/cassandra.py", line 95, in Configure
    'data_path': posixpath.join(vm.GetScratchDir(), 'cassandra'),
  File "/root/mateuszb/perfkit_ww42_4/PerfKitBenchmarker/perfkitbenchmarker/virtual_machine.py", line 154, in GetScratchDir
    disk_num, len(self.scratch_disks)))
Error: GetScratchDir(disk_num=0) is invalid, max disk_num is 0
```

As you can see, scratch_disks array is of size 0. In Kubernetes provider CreateScratchDisk() is not implemented (as it makes no sense to call this method after the instance is created). That's why scratch_disks is not populated.

Thanks,
Mateusz

Ah, I understand. I'll have a look at this today.


Resolving patch:
diff --git a/perfkitbenchmarker/benchmarks/cassandra_stress_benchmark.py b/perfkitbenchmarker/benchmarks/cassandra_stress_benchmark.py
--- a/perfkitbenchmarker/benchmarks/cassandra_stress_benchmark.py
+++ b/perfkitbenchmarker/benchmarks/cassandra_stress_benchmark.py
@@ -29,7 +29,6 @@
 
 
 from perfkitbenchmarker import configs
-from perfkitbenchmarker import disk
 from perfkitbenchmarker import errors
 from perfkitbenchmarker import flags
 from perfkitbenchmarker import sample
@@ -55,18 +54,20 @@
 cassandra_stress:
   description: Benchmark Cassandra using cassandra-stress
   vm_groups:
-    default:
+    cassandra_nodes:
+      vm_spec: *default_single_core
+      disk_spec: *default_500_gb
+      vm_count: 3
+    stress_client:
       vm_spec: *default_single_core
-      vm_count: 4
 """
 
-LOADER_NODE = 'loader'
-DATA_NODE = 'cas'
+CASSANDRA_GROUP = 'cassandra_nodes'
+CLIENT_GROUP = 'stress_client'
+
 PROPAGATION_WAIT_TIME = 30
 SLEEP_BETWEEN_CHECK_IN_SECONDS = 5
 
-DEFAULT_DISK_TYPE = disk.STANDARD
-DEFAULT_DISK_SIZE = 500
 
 # Stress test options.
 CONSISTENCY_LEVEL = 'quorum'
@@ -98,30 +99,18 @@ def Prepare(benchmark_spec):
         required to run the benchmark.
   """
   vm_dict = benchmark_spec.vm_groups
+  cassandra_vms = vm_dict[CASSANDRA_GROUP]
   logging.info('VM dictionary %s', vm_dict)
 
-  if vm_dict['default']:
-    logging.info('No config file is provided, use default settings: '
-                 '1 loader node, 3 data nodes')
-    vm_dict[LOADER_NODE] = [vm_dict['default'][-1]]
-    vm_dict[DATA_NODE] = vm_dict['default'][:3]
-    mount_point = os.path.join(vm_util.VM_TMP_DIR, 'cassandra_data')
-    disk_spec = disk.BaseDiskSpec(
-        disk_size=(FLAGS.scratch_disk_size or DEFAULT_DISK_SIZE),
-        disk_type=(FLAGS.scratch_disk_type or DEFAULT_DISK_TYPE),
-        mount_point=mount_point)
-    for vm in vm_dict[DATA_NODE]:
-      vm.CreateScratchDisk(disk_spec)
-
   logging.info('Authorizing loader[0] permission to access all other vms.')
-  vm_dict[LOADER_NODE][0].AuthenticateVm()
+  vm_dict[CLIENT_GROUP][0].AuthenticateVm()
 
   logging.info('Preparing data files and Java on all vms.')
   vm_util.RunThreaded(lambda vm: vm.Install('cassandra'), benchmark_spec.vms)
-  seed_vm = vm_dict[DATA_NODE][0]
+  seed_vm = cassandra_vms[0]
   configure = functools.partial(cassandra.Configure, seed_vms=[seed_vm])
-  vm_util.RunThreaded(configure, vm_dict[DATA_NODE])
-  cassandra.StartCluster(seed_vm, vm_dict[DATA_NODE][1:])
+  vm_util.RunThreaded(configure, cassandra_vms)
+  cassandra.StartCluster(seed_vm, cassandra_vms[1:])
 
 
 def _ResultFilePath(vm):
@@ -156,9 +145,10 @@ def RunCassandraStress(benchmark_spec):
         that is required to run the benchmark.
   """
   logging.info('Creating Keyspace.')
-  data_node_ips = [data_vm.internal_ip
-                   for data_vm in benchmark_spec.vm_groups[DATA_NODE]]
-  loader_vms = benchmark_spec.vm_groups[LOADER_NODE]
+  loader_vms = benchmark_spec.vm_groups[CLIENT_GROUP]
+  cassandra_vms = benchmark_spec.vm_groups[CASSANDRA_GROUP]
+  data_node_ips = [vm.internal_ip for vm in cassandra_vms]
+
   loader_vms[0].RemoteCommand(
       '%s '
       '--nodes %s --replication-factor %s '
@@ -170,13 +160,11 @@ def RunCassandraStress(benchmark_spec):
   time.sleep(PROPAGATION_WAIT_TIME)
 
   if not FLAGS.num_keys:
-    FLAGS.num_keys = NUM_KEYS_PER_CORE * benchmark_spec.vm_groups[
-        DATA_NODE][0].num_cpus
+    FLAGS.num_keys = NUM_KEYS_PER_CORE * cassandra_vms[0].num_cpus
     logging.info('Num keys not set, using %s in cassandra-stress test.',
                  FLAGS.num_keys)
   logging.info('Executing the benchmark.')
-  args = [((loader_vm, data_node_ips), {})
-          for loader_vm in benchmark_spec.vm_groups[LOADER_NODE]]
+  args = [((loader_vm, data_node_ips), {}) for loader_vm in loader_vms]
   vm_util.RunThreaded(RunTestOnLoader, args)
 
 
@@ -244,7 +232,7 @@ def RunCassandraStressTest(benchmark_spec):
   finally:
     logging.info('Tests running. Watching progress.')
     vm_util.RunThreaded(WaitForLoaderToFinish,
-                        benchmark_spec.vm_groups[LOADER_NODE])
+                        benchmark_spec.vm_groups[CLIENT_GROUP])
 
 
 def CollectResults(benchmark_spec):
@@ -259,6 +247,7 @@ def CollectResults(benchmark_spec):
   """
   logging.info('Gathering results.')
   vm_dict = benchmark_spec.vm_groups
+  loader_vms = vm_dict[CLIENT_GROUP]
   interval_op_rate_list = []
   interval_key_rate_list = []
   latency_median_list = []
@@ -268,12 +257,12 @@ def CollectResults(benchmark_spec):
   args = [((vm, interval_op_rate_list, interval_key_rate_list,
             latency_median_list, latency_95th_list,
             latency_99_9th_list,
-            total_operation_time_list), {}) for vm in vm_dict[LOADER_NODE]]
+            total_operation_time_list), {}) for vm in loader_vms]
   vm_util.RunThreaded(CollectResultFile, args)
   results = []
   metadata = {'num_keys': FLAGS.num_keys,
-              'num_data_nodes': len(vm_dict[DATA_NODE]),
-              'num_loader_nodes': len(vm_dict[LOADER_NODE]),
+              'num_data_nodes': len(vm_dict[CASSANDRA_GROUP]),
+              'num_loader_nodes': len(loader_vms),
               'num_cassandra_stress_threads':
               FLAGS.num_cassandra_stress_threads}
   results = [
@@ -282,17 +271,17 @@ def CollectResults(benchmark_spec):
       sample.Sample('Interval_key_rate', math.fsum(interval_key_rate_list),
                     'operations per second', metadata),
       sample.Sample('Latency median',
-                    math.fsum(latency_median_list) / len(vm_dict[LOADER_NODE]),
+                    math.fsum(latency_median_list) / len(loader_vms),
                     'ms', metadata),
       sample.Sample('Latency 95th percentile',
-                    math.fsum(latency_95th_list) / len(vm_dict[LOADER_NODE]),
+                    math.fsum(latency_95th_list) / len(loader_vms),
                     'ms', metadata),
       sample.Sample('Latency 99.9th percentile',
-                    math.fsum(latency_99_9th_list) / len(vm_dict[LOADER_NODE]),
+                    math.fsum(latency_99_9th_list) / len(loader_vms),
                     'ms', metadata),
       sample.Sample('Total operation time',
                     math.fsum(total_operation_time_list) / len(
-                        vm_dict[LOADER_NODE]), 'seconds', metadata)]
+                        loader_vms), 'seconds', metadata)]
   logging.info('Cassandra results:\n%s', results)
   return results
 
@@ -319,6 +308,7 @@ def Cleanup(benchmark_spec):
         that is required to run the benchmark.
   """
   vm_dict = benchmark_spec.vm_groups
+  cassandra_vms = vm_dict[CASSANDRA_GROUP]
 
-  vm_util.RunThreaded(cassandra.Stop, vm_dict[DATA_NODE])
-  vm_util.RunThreaded(cassandra.CleanNode, vm_dict[DATA_NODE])
+  vm_util.RunThreaded(cassandra.Stop, cassandra_vms)
+  vm_util.RunThreaded(cassandra.CleanNode, cassandra_vms)


Test patch: