Repository: google/clusterfuzz

Issue:
Account for testcase flags file when picking arguments for variant task.



Discussion:
Also, trial flags need to be accounted
```
  # Track that app args appended by trials are required.
  trial_app_args = environment.get_value('TRIAL_APP_ARGS')
  if trial_app_args:
    testcase.set_metadata('additional_required_app_args', trial_app_args)
```
variant task is broken for engine fuzzers because of this revert. i need to fix this asap. @jonathanmetzman fyi.

Resolving patch:
diff --git a/src/python/bot/tasks/setup.py b/src/python/bot/tasks/setup.py
--- a/src/python/bot/tasks/setup.py
+++ b/src/python/bot/tasks/setup.py
@@ -16,6 +16,7 @@
 from builtins import range
 import datetime
 import os
+import shlex
 import time
 import zipfile
 
@@ -93,6 +94,34 @@ def _copy_testcase_to_device_and_setup_environment(testcase,
     android.adb.run_shell_command(['chmod', '0755', device_testcase_file_path])
 
 
+def _get_application_arguments(testcase, task_name):
+  """Get application arguments to use for setting up |testcase|. Use minimized
+   arguments if available. For variant task, where we run a testcase against
+   another job type, use both minimized arguments and application arguments
+   from job."""
+  testcase_args = testcase.minimized_arguments
+  if not testcase_args:
+    return None
+
+  if task_name != 'variant':
+    return testcase_args
+
+  job_args = data_handler.get_value_from_job_definition(
+      testcase.job_type, 'APP_ARGS', default='')
+  job_args_list = shlex.split(job_args)
+  testcase_args_list = shlex.split(testcase_args)
+  testcase_args_filtered_list = [
+      arg for arg in testcase_args_list if arg not in job_args_list
+  ]
+
+  app_args = ' '.join(testcase_args_filtered_list)
+  if job_args:
+    if app_args:
+      app_args += ' '
+    app_args += job_args
+  return app_args
+
+
 def prepare_environment_for_testcase(testcase):
   """Set various environment variables based on the test case."""
   # Setup memory debugging tool environment.
@@ -100,28 +129,28 @@ def prepare_environment_for_testcase(testcase):
 
   # Setup environment variable for windows size and location properties.
   # Explicitly use empty string to indicate use of default window properties.
-  if hasattr(testcase, 'window_argument'):
+  if testcase.window_argument:
     environment.set_value('WINDOW_ARG', testcase.window_argument)
 
   # Adjust timeout based on the stored multiplier (if available).
-  if hasattr(testcase, 'timeout_multiplier') and testcase.timeout_multiplier:
+  if testcase.timeout_multiplier:
     test_timeout = environment.get_value('TEST_TIMEOUT')
     environment.set_value('TEST_TIMEOUT',
                           int(test_timeout * testcase.timeout_multiplier))
 
-  # Override APP_ARGS with minimized arguments (if available). Don't do this
-  # for variant task since other job types can have its own set of required
-  # arguments, so use the full set of arguments of that job.
-  task_name = environment.get_value('TASK_NAME')
-  if (task_name != 'variant' and hasattr(testcase, 'minimized_arguments') and
-      testcase.minimized_arguments):
-    environment.set_value('APP_ARGS', testcase.minimized_arguments)
-
   # Add FUZZ_TARGET to environment if this is a fuzz target testcase.
   fuzz_target = testcase.get_metadata('fuzzer_binary_name')
   if fuzz_target:
     environment.set_value('FUZZ_TARGET', fuzz_target)
 
+  # Override APP_ARGS with minimized arguments (if available). Don't do this
+  # for variant task since other job types can have its own set of required
+  # arguments, so use the full set of arguments of that job.
+  task_name = environment.get_value('TASK_NAME')
+  app_args = _get_application_arguments(testcase, task_name)
+  if app_args:
+    environment.set_value('APP_ARGS', app_args)
+
 
 def setup_testcase(testcase, fuzzer_override=None):
   """Sets up the testcase and needed dependencies like fuzzer,


Test patch:
diff --git a/src/python/tests/core/bot/tasks/setup_test.py b/src/python/tests/core/bot/tasks/setup_test.py
--- a/src/python/tests/core/bot/tasks/setup_test.py
+++ b/src/python/tests/core/bot/tasks/setup_test.py
@@ -16,7 +16,10 @@
 import unittest
 
 from bot.tasks import setup
+from datastore import data_types
 from system import environment
+from tests.test_libs import helpers
+from tests.test_libs import test_utils
 
 
 class IsDirectoryOnNfsTest(unittest.TestCase):
@@ -40,3 +43,80 @@ def test_is_directory_on_nfs_with_nfs_and_data_bundle_on_nfs(self):
   def test_is_directory_on_nfs_with_nfs_and_data_bundle_on_local(self):
     """Test is_directory_on_nfs with nfs and data bundle on local."""
     self.assertFalse(setup.is_directory_on_nfs('/tmp/dir1'))
+
+
+# pylint: disable=protected-access
+@test_utils.with_cloud_emulators('datastore')
+class GetApplicationArgumentsTest(unittest.TestCase):
+  """Tests _get_application_arguments."""
+
+  def setUp(self):
+    helpers.patch_environ(self)
+    data_types.Job(
+        name='linux_asan_chrome',
+        environment_string=(
+            'APP_ARGS = --arg1 --arg2 --arg3="--flag1 --flag2"')).put()
+
+    data_types.Job(name='libfuzzer_asan_chrome', environment_string=('')).put()
+
+    self.testcase = test_utils.create_generic_testcase()
+
+  def test_no_minimized_arguments(self):
+    """Test that None is returned when minimized arguments is not set."""
+    self.testcase.minimized_arguments = ''
+    self.testcase.job_type = 'linux_asan_chrome'
+    self.testcase.put()
+
+    self.assertEqual(
+        None, setup._get_application_arguments(self.testcase, 'minimize'))
+    self.assertEqual(None,
+                     setup._get_application_arguments(self.testcase, 'variant'))
+
+  def test_minimized_arguments_for_non_variant_task(self):
+    """Test that minimized arguments are returned for non-variant tasks."""
+    self.testcase.minimized_arguments = '--arg2'
+    self.testcase.job_type = 'linux_asan_chrome'
+    self.testcase.put()
+
+    self.assertEqual(
+        '--arg2', setup._get_application_arguments(self.testcase, 'minimize'))
+
+  def test_no_unique_minimized_arguments_for_variant_task(self):
+    """Test that only APP_ARGS is returned if minimized arguments have no
+    unique arguments, for variant task."""
+    self.testcase.minimized_arguments = '--arg2'
+    self.testcase.job_type = 'linux_asan_chrome'
+    self.testcase.put()
+
+    self.assertEqual('--arg1 --arg2 --arg3="--flag1 --flag2"',
+                     setup._get_application_arguments(self.testcase, 'variant'))
+
+  def test_some_duplicate_minimized_arguments_for_variant_task(self):
+    """Test that both minimized arguments and APP_ARGS are returned with
+    duplicate args stripped from minimized arguments for variant task."""
+    self.testcase.minimized_arguments = '--arg3="--flag1 --flag2" --arg4'
+    self.testcase.job_type = 'linux_asan_chrome'
+    self.testcase.put()
+
+    self.assertEqual('--arg4 --arg1 --arg2 --arg3="--flag1 --flag2"',
+                     setup._get_application_arguments(self.testcase, 'variant'))
+
+  def test_unique_minimized_arguments_for_variant_task(self):
+    """Test that both minimized arguments and APP_ARGS are returned when they
+    don't have common args for variant task."""
+    self.testcase.minimized_arguments = '--arg5'
+    self.testcase.job_type = 'linux_asan_chrome'
+    self.testcase.put()
+
+    self.assertEqual('--arg5 --arg1 --arg2 --arg3="--flag1 --flag2"',
+                     setup._get_application_arguments(self.testcase, 'variant'))
+
+  def test_no_job_app_args_for_variant_task(self):
+    """Test that only minimized arguments is returned when APP_ARGS is not set
+    in job definition."""
+    self.testcase.minimized_arguments = '--arg5'
+    self.testcase.job_type = 'libfuzzer_asan_chrome'
+    self.testcase.put()
+
+    self.assertEqual('--arg5',
+                     setup._get_application_arguments(self.testcase, 'variant'))
diff --git a/src/python/tests/core/local/butler/reproduce_test.py b/src/python/tests/core/local/butler/reproduce_test.py
--- a/src/python/tests/core/local/butler/reproduce_test.py
+++ b/src/python/tests/core/local/butler/reproduce_test.py
@@ -37,6 +37,9 @@ def _fake_get_testcase(*_):
       'fuzzer_name': 'fuzzer',
       'job_definition': 'APP_NAME = echo\nAPP_ARGS = -n\n',
       'platform': environment.platform().lower(),
+      'minimized_arguments': '',
+      'window_argument': '',
+      'timeout_multiplier': 1.0,
   }
 
   return reproduce.SerializedTestcase(testcase_map)