models/research/brain_coder/single_task/run_eval_tasks.py

#!/usr/bin/env python
from __future__ import print_function

r"""This script can launch any eval experiments from the paper.

This is a script. Run with python, not bazel.

Usage:
./single_task/run_eval_tasks.py \
    --exp EXP --desc DESC [--tuning_tasks] [--iclr_tasks] [--task TASK] \
    [--tasks TASK1 TASK2 ...]

where EXP is one of the keys in `experiments`,
and DESC is a string description of the set of experiments (such as "v0")

Set only one of these flags:
--tuning_tasks flag only runs tuning tasks.
--iclr_tasks flag only runs the tasks included in the paper.
--regression_tests flag runs tasks which function as regression tests.
--task flag manually selects a single task to run.
--tasks flag takes a custom list of tasks.

Other flags:
--reps N specifies N repetitions per experiment, Default is 25.
--training_replicas R specifies that R workers will be launched to train one
    task (for neural network algorithms). These workers will update a global
    model stored on a parameter server. Defaults to 1. If R > 1, a parameter
    server will also be launched.


Run everything:
exps=( pg-20M pg-topk-20M topk-20M ga-20M rand-20M )
BIN_DIR="single_task"
for exp in "${exps[@]}"
do
  ./$BIN_DIR/run_eval_tasks.py \
      --exp "$exp" --iclr_tasks
done
"""

import argparse
from collections import namedtuple
import subprocess


S = namedtuple('S', ['length'])
default_length = 100


iclr_tasks = [
    'reverse', 'remove-char', 'count-char', 'add', 'bool-logic', 'print-hello',
    'echo-twice', 'echo-thrice', 'copy-reverse', 'zero-cascade', 'cascade',
    'shift-left', 'shift-right', 'riffle', 'unriffle', 'middle-char',
    'remove-last', 'remove-last-two', 'echo-alternating', 'echo-half', 'length',
    'echo-second-seq', 'echo-nth-seq', 'substring', 'divide-2', 'dedup']


regression_test_tasks = ['reverse', 'test-hill-climb']


E = namedtuple(
    'E',
    ['name', 'method_type', 'config', 'simplify', 'batch_size', 'max_npe'])


def make_experiment_settings(name, **kwargs):
  # Unpack experiment info from name.
  def split_last(string, char):
    i = string.rindex(char)
    return string[:i], string[i+1:]
  def si_to_int(si_string):
    return int(
        si_string.upper().replace('K', '0'*3).replace('M', '0'*6)
        .replace('G', '0'*9))
  method_type, max_npe = split_last(name, '-')
  assert method_type
  assert max_npe
  return E(
      name=name, method_type=method_type, max_npe=si_to_int(max_npe), **kwargs)


experiments_set = {
    make_experiment_settings(
        'pg-20M',
        config='entropy_beta=0.05,lr=0.0001,topk_loss_hparam=0.0,topk=0,'
               'pi_loss_hparam=1.0,alpha=0.0',
        simplify=False,
        batch_size=64),
    make_experiment_settings(
        'pg-topk-20M',
        config='entropy_beta=0.01,lr=0.0001,topk_loss_hparam=50.0,topk=10,'
               'pi_loss_hparam=1.0,alpha=0.0',
        simplify=False,
        batch_size=64),
    make_experiment_settings(
        'topk-20M',
        config='entropy_beta=0.01,lr=0.0001,topk_loss_hparam=200.0,topk=10,'
               'pi_loss_hparam=0.0,alpha=0.0',
        simplify=False,
        batch_size=64),
    make_experiment_settings(
        'topk-0ent-20M',
        config='entropy_beta=0.000,lr=0.0001,topk_loss_hparam=200.0,topk=10,'
               'pi_loss_hparam=0.0,alpha=0.0',
        simplify=False,
        batch_size=64),
    make_experiment_settings(
        'ga-20M',
        config='crossover_rate=0.95,mutation_rate=0.15',
        simplify=False,
        batch_size=100),  # Population size.
    make_experiment_settings(
        'rand-20M',
        config='',
        simplify=False,
        batch_size=1),
    make_experiment_settings(
        'simpl-500M',
        config='entropy_beta=0.05,lr=0.0001,topk_loss_hparam=0.5,topk=10,'
               'pi_loss_hparam=1.0,alpha=0.0',
        simplify=True,
        batch_size=64),
}

experiments = {e.name: e for e in experiments_set}


# pylint: disable=redefined-outer-name
def parse_args(extra_args=()):
  """Parse arguments and extract task and experiment info."""
  parser = argparse.ArgumentParser(description='Run all eval tasks.')
  parser.add_argument('--exp', required=True)
  parser.add_argument('--tuning_tasks', action='store_true')
  parser.add_argument('--iclr_tasks', action='store_true')
  parser.add_argument('--regression_tests', action='store_true')
  parser.add_argument('--desc', default='v0')
  parser.add_argument('--reps', default=25)
  parser.add_argument('--task')
  parser.add_argument('--tasks', nargs='+')
  for arg_string, default in extra_args:
    parser.add_argument(arg_string, default=default)
  args = parser.parse_args()

  print('Running experiment: %s' % (args.exp,))
  if args.desc:
    print('Extra description: "%s"' % (args.desc,))
  if args.exp not in experiments:
    raise ValueError('Experiment name is not valid')
  experiment_name = args.exp
  experiment_settings = experiments[experiment_name]
  assert experiment_settings.name == experiment_name

  if args.tasks:
    print('Launching tasks from args: %s' % (args.tasks,))
    tasks = {t: S(length=default_length) for t in args.tasks}
  elif args.task:
    print('Launching single task "%s"' % args.task)
    tasks = {args.task: S(length=default_length)}
  elif args.tuning_tasks:
    print('Only running tuning tasks')
    tasks = {name: S(length=default_length)
             for name in ['reverse-tune', 'remove-char-tune']}
  elif args.iclr_tasks:
    print('Running eval tasks from ICLR paper.')
    tasks = {name: S(length=default_length) for name in iclr_tasks}
  elif args.regression_tests:
    tasks = {name: S(length=default_length) for name in regression_test_tasks}
  print('Tasks: %s' % tasks.keys())

  print('reps = %d' % (int(args.reps),))

  return args, tasks, experiment_settings


def run(command_string):
  subprocess.call(command_string, shell=True)


if __name__ == '__main__':
  LAUNCH_TRAINING_COMMAND = 'single_task/launch_training.sh'
  COMPILE_COMMAND = 'bazel build -c opt single_task:run.par'

  args, tasks, experiment_settings = parse_args(
      extra_args=(('--training_replicas', 1),))

  if experiment_settings.method_type in (
      'pg', 'pg-topk', 'topk', 'topk-0ent', 'simpl'):
    # Runs PG and TopK.

    def make_run_cmd(job_name, task, max_npe, num_reps, code_length,
                     batch_size, do_simplify, custom_config_str):
      """Constructs terminal command for launching NN based algorithms.

      The arguments to this function will be used to create config for the
      experiment.

      Args:
        job_name: Name of the job to launch. Should uniquely identify this
            experiment run.
        task: Name of the coding task to solve.
        max_npe: Maximum number of programs executed. An integer.
        num_reps: Number of times to run the experiment. An integer.
        code_length: Maximum allowed length of synthesized code.
        batch_size: Minibatch size for gradient descent.
        do_simplify: Whether to run the experiment in code simplification mode.
            A bool.
        custom_config_str: Additional config for the model config string.

      Returns:
        The terminal command that launches the specified experiment.
      """
      config = """
        env=c(task='{0}',correct_syntax=False),
        agent=c(
          algorithm='pg',
          policy_lstm_sizes=[35,35],value_lstm_sizes=[35,35],
          grad_clip_threshold=50.0,param_init_factor=0.5,regularizer=0.0,
          softmax_tr=1.0,optimizer='rmsprop',ema_baseline_decay=0.99,
          eos_token={3},{4}),
        timestep_limit={1},batch_size={2}
      """.replace(' ', '').replace('\n', '').format(
          task, code_length, batch_size, do_simplify, custom_config_str)
      num_ps = 0 if args.training_replicas == 1 else 1
      return (
          r'{0} --job_name={1} --config="{2}" --max_npe={3} '
          '--num_repetitions={4} --num_workers={5} --num_ps={6} '
          '--stop_on_success={7}'
          .format(LAUNCH_TRAINING_COMMAND, job_name, config, max_npe, num_reps,
                  args.training_replicas, num_ps, str(not do_simplify).lower()))

  else:
    # Runs GA and Rand.
    assert experiment_settings.method_type in ('ga', 'rand')

    def make_run_cmd(job_name, task, max_npe, num_reps, code_length,
                     batch_size, do_simplify, custom_config_str):
      """Constructs terminal command for launching GA or uniform random search.

      The arguments to this function will be used to create config for the
      experiment.

      Args:
        job_name: Name of the job to launch. Should uniquely identify this
            experiment run.
        task: Name of the coding task to solve.
        max_npe: Maximum number of programs executed. An integer.
        num_reps: Number of times to run the experiment. An integer.
        code_length: Maximum allowed length of synthesized code.
        batch_size: Minibatch size for gradient descent.
        do_simplify: Whether to run the experiment in code simplification mode.
            A bool.
        custom_config_str: Additional config for the model config string.

      Returns:
        The terminal command that launches the specified experiment.
      """
      assert not do_simplify
      if custom_config_str:
        custom_config_str = ',' + custom_config_str
      config = """
        env=c(task='{0}',correct_syntax=False),
        agent=c(
          algorithm='{4}'
          {3}),
        timestep_limit={1},batch_size={2}
      """.replace(' ', '').replace('\n', '').format(
          task, code_length, batch_size, custom_config_str,
          experiment_settings.method_type)
      num_workers = num_reps  # Do each rep in parallel.
      return (
          r'{0} --job_name={1} --config="{2}" --max_npe={3} '
          '--num_repetitions={4} --num_workers={5} --num_ps={6} '
          '--stop_on_success={7}'
          .format(LAUNCH_TRAINING_COMMAND, job_name, config, max_npe, num_reps,
                  num_workers, 0, str(not do_simplify).lower()))

  print('Compiling...')
  run(COMPILE_COMMAND)

  print('Launching %d coding tasks...' % len(tasks))
  for task, task_settings in tasks.iteritems():
    name = 'bf_rl_iclr'
    desc = '{0}.{1}_{2}'.format(args.desc, experiment_settings.name, task)
    job_name = '{}.{}'.format(name, desc)
    print('Job name: %s' % job_name)
    reps = int(args.reps) if not experiment_settings.simplify else 1
    run_cmd = make_run_cmd(
        job_name, task, experiment_settings.max_npe, reps,
        task_settings.length, experiment_settings.batch_size,
        experiment_settings.simplify,
        experiment_settings.config)
    print('Running command:\n' + run_cmd)
    run(run_cmd)

  print('Done.')
# pylint: enable=redefined-outer-name