@pmclanahan see test_robots_no_redirect below