update robots.txt and routes