Repository: vitessio/vitess
Pull request #5566 (2019-12-13 09:41:05)

Converted initial sharding test cases in Go
Converted initial_sharding to go. Following files are converted.

initial_sharding.py
initial_sharding_bytes.py
initial_sharding_multi.py
initial_sharding_multi_split_diff.py

Signed-off-by: Arindam Nayak <arindam.nayak@outlook.com>


Patch:
diff --git a/.dockerignore b/.dockerignore
index 58354ff421..5b123fd019 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,8 +5,3 @@ java/*/target
 java/*/bin
 php/vendor
 releases
-/dist/
-/py-vtdb/
-/vthook/
-/bin/
-/vtdataroot/
diff --git a/.github/bootstrap.sh b/.github/bootstrap.sh
new file mode 100755
index 0000000000..f02bffd1f1
--- /dev/null
+++ b/.github/bootstrap.sh
@@ -0,0 +1,175 @@
+#!/bin/bash
+# shellcheck disable=SC2164
+
+# Copyright 2019 The Vitess Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a next-gen bootstrap which skips Python and Java tests,
+# and does not use the VTROOT/VTTOP layout.
+#
+# My original intention was to use the same bootstrap.sh and gate
+# for new features, but it has turned out to be difficult to do,
+# due to the way that Docker cache works in the CI environment.
+
+function fail() {
+  echo "ERROR: $1"
+  exit 1
+}
+
+[[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory"
+
+# Create main directories.
+
+VTROOT="$PWD"
+
+mkdir -p dist
+mkdir -p bin
+mkdir -p lib
+mkdir -p vthook
+
+source ./dev.env
+
+go version &>/dev/null  || fail "Go is not installed or is not on \$PATH"
+goversion_min 1.12 || fail "Go is not version 1.12+"
+
+# Set up required soft links.
+# TODO(mberlin): Which of these can be deleted?
+ln -snf "$VTROOT/py" "$VTROOT/py-vtdb"
+ln -snf "$VTROOT/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
+ln -snf "$VTROOT/test/vthook-test.sh" "$VTROOT/vthook/test.sh"
+ln -snf "$VTROOT/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error"
+ln -snf "$VTROOT/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform"
+
+# git hooks are only required if someone intends to contribute.
+
+echo "creating git hooks"
+mkdir -p "$VTROOT/.git/hooks"
+ln -sf "$VTROOT/misc/git/pre-commit" "$VTROOT/.git/hooks/pre-commit"
+ln -sf "$VTROOT/misc/git/commit-msg" "$VTROOT/.git/hooks/commit-msg"
+git config core.hooksPath "$VTROOT/.git/hooks"
+
+# install_dep is a helper function to generalize the download and installation of dependencies.
+#
+# If the installation is successful, it puts the installed version string into
+# the $dist/.installed_version file. If the version has not changed, bootstrap
+# will skip future installations.
+function install_dep() {
+  if [[ $# != 4 ]]; then
+    fail "install_dep function requires exactly 4 parameters (and not $#). Parameters: $*"
+  fi
+  local name="$1"
+  local version="$2"
+  local dist="$3"
+  local install_func="$4"
+
+  version_file="$dist/.installed_version"
+  if [[ -f "$version_file" && "$(cat "$version_file")" == "$version" ]]; then
+    echo "skipping $name install. remove $dist to force re-install."
+    return
+  fi
+
+  echo "installing $name $version"
+
+  # shellcheck disable=SC2064
+  trap "fail '$name build failed'; exit 1" ERR
+
+  # Cleanup any existing data and re-create the directory.
+  rm -rf "$dist"
+  mkdir -p "$dist"
+
+  # Change $CWD to $dist before calling "install_func".
+  pushd "$dist" >/dev/null
+  # -E (same as "set -o errtrace") makes sure that "install_func" inherits the
+  # trap. If here's an error, the trap will be called which will exit this
+  # script.
+  set -E
+  $install_func "$version" "$dist"
+  set +E
+  popd >/dev/null
+
+  trap - ERR
+
+  echo "$version" > "$version_file"
+}
+
+
+#
+# 1. Installation of dependencies.
+#
+
+# Wrapper around the `arch` command which plays nice with OS X
+function get_arch() {
+  case $(uname) in
+    Linux) arch;;
+    Darwin) uname -m;;
+  esac
+}
+
+# Install protoc.
+function install_protoc() {
+  local version="$1"
+  local dist="$2"
+
+  case $(uname) in
+    Linux)  local platform=linux;;
+    Darwin) local platform=osx;;
+  esac
+
+  case $(get_arch) in
+      aarch64)  local target=aarch_64;;
+      x86_64)  local target=x86_64;;
+      *)   echo "ERROR: unsupported architecture"; exit 1;;
+  esac
+
+  wget https://github.com/protocolbuffers/protobuf/releases/download/v$version/protoc-$version-$platform-${target}.zip
+  unzip "protoc-$version-$platform-${target}.zip"
+  ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc"
+}
+protoc_ver=3.6.1
+install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc
+
+# Download and install etcd, link etcd binary into our root.
+function install_etcd() {
+  local version="$1"
+  local dist="$2"
+
+  case $(uname) in
+    Linux)  local platform=linux; local ext=tar.gz;;
+    Darwin) local platform=darwin; local ext=zip;;
+  esac
+
+  case $(get_arch) in
+      aarch64)  local target=arm64;;
+      x86_64)  local target=amd64;;
+      *)   echo "ERROR: unsupported architecture"; exit 1;;
+  esac
+
+  download_url=https://github.com/coreos/etcd/releases/download
+  file="etcd-${version}-${platform}-${target}.${ext}"
+
+  wget "$download_url/$version/$file"
+  if [ "$ext" = "tar.gz" ]; then
+    tar xzf "$file"
+  else
+    unzip "$file"
+  fi
+  rm "$file"
+  ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd"
+}
+
+# Install etcd if not detected
+which etcd || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
+
+echo
+echo "bootstrap finished"
diff --git a/.github/workflows/check_make_parser.yml b/.github/workflows/check_make_parser.yml
index efc6c737e7..1cf47a6d4a 100644
--- a/.github/workflows/check_make_parser.yml
+++ b/.github/workflows/check_make_parser.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Check Make Parser
+    name: Build
     runs-on: ubuntu-latest
     steps:
 
@@ -17,7 +17,6 @@ jobs:
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed" 
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop
@@ -25,11 +24,14 @@ jobs:
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
-    - name: Run make minimaltools
+    - name: Run bootstrap.sh
       run: |
-        make minimaltools
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
 
     - name: check_make_parser
       run: |
-        tools/check_make_parser.sh
+        export PATH=$PWD/bin:$PATH
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/check_make_parser.sh
 
diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml
index 5d31167f70..61ddea9a5b 100644
--- a/.github/workflows/cluster_endtoend.yml
+++ b/.github/workflows/cluster_endtoend.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Cluster End-to-End Test
+    name: Build
     runs-on: ubuntu-latest
     steps:
 
@@ -17,7 +17,6 @@ jobs:
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop
@@ -25,10 +24,18 @@ jobs:
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
-    - name: Run make minimaltools
+    - name: Run bootstrap.sh
       run: |
-        make minimaltools
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
+
+    - name: Build
+      run: |
+        VTROOT=$PWD VTTOP=$PWD make build
 
     - name: cluster_endtoend
       run: |
-        make e2e_test_cluster
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make e2e_test_cluster
diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml
index b10977d486..a83ad49904 100644
--- a/.github/workflows/endtoend.yml
+++ b/.github/workflows/endtoend.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: End-to-End Test
+    name: Build
     runs-on: ubuntu-latest
     steps:
 
@@ -17,7 +17,6 @@ jobs:
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop
@@ -25,14 +24,19 @@ jobs:
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
-    - name: Run make minimaltools
+    - name: Run bootstrap.sh
       run: |
-        make minimaltools
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
 
     - name: Build
       run: |
-        make build
+        VTROOT=$PWD VTTOP=$PWD make build
 
     - name: endtoend
       run: |
-        tools/e2e_test_runner.sh
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        mkdir -p /tmp/vtdataroot
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/e2e_test_runner.sh
diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml
index b08ce73f88..9e9467a7f9 100644
--- a/.github/workflows/local_example.yml
+++ b/.github/workflows/local_example.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Local Example Test
+    name: Build
     runs-on: ubuntu-latest
     steps:
 
@@ -17,7 +17,6 @@ jobs:
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop
@@ -25,15 +24,18 @@ jobs:
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
-    - name: Run make minimaltools
+    - name: Run bootstrap.sh
       run: |
-        make minimaltools
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
 
     - name: Build
       run: |
-        make build
+        VTROOT=$PWD VTTOP=$PWD make build
 
     - name: local_example
       run: |
-        test/local_example.sh
+        export PATH=$PWD/bin:$PATH
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD test/local_example.sh
 
diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
index c60e862473..774bdbd736 100644
--- a/.github/workflows/unit.yml
+++ b/.github/workflows/unit.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: Unit Test
+    name: Build
     runs-on: ubuntu-latest
     steps:
 
@@ -17,20 +17,24 @@ jobs:
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ curl git wget ant openjdk-8-jdk
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget ant openjdk-8-jdk
         sudo service mysql stop
+        sudo service etcd stop
         sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
-        mkdir -p dist bin
-        curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist
-        mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/
         go mod download
 
-    - name: Run make tools
+    - name: Run bootstrap.sh
       run: |
-        make tools
+        VTTOP=$PWD VTROOT=$PWD BUILD_PYTHON=0 ./bootstrap.sh
+
+    - name: Build
+      run: |
+        VTROOT=$PWD VTTOP=$PWD make build
 
     - name: unit
       run: |
-        make test
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        mkdir -p /tmp/vtdataroot
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/unit_test_runner.sh
diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml
index 6f065bf273..ccc73aa4f9 100644
--- a/.github/workflows/unit_race.yml
+++ b/.github/workflows/unit_race.yml
@@ -3,7 +3,7 @@ on: [push]
 jobs:
 
   build:
-    name: Unit Test (Race)
+    name: Build
     runs-on: ubuntu-latest
     steps:
 
@@ -17,7 +17,6 @@ jobs:
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop
@@ -25,10 +24,16 @@ jobs:
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
-    - name: Run make tools
+    - name: Run bootstrap.sh
       run: |
-        make tools
+        VTTOP=$PWD VTROOT=$PWD BUILD_PYTHON=0 ./bootstrap.sh
+
+    - name: Build
+      run: |
+        VTROOT=$PWD VTTOP=$PWD make build
 
     - name: unit_race
       run: |
-        make unit_test_race
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make unit_test_race
diff --git a/.gitignore b/.gitignore
index be8fe72756..86b6f1b385 100644
--- a/.gitignore
+++ b/.gitignore
@@ -79,9 +79,3 @@ releases
 
 # Vagrant
 .vagrant
-
-/dist/
-/py-vtdb
-/vthook/
-/bin/
-/vtdataroot/
diff --git a/Makefile b/Makefile
index 04b3c121e3..d0c1d58dae 100644
--- a/Makefile
+++ b/Makefile
@@ -14,15 +14,17 @@
 
 MAKEFLAGS = -s
 
-export GOBIN=$(PWD)/bin
+# Soon this can be $PWD/bin, with no dependencies
+# Waiting on https://github.com/vitessio/vitess/issues/5378
+
+export GOBIN=$(VTROOT)/bin
 export GO111MODULE=on
-export GODEBUG=tls13=0
 
 # Disabled parallel processing of target prerequisites to avoid that integration tests are racing each other (e.g. for ports) and may fail.
 # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time.
 .NOTPARALLEL:
 
-.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools
+.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race
 
 all: build
 
@@ -46,7 +48,6 @@ build:
 ifndef NOBANNER
 	echo $$(date): Building source tree
 endif
-	bash ./build.env
 	go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/...
 
 parser:
@@ -55,9 +56,8 @@ parser:
 # To pass extra flags, run test.go manually.
 # For example: go run test.go -docker=false -- --extra-flag
 # For more info see: go run test.go -help
-test: build dependency_check
-	echo $$(date): Running unit tests
-	tools/unit_test_runner.sh
+test:
+	go run test.go -docker=false
 
 site_test: unit_test site_integration_test
 
@@ -66,13 +66,25 @@ clean:
 	rm -rf third_party/acolyte
 	rm -rf go/vt/.proto.tmp
 
+# This will remove object files for all Go projects in the same GOPATH.
+# This is necessary, for example, to make sure dependencies are rebuilt
+# when switching between different versions of Go.
+clean_pkg:
+	rm -rf ../../../../pkg Godeps/_workspace/pkg
+
 # Remove everything including stuff pulled down by bootstrap.sh
 cleanall:
+	# symlinks
+	for f in config data py-vtdb; do test -L ../../../../$$f && rm ../../../../$$f; done
 	# directories created by bootstrap.sh
 	# - exclude vtdataroot and vthook as they may have data we want
-	rm -rf bin dist lib pkg
+	rm -rf ../../../../bin ../../../../dist ../../../../lib ../../../../pkg
 	# Remind people to run bootstrap.sh again
-	echo "Please run 'make tools' again to setup your environment"
+	echo "Please run bootstrap.sh again to setup your environment"
+
+unit_test: build
+	echo $$(date): Running unit tests
+	go test $(VT_GO_PARALLEL) ./go/...
 
 e2e_test: build
 	echo $$(date): Running endtoend tests
@@ -84,7 +96,7 @@ e2e_test: build
 unit_test_cover: build
 	go test $(VT_GO_PARALLEL) -cover ./go/... | misc/parse_cover.py
 
-unit_test_race: build dependency_check
+unit_test_race: build
 	tools/unit_test_race.sh
 
 e2e_test_race: build
@@ -106,7 +118,7 @@ site_integration_test:
 
 java_test:
 	go install ./go/cmd/vtgateclienttest ./go/cmd/vtcombo
-	VTROOT=${PWD} mvn -f java/pom.xml -B clean verify
+	mvn -f java/pom.xml -B clean verify
 
 install_protoc-gen-go:
 	go install github.com/golang/protobuf/protoc-gen-go
@@ -272,14 +284,3 @@ packages: docker_base
 	docker build -f docker/packaging/Dockerfile -t vitess/packaging .
 	docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t deb --deb-no-default-config-files
 	docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t rpm
-
-tools:
-	echo $$(date): Installing dependencies
-	BUILD_PYTHON=0 ./bootstrap.sh
-
-minimaltools:
-	echo $$(date): Installing minimal dependencies
-	BUILD_PYTHON=0 BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh
-
-dependency_check:
-	./tools/dependency_check.sh
diff --git a/bootstrap.sh b/bootstrap.sh
index c943c0d8bb..a5f73105d4 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -15,24 +15,69 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-### This file is executed by 'make tools'. You do not need to execute it directly.
-
-source ./dev.env
 
 # Outline of this file.
 # 0. Initialization and helper methods.
 # 1. Installation of dependencies.
 
+BUILD_TESTS=${BUILD_TESTS:-1}
 BUILD_PYTHON=${BUILD_PYTHON:-1}
 BUILD_JAVA=${BUILD_JAVA:-1}
-BUILD_CONSUL=${BUILD_CONSUL:-1}
 
 #
 # 0. Initialization and helper methods.
 #
 
+function fail() {
+  echo "ERROR: $1"
+  exit 1
+}
+
 [[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory"
 
+# Create main directories.
+VTROOT="${VTROOT:-${PWD/\/src\/vitess.io\/vitess/}}"
+mkdir -p "$VTROOT/dist"
+mkdir -p "$VTROOT/bin"
+mkdir -p "$VTROOT/lib"
+mkdir -p "$VTROOT/vthook"
+
+# This is required for VIRTUALENV
+# Used by Python below
+
+if [ "$BUILD_TESTS" == 1 ] ; then
+    source ./dev.env
+else
+    source ./build.env
+fi
+
+go version &>/dev/null  || fail "Go is not installed or is not on \$PATH"
+goversion_min 1.12 || fail "Go is not version 1.12+"
+
+if [ "$BUILD_TESTS" == 1 ] ; then
+    # Set up required soft links.
+    # TODO(mberlin): Which of these can be deleted?
+    ln -snf "$VTTOP/config" "$VTROOT/config"
+    ln -snf "$VTTOP/data" "$VTROOT/data"
+    ln -snf "$VTTOP/py" "$VTROOT/py-vtdb"
+    ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
+    ln -snf "$VTTOP/test/vthook-test.sh" "$VTROOT/vthook/test.sh"
+    ln -snf "$VTTOP/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error"
+    ln -snf "$VTTOP/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform"
+else
+    ln -snf "$VTTOP/config" "$VTROOT/config"
+    ln -snf "$VTTOP/data" "$VTROOT/data"
+    ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh"
+fi
+
+# git hooks are only required if someone intends to contribute.
+
+echo "creating git hooks"
+mkdir -p "$VTTOP/.git/hooks"
+ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
+ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
+(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
+
 # install_dep is a helper function to generalize the download and installation of dependencies.
 #
 # If the installation is successful, it puts the installed version string into
@@ -191,7 +236,7 @@ function install_etcd() {
   ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd"
   ln -snf "$dist/etcd-${version}-${platform}-${target}/etcdctl" "$VTROOT/bin/etcdctl"
 }
-command -v etcd && echo "etcd already installed" || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
+which etcd || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
 
 
 # Download and install consul, link consul binary into our root.
@@ -215,10 +260,8 @@ function install_consul() {
   unzip "consul_${version}_${platform}_${target}.zip"
   ln -snf "$dist/consul" "$VTROOT/bin/consul"
 }
+install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
 
-if [ "$BUILD_CONSUL" == 1 ] ; then
-  install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
-fi
 
 # Install py-mock.
 function install_pymock() {
@@ -230,7 +273,7 @@ function install_pymock() {
   PYTHONPATH=$(prepend_path "$PYTHONPATH" "$dist/lib/python2.7/site-packages")
   export PYTHONPATH
 
-  pushd "$VTROOT/third_party/py" >/dev/null
+  pushd "$VTTOP/third_party/py" >/dev/null
   tar -xzf "mock-$version.tar.gz"
   cd "mock-$version"
   $PYTHON ./setup.py install --prefix="$dist"
@@ -292,4 +335,4 @@ if [ "$BUILD_PYTHON" == 1 ] ; then
 fi
 
 echo
-echo "bootstrap finished - run 'make build' to compile"
+echo "bootstrap finished - run 'source dev.env' or 'source build.env' in your shell before building."
diff --git a/build.env b/build.env
old mode 100755
new mode 100644
index 467749a10f..5719714f34
--- a/build.env
+++ b/build.env
@@ -14,33 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source ./tools/shell_functions.inc
-
-go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions."
-goversion_min 1.12 || fail "Go is not version 1.12+. See https://vitess.io/contributing/build-from-source for install instructions."
-
-mkdir -p dist
-mkdir -p bin
-mkdir -p lib
-mkdir -p vthook
-
-export VTROOT="$PWD"
-export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}"
-export PATH="$PWD/bin:$PATH"
-
-mkdir -p "$VTDATAROOT"
-
-# Set up required soft links.
-# TODO(mberlin): Which of these can be deleted?
-ln -snf "$PWD/py" py-vtdb
-ln -snf "$PWD/go/vt/zkctl/zksrv.sh" bin/zksrv.sh
-ln -snf "$PWD/test/vthook-test.sh" vthook/test.sh
-ln -snf "$PWD/test/vthook-test_backup_error" vthook/test_backup_error
-ln -snf "$PWD/test/vthook-test_backup_transform" vthook/test_backup_transform
-
-# install git hooks
-
-mkdir -p .git/hooks
-ln -sf "$PWD/misc/git/pre-commit" .git/hooks/pre-commit
-ln -sf "$PWD/misc/git/commit-msg" .git/hooks/commit-msg
-git config core.hooksPath .git/hooks
+# Plese ensure dev.env is written in a way which is POSIX (bourne)
+# shell compatible.
+# - Some build systems like rpm require the different scriptlets used
+#   to build a package to be run under a POSIX shell so non-POSIX
+#   syntax will break that as dev.env will not be sourced by bash..
+
+# Import prepend_path function.
+dir="$(dirname "${BASH_SOURCE[0]}")"
+# shellcheck source=tools/shell_functions.inc
+if ! source "${dir}/tools/shell_functions.inc"; then
+  echo "failed to load tools/shell_functions.inc"
+  return 1
+fi
+
+export VTTOP=$(pwd)
+export VTROOT="${VTROOT:-${VTTOP/\/src\/vitess.io\/vitess/}}"
diff --git a/dev.env b/dev.env
index f256abbd93..060dd0ef1b 100644
--- a/dev.env
+++ b/dev.env
@@ -22,6 +22,9 @@
 
 source ./build.env
 
+export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}"
+mkdir -p "$VTDATAROOT"
+
 export VTPORTSTART=15000
 
 # Add all site-packages or dist-packages directories below $VTROOT/dist to $PYTHONPATH.
@@ -37,8 +40,8 @@ IFS="$BACKUP_IFS"
 
 PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/py-vtdb")
 PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/dist/selenium")
-PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/test")
-PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/test/cluster/sandbox")
+PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTTOP/test")
+PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTTOP/test/cluster/sandbox")
 export PYTHONPATH
 
 # Ensure bootstrap.sh uses python2 on systems which default to python3.
diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common
index 6a9255b38e..e645b31a44 100644
--- a/docker/bootstrap/Dockerfile.common
+++ b/docker/bootstrap/Dockerfile.common
@@ -30,15 +30,16 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     && rm -rf /var/lib/apt/lists/*
 
 # Install Maven 3.1+
-RUN mkdir -p /vt/src/vitess.io/vitess/dist && \
-    cd /vt/src/vitess.io/vitess/dist && \
+RUN mkdir -p /vt/dist && \
+    cd /vt/dist && \
     curl -sL --connect-timeout 10 --retry 3 \
         http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && \
     mv apache-maven-3.3.9 maven
 
 # Set up Vitess environment (equivalent to '. dev.env')
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV VTPORTSTART 15000
 ENV PYTHONPATH $VTROOT/dist/grpc/usr/local/lib/python2.7/site-packages:$VTROOT/dist/py-mock-1.0.1/lib/python2.7/site-packages:$VTROOT/py-vtdb:$VTROOT/dist/selenium/lib/python2.7/site-packages
 ENV PATH $VTROOT/bin:$VTROOT/dist/maven/bin:$VTROOT/dist/chromedriver:$PATH
@@ -63,10 +64,6 @@ RUN cd /vt/src/vitess.io/vitess && \
 # Create mount point for actual data (e.g. MySQL data dir)
 VOLUME /vt/vtdataroot
 
-# The docker lite images copy from the builder in /vt/bin
-# Add compatibility to the previous layout for now
-RUN su vitess -c "mkdir -p /vt/src/vitess.io/vitess/bin && rm -rf /vt/bin && ln -s /vt/src/vitess.io/vitess/bin /vt/bin"
-
 # If the user doesn't specify a command, load a shell.
 CMD ["/bin/bash"]
 
diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb
index 19d5c9973d..e0f6106837 100644
--- a/docker/bootstrap/Dockerfile.mariadb
+++ b/docker/bootstrap/Dockerfile.mariadb
@@ -22,6 +22,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
-ENV MYSQL_FLAVOR MariaDB
 USER vitess
 RUN ./bootstrap.sh
diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103
index c2828ddd25..024fe6a80b 100644
--- a/docker/bootstrap/Dockerfile.mariadb103
+++ b/docker/bootstrap/Dockerfile.mariadb103
@@ -11,6 +11,5 @@ RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
-ENV MYSQL_FLAVOR MariaDB103
 USER vitess
 RUN ./bootstrap.sh
diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80
index 0523a7847c..e6d00bc524 100644
--- a/docker/bootstrap/Dockerfile.mysql80
+++ b/docker/bootstrap/Dockerfile.mysql80
@@ -17,6 +17,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.poo
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
-ENV MYSQL_FLAVOR MySQL80
 USER vitess
-RUN ./bootstrap.sh
+RUN ./bootstrap.sh
\ No newline at end of file
diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80
index 1ce9c52103..c5ce5d5ee4 100644
--- a/docker/bootstrap/Dockerfile.percona80
+++ b/docker/bootstrap/Dockerfile.percona80
@@ -32,6 +32,5 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess
 
-ENV MYSQL_FLAVOR MySQL80
 USER vitess
 RUN ./bootstrap.sh
diff --git a/docker/k8s/Dockerfile b/docker/k8s/Dockerfile
index 391c352e4f..ddafcd95af 100644
--- a/docker/k8s/Dockerfile
+++ b/docker/k8s/Dockerfile
@@ -25,8 +25,9 @@ RUN apt-get update && \
    rm -rf /var/lib/apt/lists/*
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
+ENV VTROOT /vt
 ENV VTDATAROOT /vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
 
 # Prepare directory structure.
 RUN mkdir -p /vt && \
@@ -49,13 +50,13 @@ COPY --from=base /vt/bin/vtworker /vt/bin/
 COPY --from=base /vt/bin/vtbackup /vt/bin/
 
 # copy web admin files
-COPY --from=base $VTROOT/web /vt/web/
+COPY --from=base $VTTOP/web /vt/web/
 
 # copy vitess config
-COPY --from=base $VTROOT/config/init_db.sql /vt/config/
+COPY --from=base $VTTOP/config/init_db.sql /vt/config/
 
 # my.cnf include files
-COPY --from=base $VTROOT/config/mycnf /vt/config/mycnf
+COPY --from=base $VTTOP/config/mycnf /vt/config/mycnf
 
 # add vitess user and add permissions
 RUN groupadd -r --gid 2000 vitess && useradd -r -g vitess --uid 1000 vitess && \
diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile
index a63321a42f..f110b58fc6 100644
--- a/docker/lite/Dockerfile
+++ b/docker/lite/Dockerfile
@@ -62,8 +62,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
  && rm -rf /var/lib/mysql/
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
 
 # Copy binaries (placed by build.sh)
diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine
index f57b9d7a8f..a50fec877d 100644
--- a/docker/lite/Dockerfile.alpine
+++ b/docker/lite/Dockerfile.alpine
@@ -22,10 +22,10 @@ RUN echo '@edge http://nl.alpinelinux.org/alpine/edge/main' >> /etc/apk/reposito
     apk add --no-cache mariadb@edge mariadb-client@edge bzip2 bash
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
-ENV MYSQL_FLAVOR MariaDB103
 
 # Create vitess user
 RUN addgroup -S vitess && adduser -S -G vitess vitess && mkdir -p /vt
diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb
index 6907534c03..b845e19bec 100644
--- a/docker/lite/Dockerfile.mariadb
+++ b/docker/lite/Dockerfile.mariadb
@@ -33,10 +33,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
  && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
-ENV MYSQL_FLAVOR MariaDB
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/
diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103
index cd962803f7..a07de74fca 100644
--- a/docker/lite/Dockerfile.mariadb103
+++ b/docker/lite/Dockerfile.mariadb103
@@ -32,10 +32,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
  && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
-ENV MYSQL_FLAVOR MariaDB103
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/
diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56
index bb43480837..0a771265c3 100644
--- a/docker/lite/Dockerfile.mysql56
+++ b/docker/lite/Dockerfile.mysql56
@@ -36,8 +36,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
   && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
 
 # Copy binaries (placed by build.sh)
diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57
index 809a79f915..8e605fda62 100644
--- a/docker/lite/Dockerfile.mysql57
+++ b/docker/lite/Dockerfile.mysql57
@@ -36,8 +36,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
   && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
 
 # Copy binaries (placed by build.sh)
diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80
index 85383be648..bb6d5d54be 100644
--- a/docker/lite/Dockerfile.mysql80
+++ b/docker/lite/Dockerfile.mysql80
@@ -36,10 +36,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
   && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
-ENV MYSQL_FLAVOR MySQL80
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/
diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona
index 0698f4eb58..2867aa9c33 100644
--- a/docker/lite/Dockerfile.percona
+++ b/docker/lite/Dockerfile.percona
@@ -38,8 +38,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
 
 # Copy binaries (placed by build.sh)
diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57
index d52fbaa583..ef01888a16 100644
--- a/docker/lite/Dockerfile.percona57
+++ b/docker/lite/Dockerfile.percona57
@@ -39,8 +39,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
 
 # Copy binaries (placed by build.sh)
diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80
index 98b4b30a57..02c56f48b9 100644
--- a/docker/lite/Dockerfile.percona80
+++ b/docker/lite/Dockerfile.percona80
@@ -41,10 +41,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
-ENV VTROOT /vt/src/vitess.io/vitess
-ENV VTDATAROOT /vt/vtdataroot
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV VTDATAROOT $VTROOT/vtdataroot
 ENV PATH $VTROOT/bin:$PATH
-ENV MYSQL_FLAVOR MySQL80
 
 # Copy binaries (placed by build.sh)
 COPY --from=staging /vt/ /vt/
diff --git a/examples/compose/docker-compose.beginners.yml b/examples/compose/docker-compose.beginners.yml
index 67294b90c6..4f23b688f1 100644
--- a/examples/compose/docker-compose.beginners.yml
+++ b/examples/compose/docker-compose.beginners.yml
@@ -37,8 +37,8 @@ services:
     command: ["sh", "-c", " $$VTROOT/bin/vtctld \
         $TOPOLOGY_FLAGS \
         -cell $CELL \
-        -web_dir $$VTROOT/web/vtctld \
-        -web_dir2 $$VTROOT/web/vtctld2/app \
+        -web_dir $$VTTOP/web/vtctld \
+        -web_dir2 $$VTTOP/web/vtctld2/app \
         -workflow_manager_init \
         -workflow_manager_use_election \
         -service_map 'grpc-vtctl' \
diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go
index d4825214ad..54fa86e2b1 100644
--- a/examples/compose/vtcompose/vtcompose.go
+++ b/examples/compose/vtcompose/vtcompose.go
@@ -466,8 +466,8 @@ func generateVtctld() string {
     command: ["sh", "-c", " $$VTROOT/bin/vtctld \
         %[3]s \
         -cell %[4]s \
-        -web_dir $$VTROOT/web/vtctld \
-        -web_dir2 $$VTROOT/web/vtctld2/app \
+        -web_dir $$VTTOP/web/vtctld \
+        -web_dir2 $$VTTOP/web/vtctld2/app \
         -workflow_manager_init \
         -workflow_manager_use_election \
         -service_map 'grpc-vtctl' \
diff --git a/examples/demo/run.py b/examples/demo/run.py
index 5f8916bb17..d9a9aa2792 100755
--- a/examples/demo/run.py
+++ b/examples/demo/run.py
@@ -51,8 +51,8 @@ def start_vitess():
   keyspace = topology.keyspaces.add(name='lookup')
   keyspace.shards.add(name='0')
 
-  vtroot = os.environ['VTROOT']
-  args = [os.path.join(vtroot, 'py/vttest/run_local_database.py'),
+  vttop = os.environ['VTTOP']
+  args = [os.path.join(vttop, 'py/vttest/run_local_database.py'),
           '--port', '12345',
           '--proto_topo', text_format.MessageToString(topology,
                                                       as_one_line=True),
diff --git a/examples/kubernetes/guestbook/extract.sh b/examples/kubernetes/guestbook/extract.sh
index 04499c8739..e029df3fc7 100644
--- a/examples/kubernetes/guestbook/extract.sh
+++ b/examples/kubernetes/guestbook/extract.sh
@@ -17,7 +17,7 @@ set -e
 
 # Collect all the local Python libs we need.
 mkdir -p /out/pkg/py-vtdb
-cp -R $VTROOT/py/* /out/pkg/py-vtdb/
+cp -R $VTTOP/py/* /out/pkg/py-vtdb/
 cp -R /usr/local/lib/python2.7/dist-packages /out/pkg/
 cp -R /vt/dist/py-* /out/pkg/
 
diff --git a/examples/kubernetes/vtctld-controller-template.yaml b/examples/kubernetes/vtctld-controller-template.yaml
index 4e28aa8bfc..2ca6f6d80c 100644
--- a/examples/kubernetes/vtctld-controller-template.yaml
+++ b/examples/kubernetes/vtctld-controller-template.yaml
@@ -41,8 +41,8 @@ spec:
               chown -R vitess /vt &&
               su -p -c "/vt/bin/vtctld
               -cell {{cell}}
-              -web_dir $VTROOT/web/vtctld
-              -web_dir2 $VTROOT/web/vtctld2/app
+              -web_dir $VTTOP/web/vtctld
+              -web_dir2 $VTTOP/web/vtctld2/app
               -workflow_manager_init
               -workflow_manager_use_election
               -log_dir $VTDATAROOT/tmp
diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh
index 1a484d98d5..9f018da6b5 100755
--- a/examples/local/101_initial_cluster.sh
+++ b/examples/local/101_initial_cluster.sh
@@ -21,7 +21,11 @@ set -e
 
 # shellcheck disable=SC2128
 script_root=$(dirname "${BASH_SOURCE}")
-source "${script_root}/env.sh"
+
+if [[ $EUID -eq 0 ]]; then
+   echo "This script refuses to be run as root. Please switch to a regular user."
+   exit 1
+fi
 
 # start topo server
 if [ "${TOPO}" = "zk2" ]; then
diff --git a/examples/local/env.sh b/examples/local/env.sh
index d36a1cc3ff..584987bb92 100644
--- a/examples/local/env.sh
+++ b/examples/local/env.sh
@@ -16,32 +16,16 @@
 
 hostname=`hostname -f`
 vtctld_web_port=15000
-export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}"
 
-function fail() {
-  echo "ERROR: $1"
-  exit 1
-}
-
-if [[ $EUID -eq 0 ]]; then
-  fail "This script refuses to be run as root. Please switch to a regular user."
-fi
-
-# mysqld might be in /usr/sbin which will not be in the default PATH
-PATH="/usr/sbin:$PATH"
-for binary in mysqld etcd etcdctl curl vtctlclient vttablet vtgate vtctld mysqlctl; do
-  command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions."
-done;
-
-if [ -z "$VTROOT" ]; then
-  fail "VTROOT is not set. See https://vitess.io/docs/get-started/local/ for install instructions."
-fi
+# Set up environment.
+export VTTOP=${VTTOP-$VTROOT/src/vitess.io/vitess}
 
 if [ "${TOPO}" = "zk2" ]; then
     # Each ZooKeeper server needs a list of all servers in the quorum.
     # Since we're running them all locally, we need to give them unique ports.
     # In a real deployment, these should be on different machines, and their
     # respective hostnames should be given.
+    echo "enter zk2 env"
     zkcfg=(\
         "1@$hostname:28881:38881:21811" \
         "2@$hostname:28882:38882:21812" \
@@ -59,6 +43,8 @@ if [ "${TOPO}" = "zk2" ]; then
 
     mkdir -p $VTDATAROOT/tmp
 else
+    echo "enter etcd2 env"
+
     ETCD_SERVER="localhost:2379"
     TOPOLOGY_FLAGS="-topo_implementation etcd2 -topo_global_server_address $ETCD_SERVER -topo_global_root /vitess/global"
 
diff --git a/examples/local/etcd-up.sh b/examples/local/etcd-up.sh
index 36cd426956..12207e23b8 100755
--- a/examples/local/etcd-up.sh
+++ b/examples/local/etcd-up.sh
@@ -26,9 +26,6 @@ export ETCDCTL_API=2
 # shellcheck disable=SC1091
 source "${script_root}/env.sh"
 
-# Check that etcd is not already running
-curl "http://${ETCD_SERVER}" > /dev/null 2>&1 && fail "etcd is already running. Exiting."
-
 etcd --enable-v2=true --data-dir "${VTDATAROOT}/etcd/"  --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 &
 PID=$!
 echo $PID > "${VTDATAROOT}/tmp/etcd.pid"
diff --git a/examples/local/vtctld-up.sh b/examples/local/vtctld-up.sh
index 128152b837..dbd16a8de6 100755
--- a/examples/local/vtctld-up.sh
+++ b/examples/local/vtctld-up.sh
@@ -36,8 +36,8 @@ echo "Starting vtctld..."
 $VTROOT/bin/vtctld \
   $TOPOLOGY_FLAGS \
   -cell $cell \
-  -web_dir $VTROOT/web/vtctld \
-  -web_dir2 $VTROOT/web/vtctld2/app \
+  -web_dir $VTTOP/web/vtctld \
+  -web_dir2 $VTTOP/web/vtctld2/app \
   -workflow_manager_init \
   -workflow_manager_use_election \
   -service_map 'grpc-vtctl' \
diff --git a/go.mod b/go.mod
index af52af7313..6308bc259c 100644
--- a/go.mod
+++ b/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/dchest/safefile v0.0.0-20151022103144-855e8d98f185 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
-	github.com/evanphx/json-patch v4.5.0+incompatible
+	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
 	github.com/ghodss/yaml v0.0.0-20161207003320-04f313413ffd // indirect
 	github.com/go-ini/ini v1.12.0 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
@@ -25,7 +25,6 @@ require (
 	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.3.2
 	github.com/golang/snappy v0.0.0-20170215233205-553a64147049
-	github.com/google/btree v1.0.0 // indirect
 	github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect
 	github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
@@ -45,13 +44,11 @@ require (
 	github.com/klauspost/crc32 v1.2.0 // indirect
 	github.com/klauspost/pgzip v1.2.0
 	github.com/kr/pretty v0.1.0 // indirect
-	github.com/krishicks/yaml-patch v0.0.10
+	github.com/krishicks/yaml-patch v0.0.10 // indirect
 	github.com/mattn/go-runewidth v0.0.1 // indirect
 	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984
 	github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02
 	github.com/opentracing/opentracing-go v1.1.0
diff --git a/go/mysql/encoding.go b/go/mysql/encoding.go
index 53b9e2efc7..c9333522cd 100644
--- a/go/mysql/encoding.go
+++ b/go/mysql/encoding.go
@@ -281,17 +281,3 @@ func readLenEncStringAsBytes(data []byte, pos int) ([]byte, int, bool) {
 	}
 	return data[pos : pos+s], pos + s, true
 }
-
-func readLenEncStringAsBytesCopy(data []byte, pos int) ([]byte, int, bool) {
-	size, pos, ok := readLenEncInt(data, pos)
-	if !ok {
-		return nil, 0, false
-	}
-	s := int(size)
-	if pos+s-1 >= len(data) {
-		return nil, 0, false
-	}
-	result := make([]byte, size)
-	copy(result, data[pos:pos+s])
-	return result, pos + s, true
-}
diff --git a/go/mysql/query.go b/go/mysql/query.go
index f6435bf4b6..6ff9d1e9bc 100644
--- a/go/mysql/query.go
+++ b/go/mysql/query.go
@@ -261,7 +261,7 @@ func (c *Conn) parseRow(data []byte, fields []*querypb.Field) ([]sqltypes.Value,
 		}
 		var s []byte
 		var ok bool
-		s, pos, ok = readLenEncStringAsBytesCopy(data, pos)
+		s, pos, ok = readLenEncStringAsBytes(data, pos)
 		if !ok {
 			return nil, NewSQLError(CRMalformedPacket, SSUnknownSQLState, "decoding string failed")
 		}
@@ -823,7 +823,7 @@ func (c *Conn) parseStmtArgs(data []byte, typ querypb.Type, pos int) (sqltypes.V
 		}
 	case sqltypes.Decimal, sqltypes.Text, sqltypes.Blob, sqltypes.VarChar, sqltypes.VarBinary, sqltypes.Char,
 		sqltypes.Bit, sqltypes.Enum, sqltypes.Set, sqltypes.Geometry, sqltypes.Binary, sqltypes.TypeJSON:
-		val, pos, ok := readLenEncStringAsBytesCopy(data, pos)
+		val, pos, ok := readLenEncStringAsBytes(data, pos)
 		return sqltypes.MakeTrusted(sqltypes.VarBinary, val), pos, ok
 	default:
 		return sqltypes.NULL, pos, false
diff --git a/go/vt/servenv/status.go b/go/vt/servenv/status.go
index c9fdbfb3fd..f391bf470a 100644
--- a/go/vt/servenv/status.go
+++ b/go/vt/servenv/status.go
@@ -104,7 +104,6 @@ Started: {{.StartTime}}<br>
 Running on {{.Hostname}}<br>
 View <a href=/debug/vars>variables</a>,
      <a href=/debug/pprof>debugging profiles</a>,
-     <a href=/debug/scatter_stats>scatter query statistics</a>,
 </div>
 </div>`
 
diff --git a/go/vt/sqlparser/comments.go b/go/vt/sqlparser/comments.go
index afffd326c8..26c664cf91 100644
--- a/go/vt/sqlparser/comments.go
+++ b/go/vt/sqlparser/comments.go
@@ -123,9 +123,7 @@ func SplitMarginComments(sql string) (query string, comments MarginComments) {
 		Leading:  strings.TrimLeftFunc(sql[:leadingEnd], unicode.IsSpace),
 		Trailing: strings.TrimRightFunc(sql[trailingStart:], unicode.IsSpace),
 	}
-	return strings.TrimFunc(sql[leadingEnd:trailingStart], func(c rune) bool {
-		return unicode.IsSpace(c) || c == ';'
-	}), comments
+	return strings.TrimFunc(sql[leadingEnd:trailingStart], unicode.IsSpace), comments
 }
 
 // StripLeadingComments trims the SQL string and removes any leading comments
diff --git a/go/vt/topo/etcd2topo/server.go b/go/vt/topo/etcd2topo/server.go
index 5d67a6f010..8e6b2588a4 100644
--- a/go/vt/topo/etcd2topo/server.go
+++ b/go/vt/topo/etcd2topo/server.go
@@ -34,21 +34,13 @@ We follow these conventions within this package:
 package etcd2topo
 
 import (
-	"flag"
 	"strings"
 	"time"
 
 	"github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/pkg/transport"
 	"vitess.io/vitess/go/vt/topo"
 )
 
-var (
-	clientCertPath = flag.String("topo_etcd_tls_cert", "", "path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS")
-	clientKeyPath  = flag.String("topo_etcd_tls_key", "", "path to the client key to use to connect to the etcd topo server, enables TLS")
-	serverCaPath   = flag.String("topo_etcd_tls_ca", "", "path to the ca to use to validate the server cert when connecting to the etcd topo server")
-)
-
 // Factory is the consul topo.Factory implementation.
 type Factory struct{}
 
@@ -79,31 +71,12 @@ func (s *Server) Close() {
 	s.cli = nil
 }
 
-func NewServerWithOpts(serverAddr, root, certPath, keyPath, caPath string) (*Server, error) {
-	// TODO: Rename this to NewServer and change NewServer to a name that signifies it uses the process-wide TLS settings.
-
-	config := clientv3.Config{
+// NewServer returns a new etcdtopo.Server.
+func NewServer(serverAddr, root string) (*Server, error) {
+	cli, err := clientv3.New(clientv3.Config{
 		Endpoints:   strings.Split(serverAddr, ","),
 		DialTimeout: 5 * time.Second,
-	}
-
-	// If TLS is enabled, attach TLS config info.
-	if certPath != "" && keyPath != "" {
-		// Safe now to build up TLS info.
-		tlsInfo := transport.TLSInfo{
-			CertFile:      certPath,
-			KeyFile:       keyPath,
-			TrustedCAFile: caPath,
-		}
-
-		tlsConfig, err := tlsInfo.ClientConfig()
-		if err != nil {
-			return nil, err
-		}
-		config.TLS = tlsConfig
-	}
-
-	cli, err := clientv3.New(config)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -114,13 +87,6 @@ func NewServerWithOpts(serverAddr, root, certPath, keyPath, caPath string) (*Ser
 	}, nil
 }
 
-// NewServer returns a new etcdtopo.Server.
-func NewServer(serverAddr, root string) (*Server, error) {
-	// TODO: Rename this to a name to signifies this function uses the process-wide TLS settings.
-
-	return NewServerWithOpts(serverAddr, root, *clientCertPath, *clientKeyPath, *serverCaPath)
-}
-
 func init() {
 	topo.RegisterFactory("etcd2", Factory{})
 }
diff --git a/go/vt/vitessdriver/doc.go b/go/vt/vitessdriver/doc.go
index 31ea98651b..a3e2770b8a 100644
--- a/go/vt/vitessdriver/doc.go
+++ b/go/vt/vitessdriver/doc.go
@@ -28,12 +28,13 @@ Example
 Using this SQL driver is as simple as:
 
   import (
+    "time"
     "vitess.io/vitess/go/vt/vitessdriver"
   )
 
   func main() {
     // Connect to vtgate.
-    db, err := vitessdriver.Open("localhost:15991", "@master")
+    db, err := vitessdriver.Open("localhost:15991", "keyspace", "master", 30*time.Second)
 
     // Use "db" via the Golang sql interface.
   }
diff --git a/go/vt/vtgate/engine/delete.go b/go/vt/vtgate/engine/delete.go
index b0f06174c6..22e2fd61a5 100644
--- a/go/vt/vtgate/engine/delete.go
+++ b/go/vt/vtgate/engine/delete.go
@@ -67,9 +67,6 @@ type Delete struct {
 
 	// QueryTimeout contains the optional timeout (in milliseconds) to apply to this query
 	QueryTimeout int
-
-	// Delete does not take inputs
-	noInputs
 }
 
 // MarshalJSON serializes the Delete into a JSON representation.
diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go
index ec2a54f106..bf75e6a446 100644
--- a/go/vt/vtgate/engine/insert.go
+++ b/go/vt/vtgate/engine/insert.go
@@ -80,9 +80,6 @@ type Insert struct {
 
 	// QueryTimeout contains the optional timeout (in milliseconds) to apply to this query
 	QueryTimeout int
-
-	// Insert does not take inputs
-	noInputs
 }
 
 // NewQueryInsert creates an Insert with a query string.
diff --git a/go/vt/vtgate/engine/join.go b/go/vt/vtgate/engine/join.go
index e577986605..3cfb25e7a1 100644
--- a/go/vt/vtgate/engine/join.go
+++ b/go/vt/vtgate/engine/join.go
@@ -171,11 +171,6 @@ func (jn *Join) GetFields(vcursor VCursor, bindVars map[string]*querypb.BindVari
 	return result, nil
 }
 
-// Inputs returns the input primitives for this join
-func (jn *Join) Inputs() []Primitive {
-	return []Primitive{jn.Left, jn.Right}
-}
-
 func joinFields(lfields, rfields []*querypb.Field, cols []int) []*querypb.Field {
 	fields := make([]*querypb.Field, len(cols))
 	for i, index := range cols {
diff --git a/go/vt/vtgate/engine/limit.go b/go/vt/vtgate/engine/limit.go
index 410a5af624..65b6b39da3 100644
--- a/go/vt/vtgate/engine/limit.go
+++ b/go/vt/vtgate/engine/limit.go
@@ -161,11 +161,6 @@ func (l *Limit) GetFields(vcursor VCursor, bindVars map[string]*querypb.BindVari
 	return l.Input.GetFields(vcursor, bindVars)
 }
 
-// Inputs returns the input to limit
-func (l *Limit) Inputs() []Primitive {
-	return []Primitive{l.Input}
-}
-
 func (l *Limit) fetchCount(bindVars map[string]*querypb.BindVariable) (int, error) {
 	resolved, err := l.Count.ResolveValue(bindVars)
 	if err != nil {
diff --git a/go/vt/vtgate/engine/memory_sort.go b/go/vt/vtgate/engine/memory_sort.go
index f426eb2039..b9832b34e5 100644
--- a/go/vt/vtgate/engine/memory_sort.go
+++ b/go/vt/vtgate/engine/memory_sort.go
@@ -78,7 +78,7 @@ func (ms *MemorySort) SetTruncateColumnCount(count int) {
 	ms.TruncateColumnCount = count
 }
 
-// Execute satisfies the Primitive interface.
+// Execute satisfies the Primtive interface.
 func (ms *MemorySort) Execute(vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool) (*sqltypes.Result, error) {
 	count, err := ms.fetchCount(bindVars)
 	if err != nil {
@@ -105,7 +105,7 @@ func (ms *MemorySort) Execute(vcursor VCursor, bindVars map[string]*querypb.Bind
 	return result.Truncate(ms.TruncateColumnCount), nil
 }
 
-// StreamExecute satisfies the Primitive interface.
+// StreamExecute satisfies the Primtive interface.
 func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error {
 	count, err := ms.fetchCount(bindVars)
 	if err != nil {
@@ -155,16 +155,11 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp
 	return cb(&sqltypes.Result{Rows: sh.rows})
 }
 
-// GetFields satisfies the Primitive interface.
+// GetFields satisfies the Primtive interface.
 func (ms *MemorySort) GetFields(vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) {
 	return ms.Input.GetFields(vcursor, bindVars)
 }
 
-// Inputs returns the input to memory sort
-func (ms *MemorySort) Inputs() []Primitive {
-	return []Primitive{ms.Input}
-}
-
 func (ms *MemorySort) fetchCount(bindVars map[string]*querypb.BindVariable) (int, error) {
 	resolved, err := ms.UpperLimit.ResolveValue(bindVars)
 	if err != nil {
diff --git a/go/vt/vtgate/engine/ordered_aggregate.go b/go/vt/vtgate/engine/ordered_aggregate.go
index 9ec98b35d2..9abe441921 100644
--- a/go/vt/vtgate/engine/ordered_aggregate.go
+++ b/go/vt/vtgate/engine/ordered_aggregate.go
@@ -309,11 +309,6 @@ func (oa *OrderedAggregate) GetFields(vcursor VCursor, bindVars map[string]*quer
 	return qr.Truncate(oa.TruncateColumnCount), nil
 }
 
-// Inputs returns the Primitive input for this aggregation
-func (oa *OrderedAggregate) Inputs() []Primitive {
-	return []Primitive{oa.Input}
-}
-
 func (oa *OrderedAggregate) keysEqual(row1, row2 []sqltypes.Value) (bool, error) {
 	for _, key := range oa.Keys {
 		cmp, err := sqltypes.NullsafeCompare(row1[key], row2[key])
diff --git a/go/vt/vtgate/engine/primitive.go b/go/vt/vtgate/engine/primitive.go
index b69d6ea373..cb9c916118 100644
--- a/go/vt/vtgate/engine/primitive.go
+++ b/go/vt/vtgate/engine/primitive.go
@@ -119,28 +119,6 @@ func (p *Plan) Stats() (execCount uint64, execTime time.Duration, shardQueries,
 	return
 }
 
-// Match is used to check if a Primitive matches
-type Match func(node Primitive) bool
-
-// Find will return the first Primitive that matches the evaluate function. If no match is found, nil will be returned
-func Find(isMatch Match, start Primitive) Primitive {
-	if isMatch(start) {
-		return start
-	}
-	for _, input := range start.Inputs() {
-		result := Find(isMatch, input)
-		if result != nil {
-			return result
-		}
-	}
-	return nil
-}
-
-// Exists traverses recursively down the Primitive tree structure, and returns true when Match returns true
-func Exists(m Match, p Primitive) bool {
-	return Find(m, p) != nil
-}
-
 // Size is defined so that Plan can be given to a cache.LRUCache.
 // VTGate needs to maintain a cache of plans. It uses LRUCache, which
 // in turn requires its objects to define a Size function.
@@ -148,10 +126,8 @@ func (p *Plan) Size() int {
 	return 1
 }
 
-// Primitive is the building block of the engine execution plan. They form a tree structure, where the leaves typically
-// issue queries to one or more vttablet.
-// During execution, the Primitive's pass Result objects up the tree structure, until reaching the root,
-// and its result is passed to the client.
+// Primitive is the interface that needs to be satisfied by
+// all primitives of a plan.
 type Primitive interface {
 	RouteType() string
 	GetKeyspaceName() string
@@ -159,14 +135,4 @@ type Primitive interface {
 	Execute(vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool) (*sqltypes.Result, error)
 	StreamExecute(vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantields bool, callback func(*sqltypes.Result) error) error
 	GetFields(vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error)
-
-	// The inputs to this Primitive
-	Inputs() []Primitive
-}
-
-type noInputs struct{}
-
-// Inputs implements no inputs
-func (noInputs) Inputs() []Primitive {
-	return nil
 }
diff --git a/go/vt/vtgate/engine/pullout_subquery.go b/go/vt/vtgate/engine/pullout_subquery.go
index ade790599a..b632d37359 100644
--- a/go/vt/vtgate/engine/pullout_subquery.go
+++ b/go/vt/vtgate/engine/pullout_subquery.go
@@ -38,11 +38,6 @@ type PulloutSubquery struct {
 	Underlying     Primitive
 }
 
-// Inputs returns the input primitives for this join
-func (ps *PulloutSubquery) Inputs() []Primitive {
-	return []Primitive{ps.Subquery, ps.Underlying}
-}
-
 // RouteType returns a description of the query routing type used by the primitive
 func (ps *PulloutSubquery) RouteType() string {
 	return ps.Opcode.String()
diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go
index 9d5b45471b..4576894f13 100644
--- a/go/vt/vtgate/engine/route.go
+++ b/go/vt/vtgate/engine/route.go
@@ -84,9 +84,6 @@ type Route struct {
 
 	// ScatterErrorsAsWarnings is true if results should be returned even if some shards have an error
 	ScatterErrorsAsWarnings bool
-
-	// Route does not take inputs
-	noInputs
 }
 
 // NewSimpleRoute creates a Route with the bare minimum of parameters.
diff --git a/go/vt/vtgate/engine/subquery.go b/go/vt/vtgate/engine/subquery.go
index fc37cd9b3e..b9e28885dd 100644
--- a/go/vt/vtgate/engine/subquery.go
+++ b/go/vt/vtgate/engine/subquery.go
@@ -71,11 +71,6 @@ func (sq *Subquery) GetFields(vcursor VCursor, bindVars map[string]*querypb.Bind
 	return &sqltypes.Result{Fields: sq.buildFields(inner)}, nil
 }
 
-// Inputs returns the input to this primitive
-func (sq *Subquery) Inputs() []Primitive {
-	return []Primitive{sq.Subquery}
-}
-
 // buildResult builds a new result by pulling the necessary columns from
 // the subquery in the requested order.
 func (sq *Subquery) buildResult(inner *sqltypes.Result) *sqltypes.Result {
diff --git a/go/vt/vtgate/engine/update.go b/go/vt/vtgate/engine/update.go
index 49ee8324b9..dc3c591546 100644
--- a/go/vt/vtgate/engine/update.go
+++ b/go/vt/vtgate/engine/update.go
@@ -70,9 +70,6 @@ type Update struct {
 
 	// QueryTimeout contains the optional timeout (in milliseconds) to apply to this query
 	QueryTimeout int
-
-	// Update does not take inputs
-	noInputs
 }
 
 // MarshalJSON serializes the Update into a JSON representation.
diff --git a/go/vt/vtgate/engine/vindex_func.go b/go/vt/vtgate/engine/vindex_func.go
index bc1aaf61da..8ffb591405 100644
--- a/go/vt/vtgate/engine/vindex_func.go
+++ b/go/vt/vtgate/engine/vindex_func.go
@@ -39,9 +39,6 @@ type VindexFunc struct {
 	// TODO(sougou): add support for MultiColumn.
 	Vindex vindexes.SingleColumn
 	Value  sqltypes.PlanValue
-
-	// VindexFunc does not take inputs
-	noInputs
 }
 
 // MarshalJSON serializes the VindexFunc into a JSON representation.
diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go
index 63e3a0bdfb..b3c9e2c898 100644
--- a/go/vt/vtgate/executor.go
+++ b/go/vt/vtgate/executor.go
@@ -92,10 +92,6 @@ type Executor struct {
 
 var executorOnce sync.Once
 
-const pathQueryPlans = "/debug/query_plans"
-const pathScatterStats = "/debug/scatter_stats"
-const pathVSchema = "/debug/vschema"
-
 // NewExecutor creates a new Executor.
 func NewExecutor(ctx context.Context, serv srvtopo.Server, cell, statsName string, resolver *Resolver, normalize bool, streamSize int, queryPlanCacheSize int64) *Executor {
 	e := &Executor{
@@ -121,9 +117,8 @@ func NewExecutor(ctx context.Context, serv srvtopo.Server, cell, statsName strin
 		stats.Publish("QueryPlanCacheOldest", stats.StringFunc(func() string {
 			return fmt.Sprintf("%v", e.plans.Oldest())
 		}))
-		http.Handle(pathQueryPlans, e)
-		http.Handle(pathScatterStats, e)
-		http.Handle(pathVSchema, e)
+		http.Handle("/debug/query_plans", e)
+		http.Handle("/debug/vschema", e)
 	})
 	return e
 }
@@ -845,14 +840,6 @@ func (e *Executor) handleShow(ctx context.Context, safeSession *SafeSession, sql
 				}
 			}
 		}
-	case sqlparser.KeywordString(sqlparser.COLUMNS):
-		if !show.OnTable.Qualifier.IsEmpty() {
-			destKeyspace = show.OnTable.Qualifier.String()
-			show.OnTable.Qualifier = sqlparser.NewTableIdent("")
-		} else {
-			break
-		}
-		sql = sqlparser.String(show)
 	case sqlparser.KeywordString(sqlparser.TABLES):
 		if show.ShowTablesOpt != nil && show.ShowTablesOpt.DbName != "" {
 			if destKeyspace == "" {
@@ -1432,31 +1419,31 @@ func (e *Executor) ServeHTTP(response http.ResponseWriter, request *http.Request
 		acl.SendError(response, err)
 		return
 	}
-
-	switch request.URL.Path {
-	case pathQueryPlans:
-		returnAsJSON(response, e.plans.Items())
-	case pathVSchema:
-		returnAsJSON(response, e.VSchema())
-	case pathScatterStats:
-		e.WriteScatterStats(response)
-	default:
+	if request.URL.Path == "/debug/query_plans" {
+		response.Header().Set("Content-Type", "application/json; charset=utf-8")
+		buf, err := json.MarshalIndent(e.plans.Items(), "", " ")
+		if err != nil {
+			response.Write([]byte(err.Error()))
+			return
+		}
+		ebuf := bytes.NewBuffer(nil)
+		json.HTMLEscape(ebuf, buf)
+		response.Write(ebuf.Bytes())
+	} else if request.URL.Path == "/debug/vschema" {
+		response.Header().Set("Content-Type", "application/json; charset=utf-8")
+		b, err := json.MarshalIndent(e.VSchema(), "", " ")
+		if err != nil {
+			response.Write([]byte(err.Error()))
+			return
+		}
+		buf := bytes.NewBuffer(nil)
+		json.HTMLEscape(buf, b)
+		response.Write(buf.Bytes())
+	} else {
 		response.WriteHeader(http.StatusNotFound)
 	}
 }
 
-func returnAsJSON(response http.ResponseWriter, stuff interface{}) {
-	response.Header().Set("Content-Type", "application/json; charset=utf-8")
-	buf, err := json.MarshalIndent(stuff, "", " ")
-	if err != nil {
-		_, _ = response.Write([]byte(err.Error()))
-		return
-	}
-	ebuf := bytes.NewBuffer(nil)
-	json.HTMLEscape(ebuf, buf)
-	_, _ = response.Write(ebuf.Bytes())
-}
-
 // Plans returns the LRU plan cache
 func (e *Executor) Plans() *cache.LRUCache {
 	return e.plans
diff --git a/go/vt/vtgate/executor_scatter_stats.go b/go/vt/vtgate/executor_scatter_stats.go
deleted file mode 100644
index 24858564a5..0000000000
--- a/go/vt/vtgate/executor_scatter_stats.go
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
-Copyright 2019 The Vitess Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package vtgate
-
-import (
-	"fmt"
-	"html/template"
-	"net/http"
-	"time"
-
-	"vitess.io/vitess/go/vt/logz"
-
-	"vitess.io/vitess/go/vt/proto/vtrpc"
-	"vitess.io/vitess/go/vt/vterrors"
-	"vitess.io/vitess/go/vt/vtgate/engine"
-)
-
-type statsResults struct {
-	TotalReadOnlyTime  time.Duration
-	TotalScatterTime   time.Duration
-	PercentTimeScatter float64
-	Items              []*statsResultItem
-}
-
-// using this intermediate data structure to make testing easier
-type statsResultItem struct {
-	Query                  string
-	AvgTimePerQuery        time.Duration
-	PercentTimeOfReads     float64
-	PercentTimeOfScatters  float64
-	PercentCountOfReads    float64
-	PercentCountOfScatters float64
-	From                   string
-	Count                  uint64
-}
-
-func (e *Executor) gatherScatterStats() (statsResults, error) {
-	scatterExecTime := time.Duration(0)
-	readOnlyTime := time.Duration(0)
-	scatterCount := uint64(0)
-	readOnlyCount := uint64(0)
-	totalExecTime := time.Duration(0)
-	totalCount := uint64(0)
-
-	plans := make([]*engine.Plan, 0)
-	routes := make([]*engine.Route, 0)
-	// First we go over all plans and collect statistics and all query plans for scatter queries
-	for _, item := range e.plans.Items() {
-		plan := item.Value.(*engine.Plan)
-
-		scatter := engine.Find(findScatter, plan.Instructions)
-		readOnly := !engine.Exists(isUpdating, plan.Instructions)
-		isScatter := scatter != nil
-
-		if isScatter {
-			route, isRoute := scatter.(*engine.Route)
-			if !isRoute {
-				return statsResults{}, vterrors.Errorf(vtrpc.Code_INTERNAL, "expected a route, but found a %v", scatter)
-			}
-			plans = append(plans, plan)
-			routes = append(routes, route)
-			scatterExecTime += plan.ExecTime
-			scatterCount += plan.ExecCount
-		}
-		if readOnly {
-			readOnlyTime += plan.ExecTime
-			readOnlyCount += plan.ExecCount
-		}
-
-		totalExecTime += plan.ExecTime
-		totalCount += plan.ExecCount
-	}
-
-	// Now we'll go over all scatter queries we've found and produce result items for each
-	resultItems := make([]*statsResultItem, len(plans))
-	for i, plan := range plans {
-		route := routes[i]
-		resultItems[i] = &statsResultItem{
-			Query:                  plan.Original,
-			AvgTimePerQuery:        time.Duration(plan.ExecTime.Nanoseconds() / int64(plan.ExecCount)),
-			PercentTimeOfReads:     float64(100 * plan.ExecTime / readOnlyTime),
-			PercentTimeOfScatters:  float64(100 * plan.ExecTime / scatterExecTime),
-			PercentCountOfReads:    float64(100 * plan.ExecCount / readOnlyCount),
-			PercentCountOfScatters: float64(100 * plan.ExecCount / scatterCount),
-			From:                   route.Keyspace.Name + "." + route.TableName,
-			Count:                  plan.ExecCount,
-		}
-	}
-	result := statsResults{
-		TotalReadOnlyTime:  readOnlyTime,
-		TotalScatterTime:   scatterExecTime,
-		PercentTimeScatter: 100 * float64(scatterExecTime) / float64(readOnlyTime),
-		Items:              resultItems,
-	}
-	return result, nil
-}
-
-// WriteScatterStats will write an html report to the provided response writer
-func (e *Executor) WriteScatterStats(w http.ResponseWriter) {
-	logz.StartHTMLTable(w)
-	defer logz.EndHTMLTable(w)
-
-	results, err := e.gatherScatterStats()
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-
-	t := template.New("template")
-	t, err = t.Parse(html)
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-
-	err = t.Execute(w, results)
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-	}
-
-	_, err = w.Write([]byte(fmt.Sprintf("Percentage of time spent on scatter queries: %2.2f%%", results.PercentTimeScatter)))
-
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-	}
-
-}
-
-const html = `
-<thead>
-	<tr>
-		<th>Query</th>
-		<th># of executions</th>
-		<th>Avg time/query</th>
-		<th>% time of reads</th>
-		<th>% time of scatters</th>
-		<th>% of reads</th>
-		<th>% of scatters</th>
-	</tr>
-</thead>
-{{range .Items}}
-<tr class="medium">
-	<td>{{.Query}}</td>
-	<td>{{.Count}}</td>
-	<td>{{.AvgTimePerQuery}}</td>
-	<td>{{.PercentTimeOfReads}}</td>
-	<td>{{.PercentTimeOfScatters}}</td>
-	<td>{{.PercentCountOfReads}}</td>
-	<td>{{.PercentCountOfScatters}}</td>
-</tr>
-{{end}}
-`
-
-func findScatter(p engine.Primitive) bool {
-	switch v := p.(type) {
-	case *engine.Route:
-		return v.Opcode == engine.SelectScatter
-	default:
-		return false
-	}
-}
-
-func isUpdating(p engine.Primitive) bool {
-	switch p.(type) {
-	case *engine.Update, *engine.Delete, *engine.Insert:
-		return true
-	default:
-		return false
-	}
-}
diff --git a/go/vt/vtgate/executor_stats.go b/go/vt/vtgate/executor_stats.go
index 8bb5a40bef..ffbc8412c1 100644
--- a/go/vt/vtgate/executor_stats.go
+++ b/go/vt/vtgate/executor_stats.go
@@ -33,7 +33,6 @@ const (
   <a href="/debug/querylogz">Current Query Log</a><br>
   <a href="/debug/queryz">Query Plan Stats</a><br>
   <a href="/debug/query_plans">Query Plans</a><br>
-  <a href="/debug/scatter_stats">Scatter Query Statistics</a><br>
 </td>
 </tr>
 </table>
diff --git a/go/vt/vtgate/planbuilder/route_option.go b/go/vt/vtgate/planbuilder/route_option.go
index fa3bab2c6a..1d6ba1c367 100644
--- a/go/vt/vtgate/planbuilder/route_option.go
+++ b/go/vt/vtgate/planbuilder/route_option.go
@@ -87,8 +87,12 @@ func (ro *routeOption) JoinCanMerge(pb *primitiveBuilder, rro *routeOption, ajoi
 }
 
 func (ro *routeOption) MergeJoin(rro *routeOption, isLeftJoin bool) {
-	ro.merge(rro)
 	ro.vschemaTable = nil
+	ro.substitutions = append(ro.substitutions, rro.substitutions...)
+	if isLeftJoin {
+		return
+	}
+	// Add RHS vindexes only if it's not a left join.
 	for c, v := range rro.vindexMap {
 		if ro.vindexMap == nil {
 			ro.vindexMap = make(map[*column]vindexes.SingleColumn)
@@ -97,16 +101,6 @@ func (ro *routeOption) MergeJoin(rro *routeOption, isLeftJoin bool) {
 	}
 }
 
-// merge merges two routeOptions. If the LHS (ro) is a SelectReference,
-// then the RHS option values supersede LHS.
-func (ro *routeOption) merge(rro *routeOption) {
-	if ro.eroute.Opcode == engine.SelectReference {
-		// Swap the values and then merge.
-		*ro, *rro = *rro, *ro
-	}
-	ro.substitutions = append(ro.substitutions, rro.substitutions...)
-}
-
 func (ro *routeOption) SubqueryCanMerge(pb *primitiveBuilder, inner *routeOption) bool {
 	return ro.canMerge(inner, func() bool {
 		switch vals := inner.condition.(type) {
@@ -120,7 +114,7 @@ func (ro *routeOption) SubqueryCanMerge(pb *primitiveBuilder, inner *routeOption
 }
 
 func (ro *routeOption) MergeSubquery(subqueryOption *routeOption) {
-	ro.merge(subqueryOption)
+	ro.substitutions = append(ro.substitutions, subqueryOption.substitutions...)
 }
 
 func (ro *routeOption) UnionCanMerge(rro *routeOption) bool {
@@ -128,8 +122,8 @@ func (ro *routeOption) UnionCanMerge(rro *routeOption) bool {
 }
 
 func (ro *routeOption) MergeUnion(rro *routeOption) {
-	ro.merge(rro)
 	ro.vschemaTable = nil
+	ro.substitutions = append(ro.substitutions, rro.substitutions...)
 }
 
 func (ro *routeOption) SubqueryToTable(rb *route, vindexMap map[*column]vindexes.SingleColumn) {
@@ -155,7 +149,9 @@ func (ro *routeOption) canMerge(rro *routeOption, customCheck func() bool) bool
 			return true
 		}
 	case engine.SelectReference:
-		return true
+		// TODO(sougou): this can be changed to true, but we'll have
+		// to merge against rro insteal of ro.
+		return false
 	case engine.SelectNext:
 		return false
 	}
diff --git a/misc/git/hooks/golangci-lint b/misc/git/hooks/golangci-lint
deleted file mode 100755
index ea7da67e6b..0000000000
--- a/misc/git/hooks/golangci-lint
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-# Copyright 2019 The Vitess Authors.
-# 
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-# 
-#     http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Unfortunately golangci-lint does not work well on checking just modified files.
-# We will enable it for everything here, but with most of the linters disabled.
-# See: https://github.com/vitessio/vitess/issues/5503
-
-GOLANGCI_LINT=$(command -v golangci-lint >/dev/null 2>&1)
-if [ $? -eq 1 ]; then
-  echo "Downloading golangci-lint..."
-  curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.21.0
-fi
-
-golangci-lint run --disable=ineffassign,unused,gosimple,staticcheck,errcheck,structcheck,varcheck,deadcode
diff --git a/misc/git/hooks/pylint b/misc/git/hooks/pylint
index 35f04cee37..4452647f6f 100755
--- a/misc/git/hooks/pylint
+++ b/misc/git/hooks/pylint
@@ -26,7 +26,7 @@ function msg() {
 }
 
 PYLINT=${PYLINT:-/usr/bin/gpylint}
-pylint_script=$VTROOT/tools/pylint.sh
+pylint_script=$VTTOP/tools/pylint.sh
 
 # This script does not handle file names that contain spaces.
 pyfiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.*\.py$' | grep -v '^py/vtproto/')
@@ -70,7 +70,7 @@ if [[ $? -eq 0 ]]; then
   do
     echo
     msg "Press enter to show the warnings for $pyfile:"
-    read -p "  \$VTROOT/tools/pylint.sh $pyfile"
+    read -p "  \$VTTOP/tools/pylint.sh $pyfile"
     $pylint_script $pyfile
   done
   read -r -p \
diff --git a/tools/bootstrap_web.sh b/tools/bootstrap_web.sh
index cc0e40966c..aecc89b8f8 100755
--- a/tools/bootstrap_web.sh
+++ b/tools/bootstrap_web.sh
@@ -39,12 +39,12 @@ else
   # Add the node directory to PATH to make sure that the Angular
   # installation below can find the "node" binary.
   # (dev.env does actually append it to PATH.)
-  source $VTROOT/dev.env
+  source $VTTOP/dev.env
 fi
 
 echo "Installing dependencies for building web UI"
 angular_cli_dir=$VTROOT/dist/angular-cli
-web_dir2=$VTROOT/web/vtctld2
+web_dir2=$VTTOP/web/vtctld2
 angular_cli_commit=cacaa4eff10e135016ef81076fab1086a3bce92f
 if [[ -d $angular_cli_dir && `cd $angular_cli_dir && git rev-parse HEAD` == "$angular_cli_commit" ]]; then
   echo "skipping angular cli download. remove $angular_cli_dir to force download."
diff --git a/tools/check_make_parser.sh b/tools/check_make_parser.sh
index ef59eb67a9..d28d4f18f0 100755
--- a/tools/check_make_parser.sh
+++ b/tools/check_make_parser.sh
@@ -6,8 +6,6 @@
 # This is used in Travis to verify that the currently committed version was
 # generated with the proper version of goyacc.
 
-source build.env
-
 CUR="sql.go"
 TMP="/tmp/sql.$$.go"
 
diff --git a/tools/dependency_check.sh b/tools/dependency_check.sh
deleted file mode 100755
index 7d5179c161..0000000000
--- a/tools/dependency_check.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Vitess Authors.
-# 
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-# 
-#     http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source build.env
-
-function fail() {
-  echo "ERROR: $1"
-  exit 1
-}
-
-# These binaries are required to 'make test'
-# mysqld might be in /usr/sbin which will not be in the default PATH
-PATH="/usr/sbin:$PATH"
-for binary in mysqld consul etcd etcdctl zksrv.sh javadoc mvn ant curl wget zip unzip; do
-  command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/contributing/build-from-source for install instructions."
-done;
diff --git a/tools/generate_web_artifacts.sh b/tools/generate_web_artifacts.sh
index a7080db782..d21da7e84e 100755
--- a/tools/generate_web_artifacts.sh
+++ b/tools/generate_web_artifacts.sh
@@ -20,7 +20,7 @@
 
 set -e
 
-vtctld2_dir=$VTROOT/web/vtctld2
+vtctld2_dir=$VTTOP/web/vtctld2
 if [[ -d $vtctld2_dir/app ]]; then
   rm -rf $vtctld2_dir/app
 fi
diff --git a/tools/make-release-packages.sh b/tools/make-release-packages.sh
deleted file mode 100755
index 6ec36c5e85..0000000000
--- a/tools/make-release-packages.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-
-# This script builds and packages a Vitess release suitable for creating a new
-# release on https://github.com/vitessio/vitess/releases.
-
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
-
-# sudo gem install --no-ri --no-rdoc fpm
-
-source build.env
-
-SHORT_REV="$(git rev-parse --short HEAD)"
-VERSION="5.0.0" # tmp
-# TODO: We can discover version from the last tag, we just need to have this setup.
-# TAG_VERSION="$(git describe --tags --dirty --always | sed  s/v//)"
-
-RELEASE_ID="vitess-${VERSION}-${SHORT_REV}"
-RELEASE_DIR="${VTROOT}/releases/${RELEASE_ID}"
-DESCRIPTION="A database clustering system for horizontal scaling of MySQL
-
-Vitess is a database solution for deploying, scaling and managing large
-clusters of MySQL instances. It's architected to run as effectively in a public
-or private cloud architecture as it does on dedicated hardware. It combines and
-extends many important MySQL features with the scalability of a NoSQL database."
-
-DEB_FILE="vitess_${VERSION}-${SHORT_REV}_amd64.deb"
-RPM_FILE="vitess-${VERSION}-${SHORT_REV}.x86_64.rpm"
-TAR_FILE="${RELEASE_ID}.tar.gz"
-
-make tools
-make build
-
-mkdir -p releases
-
-# Copy a subset of binaries from issue #5421
-mkdir -p "${RELEASE_DIR}/bin"
-for binary in vttestserver mysqlctl mysqlctld query_analyzer topo2topo vtaclcheck vtbackup vtbench vtclient vtcombo vtctl vtctlclient vtctld vtexplain vtgate vttablet vtworker vtworkerclient zk zkctl zkctld; do 
- cp "bin/$binary" "${RELEASE_DIR}/bin/"
-done;
-
-# Copy remaining files, preserving date/permissions
-# But resolving symlinks
-cp -rpfL {config,vthook,web,examples} "${RELEASE_DIR}/"
-
-cd "${RELEASE_DIR}/.."
-tar -czf "${TAR_FILE}" "${RELEASE_ID}"
-
-fpm \
-   --force \
-   --input-type dir \
-   --name vitess \
-   --version "${VERSION}" \
-   --url "https://vitess.io/" \
-   --description "${DESCRIPTION}" \
-   --license "Apache License - Version 2.0, January 2004" \
-   --prefix "/vt" \
-   --directories "/vt" \
-   --before-install "$VTROOT/tools/preinstall.sh" \
-   -C "${RELEASE_DIR}" \
-   --package "$(dirname "${RELEASE_DIR}")" \
-   --iteration "${SHORT_REV}" \
-   -t deb --deb-no-default-config-files
-
-fpm \
-   --force \
-   --input-type dir \
-   --name vitess \
-   --version "${VERSION}" \
-   --url "https://vitess.io/" \
-   --description "${DESCRIPTION}" \
-   --license "Apache License - Version 2.0, January 2004" \
-   --prefix "/vt" \
-   --directories "/vt" \
-   --before-install "$VTROOT/tools/preinstall.sh" \
-   -C "${RELEASE_DIR}" \
-   --package "$(dirname "${RELEASE_DIR}")" \
-   --iteration "${SHORT_REV}" \
-   -t rpm
-
-echo ""
-echo "Packages created as of $(date +"%m-%d-%y") at $(date +"%r %Z")"
-echo ""
-echo "Package | SHA256"
-echo "------------ | -------------"
-echo "${TAR_FILE} | $(sha256sum "${VTROOT}/releases/${TAR_FILE}" | awk '{print $1}')"
-echo "${DEB_FILE} | $(sha256sum "${VTROOT}/releases/${DEB_FILE}" | awk '{print $1}')"
-echo "${RPM_FILE} | $(sha256sum "${VTROOT}/releases/${RPM_FILE}" | awk '{print $1}')"
diff --git a/tools/preinstall.sh b/tools/preinstall.sh
deleted file mode 100755
index 1d9c80ca78..0000000000
--- a/tools/preinstall.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-if ! /usr/bin/getent group vitess >/dev/null ; then
-   groupadd -r vitess
-fi
-
-if ! /usr/bin/getent passwd vitess >/dev/null ; then
-    useradd -r -g vitess vitess
-fi
diff --git a/tools/shell_functions.inc b/tools/shell_functions.inc
index bd344db97e..00696dee5f 100644
--- a/tools/shell_functions.inc
+++ b/tools/shell_functions.inc
@@ -60,9 +60,3 @@ function prepend_path() {
   # Return path variable unchanged.
   echo "$1"
 }
-
-function fail() {
-  echo "ERROR: $1"
-  exit 1
-}
-


Test patch:
diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml
index 2e6fb4f493..a0404cfefb 100644
--- a/.github/workflows/e2e_race.yml
+++ b/.github/workflows/e2e_race.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
 
   build:
-    name: End-to-End Test (Race)
+    name: Build
     runs-on: ubuntu-latest
     steps:
 
@@ -17,7 +17,6 @@ jobs:
 
     - name: Get dependencies
       run: |
-        sudo apt-get update || echo "update failed"
         sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
         sudo service mysql stop
         sudo service etcd stop
@@ -25,10 +24,18 @@ jobs:
         sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
-    - name: Run make minimaltools
+    - name: Run bootstrap.sh
       run: |
-        make minimaltools
+        echo "Copying new bootstrap over location of legacy one."
+        cp .github/bootstrap.sh .
+        ./bootstrap.sh
+
+    - name: Build
+      run: |
+        VTROOT=$PWD VTTOP=$PWD make build
 
     - name: e2e_race
       run: |
-        make e2e_test_race
+        export PATH=$PWD/bin:$PATH
+        source ./dev.env
+        VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make e2e_test_race
diff --git a/docker/test/run.sh b/docker/test/run.sh
index df0c6a20f8..147663d028 100755
--- a/docker/test/run.sh
+++ b/docker/test/run.sh
@@ -160,32 +160,37 @@ case "$mode" in
   "create_cache") echo "Creating cache image $cache_image ..." ;;
 esac
 
-bashcmd=""
-
+# Construct "cp" command to copy the source code.
+#
+# Copy the full source tree except:
+# - vendor
+# That's because these directories are already part of the image.
+#
+# Note that we're using the Bash extended Glob support "!(vendor)" on
+# purpose here to minimize the size of the cache image: With this trick,
+# we do not move or overwrite the existing files while copying the other
+# directories. Therefore, the existing files do not count as changed and will
+# not be part of the new Docker layer of the cache image.
+copy_src_cmd="cp -R /tmp/src/!(vendor|bootstrap.sh) ."
+# Copy the .git directory because travis/check_make_proto.sh needs a working
+# Git repository.
+copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .")
+
+# Enable gomodules
+run_bootstrap_cmd="export GO111MODULE=on"
+# Copy bootstrap.sh if it changed
+run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi")
+# run bootstrap.sh if necessary
+run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi")
+copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd")
+
+# Construct the command we will actually run.
+#
+# Uncomment the next line if you need to debug "bashcmd".
+#bashcmd="set -x"
 if [[ -z "$existing_cache_image" ]]; then
-
-  # Construct "cp" command to copy the source code.
-  bashcmd=$(append_cmd "$bashcmd" "cp -R /tmp/src/!(vtdataroot|dist|bin|lib|vthook|py-vtdb) . && cp -R /tmp/src/.git .")
-
+  bashcmd=$(append_cmd "$bashcmd" "$copy_src_cmd")
 fi
-
-# Reset the environment if this was an old bootstrap. We can detect this from VTTOP presence.
-bashcmd=$(append_cmd "$bashcmd" "export VTROOT=/vt/src/vitess.io/vitess")
-bashcmd=$(append_cmd "$bashcmd" "export VTDATAROOT=/vt/vtdataroot")
-bashcmd=$(append_cmd "$bashcmd" "export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages")
-
-bashcmd=$(append_cmd "$bashcmd" "mkdir -p dist; mkdir -p bin; mkdir -p lib; mkdir -p vthook")
-bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/dist; ln -s /vt/src/vitess.io/vitess/dist /vt/dist")
-bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/bin; ln -s /vt/src/vitess.io/vitess/bin /vt/bin")
-bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/lib; ln -s /vt/src/vitess.io/vitess/lib /vt/lib")
-bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/vthook; ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook")
-
-# Maven was setup in /vt/dist, may need to reinstall it.
-bashcmd=$(append_cmd "$bashcmd" "echo 'Checking if mvn needs installing...'; if [[ ! \$(command -v mvn) ]]; then echo 'install maven'; curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 /vt/dist/maven; fi; echo 'mvn check done'")
-
-# Run bootstrap every time now
-bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh")
-
 # At last, append the user's command.
 bashcmd=$(append_cmd "$bashcmd" "$cmd")
 
diff --git a/go/mysql/encoding_test.go b/go/mysql/encoding_test.go
index ac38b5c7db..2ca9a84c93 100644
--- a/go/mysql/encoding_test.go
+++ b/go/mysql/encoding_test.go
@@ -303,24 +303,6 @@ func TestEncString(t *testing.T) {
 			t.Errorf("readLenEncStringAsBytes returned ok=true for empty value %v", test.value)
 		}
 
-		// Check successful decoding as bytes.
-		gotbcopy, posCopy, ok := readLenEncStringAsBytesCopy(test.lenEncoded, 0)
-		if !ok || string(gotb) != test.value || pos != len(test.lenEncoded) {
-			t.Errorf("readLenEncString returned %v/%v/%v but expected %v/%v/%v", gotbcopy, posCopy, ok, test.value, len(test.lenEncoded), true)
-		}
-
-		// Check failed decoding as bytes with shorter data.
-		_, _, ok = readLenEncStringAsBytesCopy(test.lenEncoded[:len(test.lenEncoded)-1], 0)
-		if ok {
-			t.Errorf("readLenEncStringAsBytes returned ok=true for shorter value %v", test.value)
-		}
-
-		// Check failed decoding as bytes with no data.
-		_, _, ok = readLenEncStringAsBytesCopy([]byte{}, 0)
-		if ok {
-			t.Errorf("readLenEncStringAsBytes returned ok=true for empty value %v", test.value)
-		}
-
 		// null encoded tests.
 
 		// Check successful encoding.
diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go
index 5315adbd9a..b3395f0670 100644
--- a/go/test/endtoend/cluster/cluster_process.go
+++ b/go/test/endtoend/cluster/cluster_process.go
@@ -55,9 +55,10 @@ type LocalProcessCluster struct {
 	VtctlProcess       VtctlProcess
 
 	// background executable processes
-	TopoProcess   EtcdProcess
-	VtctldProcess VtctldProcess
-	VtgateProcess VtgateProcess
+	TopoProcess     EtcdProcess
+	VtctldProcess   VtctldProcess
+	VtgateProcess   VtgateProcess
+	VtworkerProcess VtworkerProcess
 
 	nextPortForProcess int
 
@@ -86,6 +87,31 @@ type Shard struct {
 	Vttablets []Vttablet
 }
 
+// MasterTablet get the 1st tablet which is master
+func (shard *Shard) MasterTablet() *Vttablet {
+	return &shard.Vttablets[0]
+}
+
+// Rdonly gets first rdonly tablet
+func (shard *Shard) Rdonly() *Vttablet {
+	for idx, tablet := range shard.Vttablets {
+		if tablet.Type == "rdonly" {
+			return &shard.Vttablets[idx]
+		}
+	}
+	return nil
+}
+
+// Replica gets the first replica tablet
+func (shard *Shard) Replica() *Vttablet {
+	for idx, tablet := range shard.Vttablets {
+		if tablet.Type == "replica" && idx > 0 {
+			return &shard.Vttablets[idx]
+		}
+	}
+	return nil
+}
+
 // Vttablet stores the properties needed to start a vttablet process
 type Vttablet struct {
 	Type      string
@@ -210,7 +236,6 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames
 				cluster.VtTabletExtraArgs,
 				cluster.EnableSemiSync)
 			tablet.Alias = tablet.VttabletProcess.TabletPath
-
 			shard.Vttablets = append(shard.Vttablets, *tablet)
 		}
 
@@ -276,11 +301,19 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames
 
 // StartVtgate starts vtgate
 func (cluster *LocalProcessCluster) StartVtgate() (err error) {
+	vtgateInstance := *cluster.GetVtgateInstance()
+	cluster.VtgateProcess = vtgateInstance
+	log.Info(fmt.Sprintf("Starting vtgate on port %d", vtgateInstance.Port))
+	log.Info(fmt.Sprintf("Vtgate started, connect to mysql using : mysql -h 127.0.0.1 -P %d", cluster.VtgateMySQLPort))
+	return cluster.VtgateProcess.Setup()
+}
+
+// GetVtgateInstance returns an instance of vtgateprocess
+func (cluster *LocalProcessCluster) GetVtgateInstance() *VtgateProcess {
 	vtgateHTTPPort := cluster.GetAndReservePort()
 	vtgateGrpcPort := cluster.GetAndReservePort()
 	cluster.VtgateMySQLPort = cluster.GetAndReservePort()
-	log.Info(fmt.Sprintf("Starting vtgate on port %d", vtgateHTTPPort))
-	cluster.VtgateProcess = *VtgateProcessInstance(
+	vtgateProcInstance := VtgateProcessInstance(
 		vtgateHTTPPort,
 		vtgateGrpcPort,
 		cluster.VtgateMySQLPort,
@@ -291,15 +324,7 @@ func (cluster *LocalProcessCluster) StartVtgate() (err error) {
 		cluster.TopoProcess.Port,
 		cluster.TmpDirectory,
 		cluster.VtGateExtraArgs)
-
-	log.Info(fmt.Sprintf("Vtgate started, connect to mysql using : mysql -h 127.0.0.1 -P %d", cluster.VtgateMySQLPort))
-	if err = cluster.VtgateProcess.Setup(); err != nil {
-		return err
-	}
-	if err = cluster.WaitForTabletsToHealthyInVtgate(); err != nil {
-		return err
-	}
-	return nil
+	return vtgateProcInstance
 }
 
 // NewCluster instantiates a new cluster
@@ -361,16 +386,18 @@ func (cluster *LocalProcessCluster) Teardown() {
 	if err := cluster.VtgateProcess.TearDown(); err != nil {
 		log.Errorf("Error in vtgate teardown - %s", err.Error())
 	}
-	mysqlctlProcessList := []*exec.Cmd{}
+
+	var mysqlctlProcessList []*exec.Cmd
 	for _, keyspace := range cluster.Keyspaces {
 		for _, shard := range keyspace.Shards {
 			for _, tablet := range shard.Vttablets {
-				if proc, err := tablet.MysqlctlProcess.StopProcess(); err != nil {
-					log.Errorf("Error in mysqlctl teardown - %s", err.Error())
-				} else {
-					mysqlctlProcessList = append(mysqlctlProcessList, proc)
+				if tablet.MysqlctlProcess.TabletUID > 0 {
+					if proc, err := tablet.MysqlctlProcess.StopProcess(); err != nil {
+						log.Errorf("Error in mysqlctl teardown - %s", err.Error())
+					} else {
+						mysqlctlProcessList = append(mysqlctlProcessList, proc)
+					}
 				}
-
 				if err := tablet.VttabletProcess.TearDown(); err != nil {
 					log.Errorf("Error in vttablet teardown - %s", err.Error())
 				}
@@ -393,6 +420,22 @@ func (cluster *LocalProcessCluster) Teardown() {
 	}
 }
 
+// StartVtworker starts a vtworker
+func (cluster *LocalProcessCluster) StartVtworker(cell string, extraArgs ...string) error {
+	httpPort := cluster.GetAndReservePort()
+	grpcPort := cluster.GetAndReservePort()
+	log.Info(fmt.Sprintf("Starting vtworker on port %d", httpPort))
+	cluster.VtworkerProcess = *VtworkerProcessInstance(
+		httpPort,
+		grpcPort,
+		cluster.TopoPort,
+		cluster.Hostname,
+		cluster.TmpDirectory)
+	cluster.VtworkerProcess.ExtraArgs = extraArgs
+	return cluster.VtworkerProcess.Setup(cell)
+
+}
+
 // GetAndReservePort gives port for required process
 func (cluster *LocalProcessCluster) GetAndReservePort() int {
 	if cluster.nextPortForProcess == 0 {
@@ -415,7 +458,7 @@ func getRandomNumber(maxNumber int32, baseNumber int) int {
 	return int(rand.Int31n(maxNumber)) + baseNumber
 }
 
-// GetVttabletInstance create a new vttablet object
+// GetVttabletInstance creates a new vttablet object
 func (cluster *LocalProcessCluster) GetVttabletInstance(UID int) *Vttablet {
 	if UID == 0 {
 		UID = cluster.GetAndReserveTabletUID()
@@ -430,7 +473,7 @@ func (cluster *LocalProcessCluster) GetVttabletInstance(UID int) *Vttablet {
 	}
 }
 
-// StartVttablet start a new tablet
+// StartVttablet starts a new tablet
 func (cluster *LocalProcessCluster) StartVttablet(tablet *Vttablet, servingStatus string,
 	supportBackup bool, cell string, keyspaceName string, hostname string, shardName string) error {
 	tablet.VttabletProcess = VttabletProcessInstance(
@@ -448,7 +491,7 @@ func (cluster *LocalProcessCluster) StartVttablet(tablet *Vttablet, servingStatu
 		cluster.VtTabletExtraArgs,
 		cluster.EnableSemiSync)
 
-	tablet.VttabletProcess.SupportBackup = supportBackup
+	tablet.VttabletProcess.SupportsBackup = supportBackup
 	tablet.VttabletProcess.ServingStatus = servingStatus
 	return tablet.VttabletProcess.Setup()
-}
\ No newline at end of file
+}
diff --git a/go/test/endtoend/cluster/mysqlctl_process.go b/go/test/endtoend/cluster/mysqlctl_process.go
index 14a5a7fe0f..7c69261c49 100644
--- a/go/test/endtoend/cluster/mysqlctl_process.go
+++ b/go/test/endtoend/cluster/mysqlctl_process.go
@@ -37,6 +37,7 @@ type MysqlctlProcess struct {
 	TabletUID    int
 	MySQLPort    int
 	InitDBFile   string
+	ExtraArgs    []string
 }
 
 // InitDb executes mysqlctl command to add cell info
@@ -68,9 +69,14 @@ func (mysqlctl *MysqlctlProcess) StartProcess() (*exec.Cmd, error) {
 		"-log_dir", mysqlctl.LogDirectory,
 		"-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID),
 		"-mysql_port", fmt.Sprintf("%d", mysqlctl.MySQLPort),
-		"init",
-		"-init_db_sql_file", mysqlctl.InitDBFile,
 	)
+
+	if len(mysqlctl.ExtraArgs) > 0 {
+		tmpProcess.Args = append(tmpProcess.Args, mysqlctl.ExtraArgs...)
+	}
+	tmpProcess.Args = append(tmpProcess.Args, "init",
+		"-init_db_sql_file", mysqlctl.InitDBFile)
+
 	return tmpProcess, tmpProcess.Start()
 }
 
@@ -88,8 +94,13 @@ func (mysqlctl *MysqlctlProcess) StopProcess() (*exec.Cmd, error) {
 	tmpProcess := exec.Command(
 		mysqlctl.Binary,
 		"-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID),
-		"shutdown",
 	)
+
+	if len(mysqlctl.ExtraArgs) > 0 {
+		tmpProcess.Args = append(tmpProcess.Args, mysqlctl.ExtraArgs...)
+	}
+	tmpProcess.Args = append(tmpProcess.Args, "shutdown")
+
 	return tmpProcess, tmpProcess.Start()
 }
 
@@ -116,7 +127,7 @@ func MysqlCtlProcessInstance(tabletUID int, mySQLPort int, tmpDirectory string)
 	return mysqlctl
 }
 
-// StartMySQL process
+// StartMySQL starts mysqlctl process
 func StartMySQL(ctx context.Context, tablet *Vttablet, username string, tmpDirectory string) error {
 	tablet.MysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, tmpDirectory)
 	err := tablet.MysqlctlProcess.Start()
@@ -148,7 +159,6 @@ func (mysqlctl *MysqlctlProcess) ExecuteCommandWithOutput(args ...string) (resul
 		mysqlctl.Binary,
 		args...,
 	)
-	println(fmt.Sprintf("Executing mysqlctl with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	log.Info(fmt.Sprintf("Executing mysqlctl with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	resultByte, err := tmpProcess.CombinedOutput()
 	return string(resultByte), err
diff --git a/go/test/endtoend/cluster/vtctl_process.go b/go/test/endtoend/cluster/vtctl_process.go
index 74511b5c70..b948a9ec8c 100644
--- a/go/test/endtoend/cluster/vtctl_process.go
+++ b/go/test/endtoend/cluster/vtctl_process.go
@@ -63,6 +63,22 @@ func (vtctl *VtctlProcess) CreateKeyspace(keyspace string) (err error) {
 	return tmpProcess.Run()
 }
 
+// ExecuteCommandWithOutput executes any vtctlclient command and returns output
+func (vtctl *VtctlProcess) ExecuteCommandWithOutput(args ...string) (result string, err error) {
+	args = append([]string{
+		"-enable_queries",
+		"-topo_implementation", vtctl.TopoImplementation,
+		"-topo_global_server_address", vtctl.TopoGlobalAddress,
+		"-topo_global_root", vtctl.TopoGlobalRoot}, args...)
+	tmpProcess := exec.Command(
+		vtctl.Binary,
+		args...,
+	)
+	log.Info(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " ")))
+	resultByte, err := tmpProcess.CombinedOutput()
+	return string(resultByte), err
+}
+
 // VtctlProcessInstance returns a VtctlProcess handle for vtctl process
 // configured with the given Config.
 // The process must be manually started by calling setup()
diff --git a/go/test/endtoend/cluster/vtctlclient_process.go b/go/test/endtoend/cluster/vtctlclient_process.go
index f7287a80f3..8a37a6e633 100644
--- a/go/test/endtoend/cluster/vtctlclient_process.go
+++ b/go/test/endtoend/cluster/vtctlclient_process.go
@@ -99,9 +99,18 @@ func VtctlClientProcessInstance(hostname string, grpcPort int, tmpDirectory stri
 
 // InitTablet initializes a tablet
 func (vtctlclient *VtctlClientProcess) InitTablet(tablet *Vttablet, cell string, keyspaceName string, hostname string, shardName string) error {
-	return vtctlclient.ExecuteCommand(
-		"InitTablet", "-hostname", hostname,
-		"-port", fmt.Sprintf("%d", tablet.HTTPPort), "-allow_update",
-		"-keyspace", keyspaceName, "-shard", shardName,
-		fmt.Sprintf("%s-%010d", cell, tablet.TabletUID), "replica")
+
+	tabletType := "replica"
+	if tablet.Type == "rdonly" {
+		tabletType = "rdonly"
+	}
+	args := []string{"InitTablet", "-hostname", hostname,
+		"-port", fmt.Sprintf("%d", tablet.HTTPPort), "-allow_update", "-parent",
+		"-keyspace", keyspaceName,
+		"-shard", shardName}
+	if tablet.MySQLPort > 0 {
+		args = append(args, "-mysql_port", fmt.Sprintf("%d", tablet.MySQLPort))
+	}
+	args = append(args, fmt.Sprintf("%s-%010d", cell, tablet.TabletUID), tabletType)
+	return vtctlclient.ExecuteCommand(args...)
 }
diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go
index f5b3655f88..a185fbb32b 100644
--- a/go/test/endtoend/cluster/vtgate_process.go
+++ b/go/test/endtoend/cluster/vtgate_process.go
@@ -95,10 +95,11 @@ func (vtgate *VtgateProcess) Setup() (err error) {
 	if err != nil {
 		return
 	}
-
 	vtgate.exit = make(chan error)
 	go func() {
-		vtgate.exit <- vtgate.proc.Wait()
+		if vtgate.proc != nil {
+			vtgate.exit <- vtgate.proc.Wait()
+		}
 	}()
 
 	timeout := time.Now().Add(60 * time.Second)
@@ -205,7 +206,7 @@ func VtgateProcessInstance(port int, grpcPort int, mySQLServerPort int, cell str
 		Binary:                "vtgate",
 		FileToLogQueries:      path.Join(tmpDirectory, "/vtgate_querylog.txt"),
 		Directory:             os.Getenv("VTDATAROOT"),
-		ServiceMap:            "grpc-vtgateservice",
+		ServiceMap:            "grpc-tabletmanager,grpc-throttler,grpc-queryservice,grpc-updatestream,grpc-vtctl,grpc-vtworker,grpc-vtgateservice",
 		LogDir:                tmpDirectory,
 		Port:                  port,
 		GrpcPort:              grpcPort,
diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go
index 292ed85a9a..5c42e22ac2 100644
--- a/go/test/endtoend/cluster/vttablet_process.go
+++ b/go/test/endtoend/cluster/vttablet_process.go
@@ -63,8 +63,10 @@ type VttabletProcess struct {
 	Directory                   string
 	VerifyURL                   string
 	EnableSemiSync              bool
-	SupportBackup               bool
+	SupportsBackup              bool
 	ServingStatus               string
+	DbPassword                  string
+	DbPort                      int
 	//Extra Args to be set before starting the vttablet process
 	ExtraArgs []string
 
@@ -98,7 +100,7 @@ func (vttablet *VttabletProcess) Setup() (err error) {
 		"-vtctld_addr", vttablet.VtctldAddress,
 	)
 
-	if vttablet.SupportBackup {
+  if vttablet.SupportsBackup {
 		vttablet.proc.Args = append(vttablet.proc.Args, "-restore_from_backup")
 	}
 	if vttablet.EnableSemiSync {
@@ -121,54 +123,110 @@ func (vttablet *VttabletProcess) Setup() (err error) {
 
 	vttablet.exit = make(chan error)
 	go func() {
-		vttablet.exit <- vttablet.proc.Wait()
+		if vttablet.proc != nil {
+			vttablet.exit <- vttablet.proc.Wait()
+		}
 	}()
 
-	timeout := time.Now().Add(60 * time.Second)
-	for time.Now().Before(timeout) {
-		if vttablet.WaitForStatus(vttablet.ServingStatus) {
-			return nil
-		}
-		select {
-		case err := <-vttablet.exit:
-			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
-		default:
-			time.Sleep(300 * time.Millisecond)
+
+	if vttablet.ServingStatus != "" {
+		if err = vttablet.WaitForTabletType(vttablet.ServingStatus); err != nil {
+			return fmt.Errorf("process '%s' timed out after 60s (err: %s)", vttablet.Name, <-vttablet.exit)
 		}
 	}
-
-	return fmt.Errorf("process '%s' timed out after 60s (err: %s)", vttablet.Name, <-vttablet.exit)
+	return nil
 }
 
-// WaitForStatus function checks if vttablet process is up and running
-func (vttablet *VttabletProcess) WaitForStatus(status string) bool {
-	return vttablet.GetTabletStatus() == status
+// GetStatus returns /debug/status endpoint result
+func (vttablet *VttabletProcess) GetStatus() string {
+	URL := fmt.Sprintf("http://%s:%d/debug/status", vttablet.TabletHostname, vttablet.Port)
+	resp, err := http.Get(URL)
+	if err != nil {
+		return ""
+	}
+	if resp.StatusCode == 200 {
+		respByte, _ := ioutil.ReadAll(resp.Body)
+		defer resp.Body.Close()
+		return string(respByte)
+	}
+	return ""
 }
 
-// GetTabletStatus function checks if vttablet process is up and running
-func (vttablet *VttabletProcess) GetTabletStatus() string {
+
+// GetVars gets the debug vars as map
+func (vttablet *VttabletProcess) GetVars() map[string]interface{} {
 	resp, err := http.Get(vttablet.VerifyURL)
 	if err != nil {
-		return ""
+		return nil
 	}
 	if resp.StatusCode == 200 {
 		resultMap := make(map[string]interface{})
 		respByte, _ := ioutil.ReadAll(resp.Body)
 		err := json.Unmarshal(respByte, &resultMap)
 		if err != nil {
-			panic(err)
+			return nil
 		}
-		status := reflect.ValueOf(resultMap["TabletStateName"]).String()
-		return status
+		return resultMap
+	}
+	return nil
+}
+
+// WaitForStatus waits till desired status of tablet is reached
+func (vttablet *VttabletProcess) WaitForStatus(status string) bool {
+	return vttablet.GetTabletStatus() == status
+}
+
+// GetTabletStatus returns the tablet state as seen in /debug/vars TabletStateName
+func (vttablet *VttabletProcess) GetTabletStatus() string {
+	resultMap := vttablet.GetVars()
+	if resultMap != nil {
+		return reflect.ValueOf(resultMap["TabletStateName"]).String()
 	}
 	return ""
 }
 
+// WaitForTabletType waits till the tablet status reaches desired type
+func (vttablet *VttabletProcess) WaitForTabletType(expectedType string) error {
+	timeout := time.Now().Add(10 * time.Second)
+	for time.Now().Before(timeout) {
+		if vttablet.GetTabletStatus() == expectedType {
+			return nil
+		}
+		select {
+		case err := <-vttablet.exit:
+			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
+		default:
+			time.Sleep(300 * time.Millisecond)
+		}
+	}
+	return fmt.Errorf("Vttablet %s, expected status not reached", vttablet.TabletPath)
+}
+
+// WaitForBinLogPlayerCount waits till binlog player count var matches
+func (vttablet *VttabletProcess) WaitForBinLogPlayerCount(expectedCount int) error {
+	timeout := time.Now().Add(10 * time.Second)
+	for time.Now().Before(timeout) {
+		if vttablet.getVReplStreamCount() == fmt.Sprintf("%d", expectedCount) {
+			return nil
+		}
+		select {
+		case err := <-vttablet.exit:
+			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
+		default:
+			time.Sleep(300 * time.Millisecond)
+		}
+	}
+	return fmt.Errorf("vttablet %s, expected status not reached", vttablet.TabletPath)
+}
+
+func (vttablet *VttabletProcess) getVReplStreamCount() string {
+	resultMap := vttablet.GetVars()
+	object := reflect.ValueOf(resultMap["VReplicationStreamCount"])
+	return fmt.Sprintf("%v", object)
+}
+
 // TearDown shuts down the running vttablet service
 func (vttablet *VttabletProcess) TearDown() error {
-	if vttablet.proc == nil {
-		fmt.Printf("No process found for vttablet %d", vttablet.TabletUID)
-	}
 	if vttablet.proc == nil || vttablet.exit == nil {
 		return nil
 	}
@@ -187,15 +245,30 @@ func (vttablet *VttabletProcess) TearDown() error {
 	}
 }
 
-// QueryTablet lets you execute query in this tablet and get the result
+
+// CreateDB creates the database for keyspace
+func (vttablet *VttabletProcess) CreateDB(keyspace string) error {
+	_, err := vttablet.QueryTablet(fmt.Sprintf("create database vt_%s", keyspace), keyspace, false)
+	return err
+}
+
+// QueryTablet lets you execute a query in this tablet and get the result
 func (vttablet *VttabletProcess) QueryTablet(query string, keyspace string, useDb bool) (*sqltypes.Result, error) {
 	dbParams := mysql.ConnParams{
-		Uname:      "vt_dba",
-		UnixSocket: path.Join(vttablet.Directory, "mysql.sock"),
+		Uname: "vt_dba",
+	}
+	if vttablet.DbPort > 0 {
+		dbParams.Port = vttablet.DbPort
+	} else {
+		dbParams.UnixSocket = path.Join(vttablet.Directory, "mysql.sock")
 	}
 	if useDb {
 		dbParams.DbName = "vt_" + keyspace
 	}
+	if vttablet.DbPassword != "" {
+		dbParams.Pass = vttablet.DbPassword
+	}
+
 	ctx := context.Background()
 	dbConn, err := mysql.Connect(ctx, &dbParams)
 	if err != nil {
@@ -232,7 +305,7 @@ func VttabletProcessInstance(port int, grpcPort int, tabletUID int, cell string,
 		VtctldAddress:               fmt.Sprintf("http://%s:%d", hostname, vtctldPort),
 		ExtraArgs:                   extraArgs,
 		EnableSemiSync:              enableSemiSync,
-		SupportBackup:               true,
+		SupportsBackup:              true,
 		ServingStatus:               "NOT_SERVING",
 	}
 
diff --git a/go/test/endtoend/cluster/vtworker_process.go b/go/test/endtoend/cluster/vtworker_process.go
new file mode 100644
index 0000000000..46ab0fb715
--- /dev/null
+++ b/go/test/endtoend/cluster/vtworker_process.go
@@ -0,0 +1,193 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+
+	"vitess.io/vitess/go/vt/log"
+)
+
+// VtworkerProcess is a generic handle for a running vtworker .
+// It can be spawned manually
+type VtworkerProcess struct {
+	Name                   string
+	Binary                 string
+	CommonArg              VtctlProcess
+	ServiceMap             string
+	LogDir                 string
+	Port                   int
+	GrpcPort               int
+	VerifyURL              string
+	Directory              string
+	ExecuteRetryTime       string
+	Cell                   string
+	Server                 string
+	CommandDisplayInterval string
+	ExtraArgs              []string
+
+	proc *exec.Cmd
+	exit chan error
+}
+
+// Setup starts vtworker process with required arguements
+func (vtworker *VtworkerProcess) Setup(cell string) (err error) {
+
+	vtworker.proc = exec.Command(
+		vtworker.Binary,
+		"-log_dir", vtworker.LogDir,
+		"-port", fmt.Sprintf("%d", vtworker.Port),
+		"-executefetch_retry_time", vtworker.ExecuteRetryTime,
+		"-tablet_manager_protocol", "grpc",
+		"-tablet_protocol", "grpc",
+		"-topo_implementation", vtworker.CommonArg.TopoImplementation,
+		"-topo_global_server_address", vtworker.CommonArg.TopoGlobalAddress,
+		"-topo_global_root", vtworker.CommonArg.TopoGlobalRoot,
+		"-service_map", vtworker.ServiceMap,
+		"-grpc_port", fmt.Sprintf("%d", vtworker.GrpcPort),
+		"-cell", cell,
+		"-command_display_interval", "10ms",
+	)
+	vtworker.proc.Args = append(vtworker.proc.Args, vtworker.ExtraArgs...)
+
+	vtworker.proc.Stderr = os.Stderr
+	vtworker.proc.Stdout = os.Stdout
+
+	vtworker.proc.Env = append(vtworker.proc.Env, os.Environ()...)
+
+	log.Infof("%v", strings.Join(vtworker.proc.Args, " "))
+
+	err = vtworker.proc.Start()
+	if err != nil {
+		return
+	}
+
+	vtworker.exit = make(chan error)
+	go func() {
+		vtworker.exit <- vtworker.proc.Wait()
+	}()
+
+	timeout := time.Now().Add(60 * time.Second)
+	for time.Now().Before(timeout) {
+		if vtworker.IsHealthy() {
+			return nil
+		}
+		select {
+		case err := <-vtworker.exit:
+			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vtworker.Name, err)
+		default:
+			time.Sleep(300 * time.Millisecond)
+		}
+	}
+
+	return fmt.Errorf("process '%s' timed out after 60s (err: %s)", vtworker.Name, <-vtworker.exit)
+}
+
+// IsHealthy function checks if vtworker process is up and running
+func (vtworker *VtworkerProcess) IsHealthy() bool {
+	resp, err := http.Get(vtworker.VerifyURL)
+	if err != nil {
+		return false
+	}
+	if resp.StatusCode == 200 {
+		return true
+	}
+	return false
+}
+
+// TearDown shutdowns the running vtworker process
+func (vtworker *VtworkerProcess) TearDown() error {
+	if vtworker.proc == nil || vtworker.exit == nil {
+		return nil
+	}
+
+	// Attempt graceful shutdown with SIGTERM first
+	vtworker.proc.Process.Signal(syscall.SIGTERM)
+
+	select {
+	case err := <-vtworker.exit:
+		vtworker.proc = nil
+		return err
+
+	case <-time.After(10 * time.Second):
+		vtworker.proc.Process.Kill()
+		vtworker.proc = nil
+		return <-vtworker.exit
+	}
+}
+
+// ExecuteCommand executes any vtworker command
+func (vtworker *VtworkerProcess) ExecuteCommand(args ...string) (err error) {
+	args = append([]string{"-vtworker_client_protocol", "grpc",
+		"-server", vtworker.Server, "-log_dir", vtworker.LogDir, "-stderrthreshold", "info"}, args...)
+	tmpProcess := exec.Command(
+		"vtworkerclient",
+		args...,
+	)
+	log.Info(fmt.Sprintf("Executing vtworkerclient with arguments %v", strings.Join(tmpProcess.Args, " ")))
+	return tmpProcess.Run()
+}
+
+// ExecuteVtworkerCommand executes any vtworker command
+func (vtworker *VtworkerProcess) ExecuteVtworkerCommand(port int, grpcPort int, args ...string) (err error) {
+	args = append([]string{
+		"-port", fmt.Sprintf("%d", port),
+		"-executefetch_retry_time", vtworker.ExecuteRetryTime,
+		"-tablet_manager_protocol", "grpc",
+		"-tablet_protocol", "grpc",
+		"-topo_implementation", vtworker.CommonArg.TopoImplementation,
+		"-topo_global_server_address", vtworker.CommonArg.TopoGlobalAddress,
+		"-topo_global_root", vtworker.CommonArg.TopoGlobalRoot,
+		"-service_map", vtworker.ServiceMap,
+		"-grpc_port", fmt.Sprintf("%d", grpcPort),
+		"-cell", vtworker.Cell,
+		"-log_dir", vtworker.LogDir, "-stderrthreshold", "1"}, args...)
+	tmpProcess := exec.Command(
+		"vtworker",
+		args...,
+	)
+	log.Info(fmt.Sprintf("Executing vtworker with arguments %v", strings.Join(tmpProcess.Args, " ")))
+	return tmpProcess.Run()
+}
+
+// VtworkerProcessInstance returns a vtworker handle
+// configured with the given Config.
+// The process must be manually started by calling Setup()
+func VtworkerProcessInstance(httpPort int, grpcPort int, topoPort int, hostname string, tmpDirectory string) *VtworkerProcess {
+	vtctl := VtctlProcessInstance(topoPort, hostname)
+	vtworker := &VtworkerProcess{
+		Name:                   "vtworker",
+		Binary:                 "vtworker",
+		CommonArg:              *vtctl,
+		ServiceMap:             "grpc-tabletmanager,grpc-throttler,grpc-queryservice,grpc-updatestream,grpc-vtctl,grpc-vtworker,grpc-vtgateservice",
+		LogDir:                 tmpDirectory,
+		Port:                   httpPort,
+		GrpcPort:               grpcPort,
+		ExecuteRetryTime:       "1s",
+		CommandDisplayInterval: "10ms",
+		Directory:              os.Getenv("VTDATAROOT"),
+		Server:                 fmt.Sprintf("%s:%d", hostname, grpcPort),
+	}
+	vtworker.VerifyURL = fmt.Sprintf("http://%s:%d/debug/vars", hostname, vtworker.Port)
+	return vtworker
+}
diff --git a/go/test/endtoend/keyspace/keyspace_test.go b/go/test/endtoend/keyspace/keyspace_test.go
index dee42ce66a..e8f7fc444a 100644
--- a/go/test/endtoend/keyspace/keyspace_test.go
+++ b/go/test/endtoend/keyspace/keyspace_test.go
@@ -384,4 +384,4 @@ func getSrvKeyspace(t *testing.T, cell string, ksname string) *topodata.SrvKeysp
 	err = json.Unmarshal([]byte(output), &srvKeyspace)
 	assert.Nil(t, err)
 	return &srvKeyspace
-}
\ No newline at end of file
+}
diff --git a/go/test/endtoend/mysqlctl/mysqlctl_test.go b/go/test/endtoend/mysqlctl/mysqlctl_test.go
index e820d120af..1e52e5bc90 100644
--- a/go/test/endtoend/mysqlctl/mysqlctl_test.go
+++ b/go/test/endtoend/mysqlctl/mysqlctl_test.go
@@ -164,4 +164,4 @@ func TestAutoDetect(t *testing.T) {
 
 	//Reset flavor
 	os.Setenv("MYSQL_FLAVOR", sqlFlavor)
-}
\ No newline at end of file
+}
diff --git a/go/test/endtoend/sharded/shared_keyspace_test.go b/go/test/endtoend/sharded/sharded_keyspace_test.go
similarity index 99%
rename from go/test/endtoend/sharded/shared_keyspace_test.go
rename to go/test/endtoend/sharded/sharded_keyspace_test.go
index 60d6cea3f9..aa4f014de7 100644
--- a/go/test/endtoend/sharded/shared_keyspace_test.go
+++ b/go/test/endtoend/sharded/sharded_keyspace_test.go
@@ -194,6 +194,7 @@ func initCluster(shardNames []string, totalTabletsRequired int) {
 		shard := &cluster.Shard{
 			Name: shardName,
 		}
+
 		var mysqlCtlProcessList []*exec.Cmd
 
 		for i := 0; i < totalTabletsRequired; i++ {
@@ -253,10 +254,9 @@ func initCluster(shardNames []string, totalTabletsRequired int) {
 				log.Error(err.Error())
 				return
 			}
-
 		}
 
 		keyspace.Shards = append(keyspace.Shards, *shard)
 	}
 	clusterInstance.Keyspaces = append(clusterInstance.Keyspaces, keyspace)
-}
\ No newline at end of file
+}
diff --git a/go/test/endtoend/sharding/base_sharding.go b/go/test/endtoend/sharding/base_sharding.go
new file mode 100644
index 0000000000..b53884b398
--- /dev/null
+++ b/go/test/endtoend/sharding/base_sharding.go
@@ -0,0 +1,357 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+*/
+
+package sharding
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math"
+	"net/http"
+	"reflect"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"vitess.io/vitess/go/test/endtoend/cluster"
+	querypb "vitess.io/vitess/go/vt/proto/query"
+	"vitess.io/vitess/go/vt/proto/topodata"
+)
+
+var (
+	lotRange1 uint64 = 0xA000000000000000
+	lotRange2 uint64 = 0xE000000000000000
+	// InsertTabletTemplateKsID common insert format to be used for different tests
+	InsertTabletTemplateKsID = `insert into %s (id, msg) values (%d, '%s') /* id:%d */`
+)
+
+// CheckSrvKeyspace verifies the schema with expectedPartition
+func CheckSrvKeyspace(t *testing.T, cell string, ksname string, shardingCol string, colType topodata.KeyspaceIdType, expectedPartition map[topodata.TabletType][]string, ci cluster.LocalProcessCluster) {
+	srvKeyspace := getSrvKeyspace(t, cell, ksname, ci)
+	if shardingCol != "" {
+		assert.Equal(t, srvKeyspace.ShardingColumnName, shardingCol)
+	}
+	if colType != 0 {
+		assert.Equal(t, srvKeyspace.ShardingColumnType, colType)
+	}
+
+	currentPartition := map[topodata.TabletType][]string{}
+
+	for _, partition := range srvKeyspace.Partitions {
+		currentPartition[partition.ServedType] = []string{}
+		for _, shardRef := range partition.ShardReferences {
+			currentPartition[partition.ServedType] = append(currentPartition[partition.ServedType], shardRef.Name)
+		}
+	}
+
+	assert.True(t, reflect.DeepEqual(currentPartition, expectedPartition))
+}
+
+func getSrvKeyspace(t *testing.T, cell string, ksname string, ci cluster.LocalProcessCluster) *topodata.SrvKeyspace {
+	output, err := ci.VtctlclientProcess.ExecuteCommandWithOutput("GetSrvKeyspace", cell, ksname)
+	assert.Nil(t, err)
+	var srvKeyspace topodata.SrvKeyspace
+
+	err = json.Unmarshal([]byte(output), &srvKeyspace)
+	assert.Nil(t, err)
+	return &srvKeyspace
+}
+
+// VerifyReconciliationCounters checks that the reconciliation Counters have the expected values.
+func VerifyReconciliationCounters(t *testing.T, vtworkerURL string, availabilityType string, table string,
+	inserts int, updates int, deletes int, equals int) {
+	resp, err := http.Get(vtworkerURL)
+	assert.Nil(t, err)
+	assert.Equal(t, resp.StatusCode, 200)
+
+	resultMap := make(map[string]interface{})
+	respByte, _ := ioutil.ReadAll(resp.Body)
+	err = json.Unmarshal(respByte, &resultMap)
+	assert.Nil(t, err)
+
+	value := getValueFromJSON(resultMap, "Worker"+availabilityType+"InsertsCounters", table)
+	if inserts == 0 {
+		assert.Equal(t, value, "")
+	} else {
+		assert.Equal(t, value, fmt.Sprintf("%d", inserts))
+	}
+
+	value = getValueFromJSON(resultMap, "Worker"+availabilityType+"UpdatesCounters", table)
+	if updates == 0 {
+		assert.Equal(t, value, "")
+	} else {
+		assert.Equal(t, value, fmt.Sprintf("%d", updates))
+	}
+
+	value = getValueFromJSON(resultMap, "Worker"+availabilityType+"DeletesCounters", table)
+	if deletes == 0 {
+		assert.Equal(t, value, "")
+	} else {
+		assert.Equal(t, value, fmt.Sprintf("%d", deletes))
+	}
+
+	value = getValueFromJSON(resultMap, "Worker"+availabilityType+"EqualRowsCounters", table)
+	if equals == 0 {
+		assert.Equal(t, value, "")
+	} else {
+		assert.Equal(t, value, fmt.Sprintf("%d", equals))
+	}
+}
+
+func getValueFromJSON(jsonMap map[string]interface{}, keyname string, tableName string) string {
+	object := reflect.ValueOf(jsonMap[keyname])
+	if object.Kind() == reflect.Map {
+		for _, key := range object.MapKeys() {
+			if key.String() == tableName {
+				return fmt.Sprintf("%v", object.MapIndex(key))
+			}
+		}
+	}
+	return ""
+}
+
+// CheckValues check value from sql query to table with expected values
+func CheckValues(t *testing.T, vttablet cluster.Vttablet, id uint64, msg string, exists bool, tableName string, ks string, keyType querypb.Type) bool {
+	query := fmt.Sprintf("select id, msg from %s where id = %d", tableName, id)
+	if keyType == querypb.Type_VARBINARY {
+		query = fmt.Sprintf("select id, msg from %s where id = '%d'", tableName, id)
+	}
+
+	result, err := vttablet.VttabletProcess.QueryTablet(query, ks, true)
+	assert.Nil(t, err)
+	isFound := false
+	if exists && len(result.Rows) > 0 {
+		if keyType == querypb.Type_VARBINARY {
+			isFound = assert.Equal(t, fmt.Sprintf("%v", result.Rows), fmt.Sprintf(`[[VARBINARY("%d") VARCHAR("%s")]]`, id, msg))
+		} else {
+			isFound = assert.Equal(t, fmt.Sprintf("%v", result.Rows), fmt.Sprintf(`[[UINT64(%d) VARCHAR("%s")]]`, id, msg))
+		}
+
+	} else {
+		assert.Equal(t, len(result.Rows), 0)
+	}
+	return isFound
+}
+
+// CheckDestinationMaster performs multiple checks on a destination master.
+func CheckDestinationMaster(t *testing.T, vttablet cluster.Vttablet, sourceShards []string, ci cluster.LocalProcessCluster) {
+	_ = vttablet.VttabletProcess.WaitForBinLogPlayerCount(len(sourceShards))
+	CheckBinlogPlayerVars(t, vttablet, sourceShards, 0)
+	checkStreamHealthEqualsBinlogPlayerVars(t, vttablet, len(sourceShards), ci)
+}
+
+// CheckBinlogPlayerVars Checks the binlog player variables are correctly exported.
+func CheckBinlogPlayerVars(t *testing.T, vttablet cluster.Vttablet, sourceShards []string, secondBehindMaster int64) {
+	tabletVars := vttablet.VttabletProcess.GetVars()
+
+	assert.Contains(t, tabletVars, "VReplicationStreamCount")
+	assert.Contains(t, tabletVars, "VReplicationSecondsBehindMasterMax")
+	assert.Contains(t, tabletVars, "VReplicationSecondsBehindMaster")
+	assert.Contains(t, tabletVars, "VReplicationSource")
+	assert.Contains(t, tabletVars, "VReplicationSourceTablet")
+
+	streamCountStr := fmt.Sprintf("%v", reflect.ValueOf(tabletVars["VReplicationStreamCount"]))
+	streamCount, _ := strconv.Atoi(streamCountStr)
+	assert.Equal(t, streamCount, len(sourceShards))
+
+	replicationSourceObj := reflect.ValueOf(tabletVars["VReplicationSource"])
+	replicationSourceValue := []string{}
+
+	assert.Equal(t,
+		fmt.Sprintf("%v", replicationSourceObj.MapKeys()),
+		fmt.Sprintf("%v", reflect.ValueOf(tabletVars["VReplicationSourceTablet"]).MapKeys()))
+
+	for _, key := range replicationSourceObj.MapKeys() {
+		replicationSourceValue = append(replicationSourceValue,
+			fmt.Sprintf("%v", replicationSourceObj.MapIndex(key)))
+	}
+
+	assert.True(t, reflect.DeepEqual(replicationSourceValue, sourceShards))
+
+	if secondBehindMaster != 0 {
+		secondBehindMaserMaxStr := fmt.Sprintf("%v", reflect.ValueOf(tabletVars["VReplicationSecondsBehindMasterMax"]))
+		secondBehindMaserMax, _ := strconv.ParseFloat(secondBehindMaserMaxStr, 64)
+
+		assert.True(t, secondBehindMaserMax < float64(secondBehindMaster))
+
+		replicationSecondBehindMasterObj := reflect.ValueOf(tabletVars["VReplicationSecondsBehindMaster"])
+		for _, key := range replicationSourceObj.MapKeys() {
+			str := fmt.Sprintf("%v", replicationSecondBehindMasterObj.MapIndex(key))
+			flt, _ := strconv.ParseFloat(str, 64)
+			assert.True(t, flt < float64(secondBehindMaster))
+		}
+	}
+}
+
+// checkStreamHealthEqualsBinlogPlayerVars - Checks the variables exported by streaming health check match vars.
+func checkStreamHealthEqualsBinlogPlayerVars(t *testing.T, vttablet cluster.Vttablet, count int, ci cluster.LocalProcessCluster) {
+	tabletVars := vttablet.VttabletProcess.GetVars()
+
+	streamCountStr := fmt.Sprintf("%v", reflect.ValueOf(tabletVars["VReplicationStreamCount"]))
+	streamCount, _ := strconv.Atoi(streamCountStr)
+
+	secondBehindMaserMaxStr := fmt.Sprintf("%v", reflect.ValueOf(tabletVars["VReplicationSecondsBehindMasterMax"]))
+	secondBehindMaserMax, _ := strconv.ParseFloat(secondBehindMaserMaxStr, 64)
+
+	assert.Equal(t, streamCount, count)
+	// Enforce health check because it's not running by default as
+	// tablets may not be started with it, or may not run it in time.
+	_ = ci.VtctlclientProcess.ExecuteCommand("RunHealthCheck", vttablet.Alias)
+	streamHealth, err := ci.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", vttablet.Alias)
+	assert.Nil(t, err)
+
+	var streamHealthResponse querypb.StreamHealthResponse
+	err = json.Unmarshal([]byte(streamHealth), &streamHealthResponse)
+	assert.Nil(t, err, "error should be Nil")
+	assert.Equal(t, streamHealthResponse.Serving, false)
+	assert.NotNil(t, streamHealthResponse.RealtimeStats)
+	assert.Equal(t, streamHealthResponse.RealtimeStats.HealthError, "")
+	assert.NotNil(t, streamHealthResponse.RealtimeStats.BinlogPlayersCount)
+
+	assert.Equal(t, streamCount, int(streamHealthResponse.RealtimeStats.BinlogPlayersCount))
+	assert.Equal(t, secondBehindMaserMax, float64(streamHealthResponse.RealtimeStats.SecondsBehindMasterFilteredReplication))
+}
+
+// CheckBinlogServerVars checks the binlog server variables are correctly exported.
+func CheckBinlogServerVars(t *testing.T, vttablet cluster.Vttablet, minStatement int, minTxn int) {
+	resultMap := vttablet.VttabletProcess.GetVars()
+	assert.Contains(t, resultMap, "UpdateStreamKeyRangeStatements")
+	assert.Contains(t, resultMap, "UpdateStreamKeyRangeTransactions")
+	if minStatement > 0 {
+		value := fmt.Sprintf("%v", reflect.ValueOf(resultMap["UpdateStreamKeyRangeStatements"]))
+		iValue, _ := strconv.Atoi(value)
+		assert.True(t, iValue >= minStatement)
+	}
+
+	if minTxn > 0 {
+		value := fmt.Sprintf("%v", reflect.ValueOf(resultMap["UpdateStreamKeyRangeStatements"]))
+		iValue, _ := strconv.Atoi(value)
+		assert.True(t, iValue >= minTxn)
+	}
+}
+
+// InsertLots inserts multiple values to vttablet
+func InsertLots(t *testing.T, count uint64, vttablet cluster.Vttablet, table string, ks string) {
+	var query1, query2 string
+	var i uint64
+	for i = 0; i < count; i++ {
+		query1 = fmt.Sprintf(InsertTabletTemplateKsID, table, lotRange1+i, fmt.Sprintf("msg-range1-%d", 10000+i), lotRange1+i)
+		query2 = fmt.Sprintf(InsertTabletTemplateKsID, table, lotRange2+i, fmt.Sprintf("msg-range2-%d", 20000+i), lotRange2+i)
+
+		InsertToTablet(t, query1, vttablet, ks, false)
+		InsertToTablet(t, query2, vttablet, ks, false)
+	}
+}
+
+// InsertToTablet inserts a single row to vttablet
+func InsertToTablet(t *testing.T, query string, vttablet cluster.Vttablet, ks string, expectFail bool) {
+	_, _ = vttablet.VttabletProcess.QueryTablet("begin", ks, true)
+	_, err := vttablet.VttabletProcess.QueryTablet(query, ks, true)
+	if expectFail {
+		assert.NotNil(t, err)
+	} else {
+		assert.Nil(t, err)
+	}
+	_, _ = vttablet.VttabletProcess.QueryTablet("commit", ks, true)
+}
+
+// CheckLotsTimeout waits till all values are inserted
+func CheckLotsTimeout(t *testing.T, vttablet cluster.Vttablet, count uint64, table string, ks string, keyType querypb.Type, pctFound int) bool {
+	timeout := time.Now().Add(10 * time.Second)
+	var percentFound float64
+	for time.Now().Before(timeout) {
+		percentFound = checkLots(t, vttablet, count, table, ks, keyType)
+		if int(math.Round(percentFound)) == pctFound {
+			return true
+		}
+		time.Sleep(300 * time.Millisecond)
+	}
+	println(fmt.Sprintf("expected pct %d, got pct %f", pctFound, percentFound))
+	return false
+}
+
+// CheckLotsNotPresent verifies that no rows should be present in vttablet
+func CheckLotsNotPresent(t *testing.T, vttablet cluster.Vttablet, count uint64, table string, ks string, keyType querypb.Type) {
+	var i uint64
+	for i = 0; i < count; i++ {
+		assert.False(t, CheckValues(t, vttablet,
+			lotRange1+i, fmt.Sprintf("msg-range1-%d", 10000+i), true, table, ks, keyType))
+
+		assert.False(t, CheckValues(t, vttablet,
+			lotRange2+i, fmt.Sprintf("msg-range2-%d", 20000+i), true, table, ks, keyType))
+	}
+}
+
+func checkLots(t *testing.T, vttablet cluster.Vttablet, count uint64, table string, ks string, keyType querypb.Type) float64 {
+	var isFound bool
+	var totalFound int
+	var i uint64
+
+	for i = 0; i < count; i++ {
+		isFound = CheckValues(t, vttablet,
+			lotRange1+i, fmt.Sprintf("msg-range1-%d", 10000+i), true, table, ks, keyType)
+		if isFound {
+			totalFound++
+		}
+
+		isFound = CheckValues(t, vttablet,
+			lotRange2+i, fmt.Sprintf("msg-range2-%d", 20000+i), true, table, ks, keyType)
+		if isFound {
+			totalFound++
+		}
+	}
+	println(fmt.Sprintf("Total found %d", totalFound))
+	return float64(float64(totalFound) * 100 / float64(count) / 2)
+}
+
+// CheckRunningBinlogPlayer Checks binlog player is running and showing in status
+func CheckRunningBinlogPlayer(t *testing.T, vttablet cluster.Vttablet, numberOfQueries int, numberOfTxns int) {
+	status := vttablet.VttabletProcess.GetStatus()
+	assert.Contains(t, status, "VReplication state: Open")
+	assert.Contains(t, status, fmt.Sprintf("<td><b>All</b>: %d<br><b>Query</b>: %d<br><b>Transaction</b>: %d<br></td>", numberOfQueries+numberOfTxns, numberOfQueries, numberOfTxns))
+	assert.Contains(t, status, "</html>")
+}
+
+// CheckTabletQueryServices check that the query service is enabled or disabled on the specified tablets.
+func CheckTabletQueryServices(t *testing.T, vttablets []cluster.Vttablet, expectedStatus string, tabletControlEnabled bool, ci cluster.LocalProcessCluster) {
+	for _, tablet := range vttablets {
+		CheckTabletQueryService(t, tablet, expectedStatus, tabletControlEnabled, ci)
+	}
+}
+
+// CheckTabletQueryService check that the query service is enabled or disabled on the tablet
+func CheckTabletQueryService(t *testing.T, vttablet cluster.Vttablet, expectedStatus string, tabletControlEnabled bool, ci cluster.LocalProcessCluster) {
+	tabletStatus := vttablet.VttabletProcess.GetTabletStatus()
+	assert.Equal(t, tabletStatus, expectedStatus)
+
+	queryServiceDisabled := "Query Service disabled: TabletControl.DisableQueryService set"
+	status := vttablet.VttabletProcess.GetStatus()
+	if tabletControlEnabled {
+		assert.Contains(t, status, queryServiceDisabled)
+	} else {
+		assert.NotContains(t, status, queryServiceDisabled)
+	}
+
+	if vttablet.Type == "rdonly" {
+		// Run RunHealthCheck to be sure the tablet doesn't change its serving state.
+		_ = ci.VtctlclientProcess.ExecuteCommand("RunHealthCheck", vttablet.Alias)
+		tabletStatus = vttablet.VttabletProcess.GetTabletStatus()
+		assert.Equal(t, tabletStatus, expectedStatus)
+	}
+}
diff --git a/go/test/endtoend/sharding/initialsharding/bytes/initial_sharding_bytes_test.go b/go/test/endtoend/sharding/initialsharding/bytes/initial_sharding_bytes_test.go
new file mode 100644
index 0000000000..6202fb3f2f
--- /dev/null
+++ b/go/test/endtoend/sharding/initialsharding/bytes/initial_sharding_bytes_test.go
@@ -0,0 +1,36 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+"""Re-runs initial_sharding_test.go with a varbinary keyspace_id."""
+*/
+
+package bytes
+
+import (
+	"testing"
+
+	sharding "vitess.io/vitess/go/test/endtoend/sharding/initialsharding"
+	querypb "vitess.io/vitess/go/vt/proto/query"
+)
+
+func TestInitialShardingBytes(t *testing.T) {
+	code, err := sharding.ClusterWrapper(false)
+	if err != nil {
+		t.Errorf("setup failed with status code %d", code)
+	}
+	sharding.TestInitialSharding(t, &sharding.ClusterInstance.Keyspaces[0], querypb.Type_VARBINARY, false, false)
+	defer sharding.ClusterInstance.Teardown()
+}
diff --git a/go/test/endtoend/sharding/initialsharding/multi/initial_sharding_multi_test.go b/go/test/endtoend/sharding/initialsharding/multi/initial_sharding_multi_test.go
new file mode 100644
index 0000000000..ccc5052fc5
--- /dev/null
+++ b/go/test/endtoend/sharding/initialsharding/multi/initial_sharding_multi_test.go
@@ -0,0 +1,62 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+This test simulates the first time a database has to be split
+in a multi-vttablet-single-mysql environment
+
+We have 2 keyspaces. One keyspace is in managing mode. It's vttablets
+own the MySQL instances and can reparent, start/stop server, start/stop
+replication etc. Other keyspace is in non-managing mode and cannot do
+any of these actions. Only TabletExternallyReparented is allowed, but
+resharding should still work.
+
+For each keyspace:
+- we start with a keyspace with a single shard and a single table
+- we add and populate the sharding key
+- we set the sharding key in the topology
+- we clone into 2 instances
+- we enable filtered replication
+- we move all serving types
+- we remove the source tablets
+- we remove the original shard
+
+*/
+
+package multi
+
+import (
+	"testing"
+
+	sharding "vitess.io/vitess/go/test/endtoend/sharding/initialsharding"
+	querypb "vitess.io/vitess/go/vt/proto/query"
+)
+
+func TestInitialShardingMulti(t *testing.T) {
+	code, err := sharding.ClusterWrapper(true)
+	if err != nil {
+		t.Errorf("setup failed with status code %d", code)
+	}
+	sharding.AssignMysqlPortFromKs1ToKs2()
+	sharding.TestInitialSharding(t, &sharding.ClusterInstance.Keyspaces[0], querypb.Type_UINT64, true, false)
+	println("-----------------------------")
+	println("Done with 1st keyspace test")
+	println("-----------------------------")
+	sharding.TestInitialSharding(t, &sharding.ClusterInstance.Keyspaces[1], querypb.Type_UINT64, true, true)
+	println("----------Done with 2nd keyspace test----------")
+	sharding.KillVtgateInstances()
+	sharding.KillTabletsInKeyspace(&sharding.ClusterInstance.Keyspaces[0])
+	sharding.KillTabletsInKeyspace(&sharding.ClusterInstance.Keyspaces[1])
+	defer sharding.ClusterInstance.Teardown()
+}
diff --git a/go/test/endtoend/sharding/initialsharding/sharding_util.go b/go/test/endtoend/sharding/initialsharding/sharding_util.go
new file mode 100644
index 0000000000..ce3e92ac1c
--- /dev/null
+++ b/go/test/endtoend/sharding/initialsharding/sharding_util.go
@@ -0,0 +1,743 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package initialsharding
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/sharding"
+	querypb "vitess.io/vitess/go/vt/proto/query"
+	"vitess.io/vitess/go/vt/proto/topodata"
+)
+
+var (
+	// ClusterInstance instance to be used for test with different params
+	ClusterInstance  *cluster.LocalProcessCluster
+	hostname         = "localhost"
+	keyspaceName1    = "ks1"
+	keyspaceName2    = "ks2"
+	dbPwd            = ""
+	cell             = "zone1"
+	newInitDbFile    string
+	dbCredentialFile string
+	vtgateInstances  []*cluster.VtgateProcess
+	commonTabletArg  = []string{
+		"-vreplication_healthcheck_topology_refresh", "1s",
+		"-vreplication_healthcheck_retry_delay", "1s",
+		"-vreplication_retry_delay", "1s",
+		"-degraded_threshold", "5s",
+		"-lock_tables_timeout", "5s",
+		"-watch_replication_stream",
+		"-enable_replication_reporter",
+		"-serving_state_grace_period", "1s",
+		"-binlog_use_v3_resharding_mode=true"}
+	createTabletTemplate = `
+							create table %s(
+							msg varchar(64),
+							id bigint(20) unsigned,
+							primary key (id)
+							) Engine=InnoDB;
+`
+	createTabletTemplateByte = `
+							create table %s(       
+                            msg varchar(64),       
+                            id varbinary(64),
+                            primary key (id)       
+                            ) Engine=InnoDB;       
+`
+	insertTabletTemplate = `insert into %s(id, msg) values(%d, "%s")`
+	tableName            = "resharding1"
+	vSchema              = `
+								{
+								  "sharded": true,
+								  "vindexes": {
+									"hash_index": {
+									  "type": "hash"
+									}
+								  },
+								  "tables": {
+									"%s": {
+									   "column_vindexes": [
+										{
+										  "column": "%s",
+										  "name": "hash_index"
+										}
+									  ] 
+									}
+								  }
+								}
+							`
+)
+
+// ClusterWrapper common wrapper code for cluster
+func ClusterWrapper(isMulti bool) (int, error) {
+	ClusterInstance = nil
+	ClusterInstance = &cluster.LocalProcessCluster{Cell: cell, Hostname: hostname}
+
+	// Start topo server
+	if err := ClusterInstance.StartTopo(); err != nil {
+		return 1, err
+	}
+
+	if isMulti {
+		writeDbCredentialToTmp()
+		tablet := &cluster.Vttablet{
+			Type:            "relpica",
+			TabletUID:       100,
+			MySQLPort:       15000,
+			MysqlctlProcess: *cluster.MysqlCtlProcessInstance(100, 15000, ClusterInstance.TmpDirectory),
+		}
+		writeInitDBFile(tablet)
+		tablet = nil
+		dbPwd = "VtDbaPass"
+	}
+
+	if err := ClusterInstance.VtctlProcess.CreateKeyspace(keyspaceName1); err != nil {
+		return 1, err
+	} else {
+		ClusterInstance.Keyspaces = append(ClusterInstance.Keyspaces, cluster.Keyspace{Name: keyspaceName1})
+	}
+	if isMulti {
+		if err := ClusterInstance.VtctlProcess.CreateKeyspace(keyspaceName2); err != nil {
+			return 1, err
+		} else {
+			ClusterInstance.Keyspaces = append(ClusterInstance.Keyspaces, cluster.Keyspace{Name: keyspaceName2})
+		}
+	}
+
+	initClusterForInitialSharding(keyspaceName1, []string{"0"}, 3, true, isMulti)
+	initClusterForInitialSharding(keyspaceName1, []string{"-80", "80-"}, 3, true, isMulti)
+
+	if isMulti {
+		initClusterForInitialSharding(keyspaceName2, []string{"0"}, 3, true, isMulti)
+		initClusterForInitialSharding(keyspaceName2, []string{"-80", "80-"}, 3, true, isMulti)
+	}
+	return 0, nil
+}
+
+func initClusterForInitialSharding(keyspaceName string, shardNames []string, totalTabletsRequired int, rdonly bool, isMulti bool) {
+	var mysqlProcesses []*exec.Cmd
+	var extraArgs []string
+	if isMulti {
+		extraArgs = []string{"-db-credentials-file", dbCredentialFile}
+	}
+	os.Setenv("EXTRA_MY_CNF", path.Join(os.Getenv("VTROOT"), "config", "mycnf", "rbr.cnf"))
+	for _, shardName := range shardNames {
+		shard := &cluster.Shard{
+			Name: shardName,
+		}
+
+		for i := 0; i < totalTabletsRequired; i++ {
+			// instantiate vttablet object with reserved ports
+			tabletUID := ClusterInstance.GetAndReserveTabletUID()
+			tablet := &cluster.Vttablet{
+				TabletUID: tabletUID,
+				HTTPPort:  ClusterInstance.GetAndReservePort(),
+				GrpcPort:  ClusterInstance.GetAndReservePort(),
+				MySQLPort: ClusterInstance.GetAndReservePort(),
+				Alias:     fmt.Sprintf("%s-%010d", ClusterInstance.Cell, tabletUID),
+				Type:      "replica",
+			}
+			if i == 0 { // Make the first one as master
+				tablet.Type = "master"
+			} else if i == totalTabletsRequired-1 && rdonly { // Make the last one as rdonly if rdonly flag is passed
+				tablet.Type = "rdonly"
+			}
+			// Start Mysqlctl process
+			tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, ClusterInstance.TmpDirectory)
+
+			if isMulti {
+				tablet.MysqlctlProcess.InitDBFile = newInitDbFile
+				tablet.MysqlctlProcess.ExtraArgs = extraArgs
+			}
+			// Start Mysqlctl process, for multi keyspace we need only 1st keyspace sql procs, that is why this check is added
+			if keyspaceName == keyspaceName1 {
+				if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil {
+					return
+				} else {
+					mysqlProcesses = append(mysqlProcesses, proc)
+				}
+			} else { // Since we'll be using mysql procs of keyspace-1 for ks-2, resetting this to 0
+				tablet.MysqlctlProcess.TabletUID = 0
+			}
+
+			// start vttablet process
+			tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort,
+				tablet.GrpcPort,
+				tablet.TabletUID,
+				ClusterInstance.Cell,
+				shardName,
+				keyspaceName,
+				ClusterInstance.VtctldProcess.Port,
+				tablet.Type,
+				ClusterInstance.TopoProcess.Port,
+				ClusterInstance.Hostname,
+				ClusterInstance.TmpDirectory,
+				ClusterInstance.VtTabletExtraArgs,
+				ClusterInstance.EnableSemiSync)
+			tablet.Alias = tablet.VttabletProcess.TabletPath
+			tablet.VttabletProcess.DbPassword = dbPwd
+			tablet.VttabletProcess.EnableSemiSync = true
+			tablet.VttabletProcess.SupportsBackup = false
+			shard.Vttablets = append(shard.Vttablets, *tablet)
+		}
+		for idx, ks := range ClusterInstance.Keyspaces {
+			if ks.Name == keyspaceName {
+				ClusterInstance.Keyspaces[idx].Shards = append(ClusterInstance.Keyspaces[idx].Shards, *shard)
+			}
+		}
+	}
+	for _, proc := range mysqlProcesses {
+		proc.Wait()
+	}
+
+}
+
+// AssignMysqlPortFromKs1ToKs2 assigns mysql port of all tablets of ks1 to all corresponding tablets of ks2
+func AssignMysqlPortFromKs1ToKs2() {
+	portMap := map[string]int{}
+	for _, shard := range ClusterInstance.Keyspaces[0].Shards {
+		for _, tablet := range shard.Vttablets {
+			portMap[fmt.Sprintf("%s-%s", shard.Name, tablet.Type)] = tablet.MySQLPort
+		}
+	}
+
+	for _, shard := range ClusterInstance.Keyspaces[1].Shards {
+		for idx, tablet := range shard.Vttablets {
+			port := portMap[fmt.Sprintf("%s-%s", shard.Name, tablet.Type)]
+			shard.Vttablets[idx].MySQLPort = port
+			shard.Vttablets[idx].VttabletProcess.DbPort = port
+			shard.Vttablets[idx].VttabletProcess.PidFile = path.Join(shard.Vttablets[idx].VttabletProcess.LogDir, fmt.Sprintf("vttablet-%d.pid", port))
+		}
+	}
+}
+
+// TestInitialSharding - main test which accepts different params for various test
+func TestInitialSharding(t *testing.T, keyspace *cluster.Keyspace, keyType querypb.Type, isMulti bool, isExternal bool) {
+	if isExternal {
+		commonTabletArg = append(commonTabletArg, "-db_host", "127.0.0.1")
+		commonTabletArg = append(commonTabletArg, "-disable_active_reparents")
+		for _, shard := range keyspace.Shards {
+			for _, tablet := range shard.Vttablets {
+				tablet.VttabletProcess.ExtraArgs = append(tablet.VttabletProcess.ExtraArgs, "-db_port", fmt.Sprintf("%d", tablet.MySQLPort))
+				tablet.VttabletProcess.DbPassword = dbPwd
+			}
+		}
+	}
+	if isMulti {
+		commonTabletArg = append(commonTabletArg, "-db-credentials-file", dbCredentialFile)
+	}
+	// Start the master and rdonly of 1st shard
+	shard1 := keyspace.Shards[0]
+	keyspaceName := keyspace.Name
+	shard1Ks := fmt.Sprintf("%s/%s", keyspaceName, shard1.Name)
+	shard1MasterTablet := *shard1.MasterTablet()
+
+	// master tablet start
+	shard1MasterTablet.VttabletProcess.ExtraArgs = append(shard1MasterTablet.VttabletProcess.ExtraArgs, commonTabletArg...)
+	//var err error
+	err := ClusterInstance.VtctlclientProcess.InitTablet(&shard1MasterTablet, cell, keyspaceName, hostname, shard1.Name)
+	assert.Nil(t, err)
+	shard1.Replica().VttabletProcess.ExtraArgs = append(shard1.Replica().VttabletProcess.ExtraArgs, commonTabletArg...)
+	shard1.Rdonly().VttabletProcess.ExtraArgs = append(shard1.Rdonly().VttabletProcess.ExtraArgs, commonTabletArg...)
+
+	for _, tablet := range shard1.Vttablets {
+		_ = tablet.VttabletProcess.CreateDB(keyspaceName)
+	}
+	err = shard1MasterTablet.VttabletProcess.Setup()
+	assert.Nil(t, err)
+
+	// replica tablet init
+	err = ClusterInstance.VtctlclientProcess.InitTablet(shard1.Replica(), cell, keyspaceName, hostname, shard1.Name)
+	assert.Nil(t, err)
+
+	// rdonly tablet start
+	err = ClusterInstance.VtctlclientProcess.InitTablet(shard1.Rdonly(), cell, keyspaceName, hostname, shard1.Name)
+	assert.Nil(t, err)
+	if isExternal {
+		shard1.Rdonly().VttabletProcess.ServingStatus = "SERVING"
+		shard1.Replica().VttabletProcess.ServingStatus = "SERVING"
+	}
+	err = shard1.Rdonly().VttabletProcess.Setup()
+	assert.Nil(t, err)
+
+	if !isMulti {
+		output, err := ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("InitShardMaster",
+			"-force", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name), shard1MasterTablet.Alias)
+		assert.NotNil(t, err, "Should fail as no replica tablet is present.")
+		assert.Contains(t, output, fmt.Sprintf("tablet %s ResetReplication failed", shard1.Replica().Alias))
+	}
+	// start replica
+	err = shard1.Replica().VttabletProcess.Setup()
+	assert.Nil(t, err)
+
+	// reparent to make the tablets work
+	if !isExternal {
+		// reparent to make the tablets work
+		err = ClusterInstance.VtctlclientProcess.InitShardMaster(keyspace.Name, shard1.Name, cell, shard1MasterTablet.TabletUID)
+		assert.Nil(t, err)
+	} else {
+		err = shard1.Replica().VttabletProcess.WaitForTabletType("SERVING")
+		assert.Nil(t, err)
+		_, err = ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("TabletExternallyReparented", shard1MasterTablet.Alias)
+		assert.Nil(t, err)
+	}
+
+	err = shard1.Replica().VttabletProcess.WaitForTabletType("SERVING")
+	assert.Nil(t, err)
+	err = shard1.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
+	assert.Nil(t, err)
+	for _, vttablet := range shard1.Vttablets {
+		assert.Equal(t, vttablet.VttabletProcess.GetTabletStatus(), "SERVING")
+	}
+	// create the tables and add startup values
+	sqlSchemaToApply := createTabletTemplate
+	if keyType == querypb.Type_VARBINARY {
+		sqlSchemaToApply = createTabletTemplateByte
+	}
+	err = ClusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, fmt.Sprintf(sqlSchemaToApply, tableName))
+	assert.Nil(t, err)
+
+	err = ClusterInstance.VtctlclientProcess.ApplyVSchema(keyspaceName, fmt.Sprintf(vSchema, tableName, "id"))
+	assert.Nil(t, err)
+	_, err = shard1MasterTablet.VttabletProcess.QueryTablet(fmt.Sprintf(insertTabletTemplate, tableName, uint64(0x1000000000000000), "msg1"), keyspaceName, true)
+	assert.Nil(t, err)
+	_, err = shard1MasterTablet.VttabletProcess.QueryTablet(fmt.Sprintf(insertTabletTemplate, tableName, uint64(0x9000000000000000), "msg2"), keyspaceName, true)
+	assert.Nil(t, err)
+	_, err = shard1MasterTablet.VttabletProcess.QueryTablet(fmt.Sprintf(insertTabletTemplate, tableName, uint64(0xD000000000000000), "msg3"), keyspaceName, true)
+	assert.Nil(t, err)
+
+	// reload schema on all tablets so we can query them
+	for _, vttablet := range shard1.Vttablets {
+		_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("ReloadSchema", vttablet.Alias)
+	}
+	vtgateInstance := ClusterInstance.GetVtgateInstance()
+	vtgateInstance.PidFile = path.Join(ClusterInstance.TmpDirectory, fmt.Sprintf("vtgate-%s.pid", keyspaceName))
+	vtgateInstance.MySQLServerSocketPath = path.Join(ClusterInstance.TmpDirectory, fmt.Sprintf("mysql-%s.sock", keyspaceName))
+	vtgateInstance.ExtraArgs = []string{"-retry-count", fmt.Sprintf("%d", 2), "-tablet_protocol", "grpc", "-normalize_queries", "-tablet_refresh_interval", "2s"}
+	err = vtgateInstance.Setup()
+	vtgateInstances = append(vtgateInstances, vtgateInstance)
+	assert.Nil(t, err)
+
+	for _, tabletType := range []string{"master", "replica", "rdonly"} {
+		if err = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.%s", keyspaceName, shard1.Name, tabletType)); err != nil {
+			assert.Fail(t, err.Error())
+		}
+	}
+
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", keyspaceName)
+
+	// run a health check on source replica so it responds to discovery
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", shard1.Replica().Alias)
+	assert.Nil(t, err)
+
+	// create the split shards
+	shard21 := keyspace.Shards[1]
+	shard22 := keyspace.Shards[2]
+
+	for _, shard := range []cluster.Shard{shard21, shard22} {
+		for idx, vttablet := range shard.Vttablets {
+			vttablet.VttabletProcess.ExtraArgs = append(vttablet.VttabletProcess.ExtraArgs, commonTabletArg...)
+			err = ClusterInstance.VtctlclientProcess.InitTablet(&vttablet, cell, keyspaceName, hostname, shard.Name)
+			assert.Nil(t, err)
+			_ = vttablet.VttabletProcess.CreateDB(keyspaceName)
+			if isExternal {
+				shard.Vttablets[idx].VttabletProcess.ServingStatus = ""
+			}
+			err = vttablet.VttabletProcess.Setup()
+			assert.Nil(t, err)
+		}
+	}
+	if !isExternal {
+		_ = ClusterInstance.VtctlclientProcess.InitShardMaster(keyspaceName, shard21.Name, cell, shard21.MasterTablet().TabletUID)
+		_ = ClusterInstance.VtctlclientProcess.InitShardMaster(keyspaceName, shard22.Name, cell, shard22.MasterTablet().TabletUID)
+		_ = ClusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, fmt.Sprintf(sqlSchemaToApply, tableName))
+		_ = ClusterInstance.VtctlclientProcess.ApplyVSchema(keyspaceName, fmt.Sprintf(vSchema, tableName, "id"))
+
+		for _, shard := range []cluster.Shard{shard21, shard22} {
+			_ = shard.Replica().VttabletProcess.WaitForTabletType("SERVING")
+			_ = shard.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
+		}
+
+		for _, shard := range []cluster.Shard{shard21, shard22} {
+			for _, vttablet := range shard.Vttablets {
+				assert.Equal(t, vttablet.VttabletProcess.GetTabletStatus(), "SERVING")
+			}
+		}
+	} else {
+		_, err = ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("TabletExternallyReparented", shard21.MasterTablet().Alias)
+		assert.Nil(t, err)
+		_, err = ClusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("TabletExternallyReparented", shard22.MasterTablet().Alias)
+		assert.Nil(t, err)
+	}
+
+	// must restart vtgate after tablets are up, or else wait until 1min refresh
+	// we want cache_ttl at zero so we re-read the topology for every test query.
+
+	_ = vtgateInstance.TearDown()
+	_ = vtgateInstance.Setup()
+
+	// Wait for the endpoints, either local or remote.
+	for _, shard := range []cluster.Shard{shard1, shard21, shard22} {
+		err = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", keyspaceName, shard.Name))
+		assert.Nil(t, err)
+		err = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.replica", keyspaceName, shard.Name))
+		assert.Nil(t, err)
+		err = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspaceName, shard.Name))
+		assert.Nil(t, err)
+	}
+
+	// Check srv keyspace
+	expectedPartitions := map[topodata.TabletType][]string{}
+	expectedPartitions[topodata.TabletType_MASTER] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_REPLICA] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_RDONLY] = []string{shard1.Name}
+	checkSrvKeyspaceForSharding(t, keyspaceName, expectedPartitions)
+
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("CopySchemaShard",
+		"--exclude_tables", "unrelated",
+		shard1.Rdonly().Alias, fmt.Sprintf("%s/%s", keyspaceName, shard21.Name))
+	assert.Nil(t, err)
+
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("CopySchemaShard",
+		"--exclude_tables", "unrelated",
+		shard1.Rdonly().Alias, fmt.Sprintf("%s/%s", keyspaceName, shard22.Name))
+	assert.Nil(t, err)
+
+	err = ClusterInstance.StartVtworker(cell, "--use_v3_resharding_mode=true")
+	assert.Nil(t, err)
+
+	// Initial clone (online).
+	_ = ClusterInstance.VtworkerProcess.ExecuteCommand("SplitClone",
+		"--offline=false",
+		"--exclude_tables", "unrelated",
+		"--chunk_count", "10",
+		"--min_rows_per_chunk", "1",
+		"--min_healthy_rdonly_tablets", "1",
+		fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
+
+	// Reset vtworker such that we can run the next command.
+	_ = ClusterInstance.VtworkerProcess.ExecuteCommand("Reset")
+
+	// Modify the destination shard. SplitClone will revert the changes.
+	// Delete row 1 (provokes an insert).
+	_, _ = shard21.MasterTablet().VttabletProcess.QueryTablet(fmt.Sprintf("delete from %s where id=%d", tableName, uint64(0x1000000000000000)), keyspaceName, true)
+	// Delete row 2 (provokes an insert).
+	_, _ = shard22.MasterTablet().VttabletProcess.QueryTablet(fmt.Sprintf("delete from %s where id=%d", tableName, uint64(0x9000000000000000)), keyspaceName, true)
+	//  Update row 3 (provokes an update).
+	_, _ = shard22.MasterTablet().VttabletProcess.QueryTablet(fmt.Sprintf("update %s set msg='msg-not-3' where id=%d", tableName, uint64(0xD000000000000000)), keyspaceName, true)
+	// Insert row 4 (provokes a delete).
+	var ksid uint64 = 0xD000000000000000
+	insertSQL := fmt.Sprintf(sharding.InsertTabletTemplateKsID, tableName, ksid, "msg4", ksid)
+	sharding.InsertToTablet(t, insertSQL, *shard22.MasterTablet(), keyspaceName, true)
+
+	_ = ClusterInstance.VtworkerProcess.ExecuteCommand("SplitClone",
+		"--exclude_tables", "unrelated",
+		"--chunk_count", "10",
+		"--min_rows_per_chunk", "1",
+		"--min_healthy_rdonly_tablets", "1",
+		fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
+
+	// check first value is in the left shard
+	for _, tablet := range shard21.Vttablets {
+		sharding.CheckValues(t, tablet, 0x1000000000000000, "msg1", true, tableName, keyspaceName, keyType)
+	}
+
+	for _, tablet := range shard22.Vttablets {
+		sharding.CheckValues(t, tablet, 0x1000000000000000, "msg1", false, tableName, keyspaceName, keyType)
+	}
+
+	for _, tablet := range shard21.Vttablets {
+		sharding.CheckValues(t, tablet, 0x9000000000000000, "msg2", false, tableName, keyspaceName, keyType)
+	}
+
+	for _, tablet := range shard22.Vttablets {
+		sharding.CheckValues(t, tablet, 0x9000000000000000, "msg2", true, tableName, keyspaceName, keyType)
+	}
+
+	for _, tablet := range shard21.Vttablets {
+		sharding.CheckValues(t, tablet, 0xD000000000000000, "msg3", false, tableName, keyspaceName, keyType)
+	}
+
+	for _, tablet := range shard22.Vttablets {
+		sharding.CheckValues(t, tablet, 0xD000000000000000, "msg3", true, tableName, keyspaceName, keyType)
+	}
+
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("ValidateSchemaKeyspace", keyspaceName)
+	assert.Nil(t, err)
+
+	// check the binlog players are running
+	sharding.CheckDestinationMaster(t, *shard21.MasterTablet(), []string{shard1Ks}, *ClusterInstance)
+	sharding.CheckDestinationMaster(t, *shard22.MasterTablet(), []string{shard1Ks}, *ClusterInstance)
+
+	//  check that binlog server exported the stats vars
+	sharding.CheckBinlogServerVars(t, *shard1.Replica(), 0, 0)
+
+	for _, tablet := range []cluster.Vttablet{*shard21.Rdonly(), *shard22.Rdonly()} {
+		err = ClusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", tablet.Alias)
+		assert.Nil(t, err)
+	}
+
+	// testing filtered replication: insert a bunch of data on shard 1,
+	// check we get most of it after a few seconds, wait for binlog server
+	// timeout, check we get all of it.
+	sharding.InsertLots(t, 1000, shard1MasterTablet, tableName, keyspaceName)
+
+	assert.True(t, sharding.CheckLotsTimeout(t, *shard21.Replica(), 1000, tableName, keyspaceName, keyType, 49))
+	assert.True(t, sharding.CheckLotsTimeout(t, *shard22.Replica(), 1000, tableName, keyspaceName, keyType, 51))
+
+	sharding.CheckDestinationMaster(t, *shard21.MasterTablet(), []string{shard1Ks}, *ClusterInstance)
+	sharding.CheckDestinationMaster(t, *shard22.MasterTablet(), []string{shard1Ks}, *ClusterInstance)
+	sharding.CheckBinlogServerVars(t, *shard1.Replica(), 1000, 1000)
+
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", shard21.Rdonly().Alias)
+	assert.Nil(t, err)
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", shard22.Rdonly().Alias)
+	assert.Nil(t, err)
+
+	//use vtworker to compare the data
+	ClusterInstance.VtworkerProcess.Cell = cell
+	if !isMulti {
+		err = ClusterInstance.VtworkerProcess.ExecuteVtworkerCommand(ClusterInstance.GetAndReservePort(),
+			ClusterInstance.GetAndReservePort(),
+			"--use_v3_resharding_mode=true",
+			"MultiSplitDiff",
+			fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
+		assert.Nil(t, err)
+	}
+
+	for _, shard := range []string{shard21.Name, shard22.Name} {
+		err = ClusterInstance.VtworkerProcess.ExecuteVtworkerCommand(ClusterInstance.GetAndReservePort(),
+			ClusterInstance.GetAndReservePort(),
+			"--use_v3_resharding_mode=true",
+			"SplitDiff",
+			"--min_healthy_rdonly_tablets", "1",
+			fmt.Sprintf("%s/%s", keyspaceName, shard))
+		assert.Nil(t, err)
+	}
+
+	// get status for the destination master tablet, make sure we have it all
+	sharding.CheckRunningBinlogPlayer(t, *shard21.MasterTablet(), 3954, 2000)
+	sharding.CheckRunningBinlogPlayer(t, *shard22.MasterTablet(), 4046, 2000)
+
+	// check we can't migrate the master just yet
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("MigrateServedTypes", shard1Ks, "master")
+	assert.NotNil(t, err)
+
+	// now serve rdonly from the split shards
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("MigrateServedTypes", shard1Ks, "rdonly")
+	assert.Nil(t, err)
+	expectedPartitions = map[topodata.TabletType][]string{}
+	expectedPartitions[topodata.TabletType_MASTER] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_REPLICA] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_RDONLY] = []string{shard21.Name, shard22.Name}
+	checkSrvKeyspaceForSharding(t, keyspaceName, expectedPartitions)
+
+	_ = shard21.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
+	_ = shard22.Rdonly().VttabletProcess.WaitForTabletType("SERVING")
+
+	_ = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspaceName, shard21.Name))
+	_ = vtgateInstance.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspaceName, shard22.Name))
+
+	//then serve replica from the split shards
+
+	sourceTablet := shard1.Replica()
+	destinationTablets := []cluster.Vttablet{*shard21.Replica(), *shard22.Replica()}
+
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("MigrateServedTypes", shard1Ks, "replica")
+	expectedPartitions = map[topodata.TabletType][]string{}
+	expectedPartitions[topodata.TabletType_MASTER] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_REPLICA] = []string{shard21.Name, shard22.Name}
+	expectedPartitions[topodata.TabletType_RDONLY] = []string{shard21.Name, shard22.Name}
+	checkSrvKeyspaceForSharding(t, keyspaceName, expectedPartitions)
+
+	//move replica back and forth
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("MigrateServedTypes", "-reverse", shard1Ks, "replica")
+
+	// After a backwards migration, queryservice should be enabled on source and disabled on destinations
+	sharding.CheckTabletQueryService(t, *sourceTablet, "SERVING", false, *ClusterInstance)
+	sharding.CheckTabletQueryServices(t, destinationTablets, "NOT_SERVING", true, *ClusterInstance)
+
+	expectedPartitions = map[topodata.TabletType][]string{}
+	expectedPartitions[topodata.TabletType_MASTER] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_REPLICA] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_RDONLY] = []string{shard21.Name, shard22.Name}
+	checkSrvKeyspaceForSharding(t, keyspaceName, expectedPartitions)
+
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("MigrateServedTypes", shard1Ks, "replica")
+
+	// After a forwards migration, queryservice should be disabled on source and enabled on destinations
+	sharding.CheckTabletQueryService(t, *sourceTablet, "NOT_SERVING", true, *ClusterInstance)
+	sharding.CheckTabletQueryServices(t, destinationTablets, "SERVING", false, *ClusterInstance)
+	expectedPartitions = map[topodata.TabletType][]string{}
+	expectedPartitions[topodata.TabletType_MASTER] = []string{shard1.Name}
+	expectedPartitions[topodata.TabletType_REPLICA] = []string{shard21.Name, shard22.Name}
+	expectedPartitions[topodata.TabletType_RDONLY] = []string{shard21.Name, shard22.Name}
+	checkSrvKeyspaceForSharding(t, keyspaceName, expectedPartitions)
+
+	// then serve master from the split shards
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("MigrateServedTypes", shard1Ks, "master")
+	expectedPartitions = map[topodata.TabletType][]string{}
+	expectedPartitions[topodata.TabletType_MASTER] = []string{shard21.Name, shard22.Name}
+	expectedPartitions[topodata.TabletType_REPLICA] = []string{shard21.Name, shard22.Name}
+	expectedPartitions[topodata.TabletType_RDONLY] = []string{shard21.Name, shard22.Name}
+	checkSrvKeyspaceForSharding(t, keyspaceName, expectedPartitions)
+
+	// check the binlog players are gone now
+	err = shard21.MasterTablet().VttabletProcess.WaitForBinLogPlayerCount(0)
+	assert.Nil(t, err)
+	err = shard22.MasterTablet().VttabletProcess.WaitForBinLogPlayerCount(0)
+	assert.Nil(t, err)
+
+	// make sure we can't delete a shard with tablets
+	err = ClusterInstance.VtctlclientProcess.ExecuteCommand("DeleteShard", shard1Ks)
+	assert.NotNil(t, err)
+	if !isMulti {
+		KillTabletsInKeyspace(keyspace)
+		KillVtgateInstances()
+	}
+	ClusterInstance.VtworkerProcess.TearDown()
+
+}
+
+// KillTabletsInKeyspace kill the first shard tablets in ordered way
+func KillTabletsInKeyspace(keyspace *cluster.Keyspace) {
+	// Teardown
+	shard1 := keyspace.Shards[0]
+	var mysqlctlProcessList []*exec.Cmd
+	for _, tablet := range []cluster.Vttablet{*shard1.MasterTablet(), *shard1.Replica(), *shard1.Rdonly()} {
+		proc, _ := tablet.MysqlctlProcess.StopProcess()
+		mysqlctlProcessList = append(mysqlctlProcessList, proc)
+		_ = tablet.VttabletProcess.TearDown()
+	}
+	for _, proc := range mysqlctlProcessList {
+		proc.Wait()
+	}
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", shard1.Replica().Alias)
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", shard1.Rdonly().Alias)
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", "-allow_master", shard1.MasterTablet().Alias)
+
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", keyspace.Name)
+	_ = ClusterInstance.VtctlclientProcess.ExecuteCommand("DeleteShard", keyspace.Name+"/"+shard1.Name)
+}
+
+// KillVtgateInstances stops the vtgate process
+func KillVtgateInstances() {
+	if len(vtgateInstances) > 0 {
+		for _, vtgateInstance := range vtgateInstances {
+			_ = vtgateInstance.TearDown()
+		}
+	}
+}
+
+func checkSrvKeyspaceForSharding(t *testing.T, ksName string, expectedPartitions map[topodata.TabletType][]string) {
+	sharding.CheckSrvKeyspace(t, cell, ksName, "", 0, expectedPartitions, *ClusterInstance)
+}
+
+// Create a new init_db.sql file that sets up passwords for all users.
+// Then we use a db-credentials-file with the passwords.
+func writeInitDBFile(vttablet *cluster.Vttablet) {
+	initDb, _ := ioutil.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql"))
+	sql := string(initDb)
+	newInitDbFile = path.Join(ClusterInstance.TmpDirectory, "init_db_with_passwords.sql")
+	pwdChangeCmd := `
+					# Set real passwords for all users.
+					UPDATE mysql.user SET %s = PASSWORD('RootPass')
+					  WHERE User = 'root' AND Host = 'localhost';
+					UPDATE mysql.user SET %s = PASSWORD('VtDbaPass')
+					  WHERE User = 'vt_dba' AND Host = 'localhost';
+					UPDATE mysql.user SET %s = PASSWORD('VtAppPass')
+					  WHERE User = 'vt_app' AND Host = 'localhost';
+					UPDATE mysql.user SET %s = PASSWORD('VtAllprivsPass')
+					  WHERE User = 'vt_allprivs' AND Host = 'localhost';
+					UPDATE mysql.user SET %s = PASSWORD('VtReplPass')
+					  WHERE User = 'vt_repl' AND Host = '%%';
+					UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass')
+					  WHERE User = 'vt_filtered' AND Host = 'localhost';
+					FLUSH PRIVILEGES;
+					`
+	pwdCol, _ := getPasswordField(vttablet)
+	sql = sql + fmt.Sprintf(pwdChangeCmd, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol) + `
+# connecting through a port requires 127.0.0.1
+# --host=localhost will connect through socket
+CREATE USER 'vt_dba'@'127.0.0.1' IDENTIFIED BY 'VtDbaPass';
+GRANT ALL ON *.* TO 'vt_dba'@'127.0.0.1';
+GRANT GRANT OPTION ON *.* TO 'vt_dba'@'127.0.0.1';
+# User for app traffic, with global read-write access.
+CREATE USER 'vt_app'@'127.0.0.1' IDENTIFIED BY 'VtAppPass';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+  REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_app'@'127.0.0.1';
+# User for administrative operations that need to be executed as non-SUPER.
+# Same permissions as vt_app here.
+CREATE USER 'vt_allprivs'@'127.0.0.1' IDENTIFIED BY 'VtAllPrivsPass';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+  REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_allprivs'@'127.0.0.1';
+# User for Vitess filtered replication (binlog player).
+# Same permissions as vt_app.
+CREATE USER 'vt_filtered'@'127.0.0.1' IDENTIFIED BY 'VtFilteredPass';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+  REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_filtered'@'127.0.0.1';
+FLUSH PRIVILEGES;
+`
+	ioutil.WriteFile(newInitDbFile, []byte(sql), 0666)
+
+}
+
+func writeDbCredentialToTmp() {
+	data := []byte(`{
+        "vt_dba": ["VtDbaPass"],
+        "vt_app": ["VtAppPass"],
+        "vt_allprivs": ["VtAllprivsPass"],
+        "vt_repl": ["VtReplPass"],
+        "vt_filtered": ["VtFilteredPass"]
+    	}`)
+	dbCredentialFile = path.Join(ClusterInstance.TmpDirectory, "db_credentials.json")
+	ioutil.WriteFile(dbCredentialFile, data, 0666)
+}
+
+// getPasswordField Determines which column is used for user passwords in this MySQL version.
+func getPasswordField(tablet *cluster.Vttablet) (pwdCol string, err error) {
+	if err = tablet.MysqlctlProcess.Start(); err != nil {
+		return "", err
+	}
+	tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, tablet.GrpcPort, tablet.TabletUID, "", "", "", 0,
+		tablet.Type, ClusterInstance.TopoPort, "", "", nil, false)
+	result, err := tablet.VttabletProcess.QueryTablet("select password from mysql.user limit 0", "", false)
+	if err == nil && len(result.Rows) > 0 {
+		return "password", nil
+	}
+	tablet.MysqlctlProcess.Stop()
+	os.RemoveAll(path.Join(tablet.VttabletProcess.Directory))
+	return "authentication_string", nil
+
+}
diff --git a/go/test/endtoend/sharding/initialsharding/v3/initial_sharding_test.go b/go/test/endtoend/sharding/initialsharding/v3/initial_sharding_test.go
new file mode 100644
index 0000000000..d971c87490
--- /dev/null
+++ b/go/test/endtoend/sharding/initialsharding/v3/initial_sharding_test.go
@@ -0,0 +1,45 @@
+/*
+Copyright 2019 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+This test simulates the first time a database has to be split.
+
+- we start with a keyspace with a single shard and a single table
+- we add and populate the sharding key
+- we set the sharding key in the topology
+- we clone into 2 instances
+- we enable filtered replication
+- we move all serving types
+- we remove the source tablets
+- we remove the original shard
+
+*/
+
+package v3
+
+import (
+	"testing"
+
+	sharding "vitess.io/vitess/go/test/endtoend/sharding/initialsharding"
+	querypb "vitess.io/vitess/go/vt/proto/query"
+)
+
+func TestInitialSharding(t *testing.T) {
+	code, err := sharding.ClusterWrapper(false)
+	if err != nil {
+		t.Errorf("setup failed with status code %d", code)
+	}
+	sharding.TestInitialSharding(t, &sharding.ClusterInstance.Keyspaces[0], querypb.Type_UINT64, false, false)
+	defer sharding.ClusterInstance.Teardown()
+}
diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go
index bf6ed5e2e2..c0c24bb555 100644
--- a/go/test/endtoend/tabletmanager/tablet_health_test.go
+++ b/go/test/endtoend/tabletmanager/tablet_health_test.go
@@ -300,4 +300,4 @@ func killTablets(t *testing.T, tablets ...*cluster.Vttablet) {
 		tablet.VttabletProcess.TearDown()
 
 	}
-}
\ No newline at end of file
+}
diff --git a/go/vt/discovery/healthcheck_flaky_test.go b/go/vt/discovery/healthcheck_test.go
similarity index 100%
rename from go/vt/discovery/healthcheck_flaky_test.go
rename to go/vt/discovery/healthcheck_test.go
diff --git a/go/vt/logutil/logutil_flaky_test.go b/go/vt/logutil/logutil_test.go
similarity index 100%
rename from go/vt/logutil/logutil_flaky_test.go
rename to go/vt/logutil/logutil_test.go
diff --git a/go/vt/mysqlctl/mycnf_test.go b/go/vt/mysqlctl/mycnf_test.go
index 59c4247eac..0fd3e51083 100644
--- a/go/vt/mysqlctl/mycnf_test.go
+++ b/go/vt/mysqlctl/mycnf_test.go
@@ -24,7 +24,6 @@ import (
 	"testing"
 
 	"vitess.io/vitess/go/vt/dbconfigs"
-	"vitess.io/vitess/go/vt/env"
 	"vitess.io/vitess/go/vt/servenv"
 )
 
@@ -37,12 +36,8 @@ func TestMycnf(t *testing.T) {
 	// Assigning ServerID to be different from tablet UID to make sure that there are no
 	// assumptions in the code that those IDs are the same.
 	cnf.ServerID = 22222
-	root, err := env.VtRoot()
-	if err != nil {
-		t.Errorf("err: %v", err)
-	}
 	cnfTemplatePaths := []string{
-		path.Join(root, "config/mycnf/default.cnf"),
+		path.Join(os.Getenv("VTTOP"), "/config/mycnf/default.cnf"),
 	}
 	data, err := cnf.makeMycnf(cnfTemplatePaths)
 	if err != nil {
@@ -79,7 +74,7 @@ func TestMycnf(t *testing.T) {
 
 // Run this test if any changes are made to hook handling / make_mycnf hook
 // other tests fail if we keep the hook around
-// 1. ln -snf $VTROOT/test/vthook-make_mycnf $VTROOT/vthook/make_mycnf
+// 1. ln -snf $VTTOP/test/vthook-make_mycnf $VTROOT/vthook/make_mycnf
 // 2. Remove "No" prefix from func name
 // 3. go test
 // 4. \rm $VTROOT/vthook/make_mycnf
diff --git a/go/vt/sqlparser/analyzer_test.go b/go/vt/sqlparser/analyzer_test.go
index f97f36ccbc..8ecc3ce0db 100644
--- a/go/vt/sqlparser/analyzer_test.go
+++ b/go/vt/sqlparser/analyzer_test.go
@@ -50,9 +50,6 @@ func TestPreview(t *testing.T) {
 		{"begin ...", StmtUnknown},
 		{"begin /* ... */", StmtBegin},
 		{"begin /* ... *//*test*/", StmtBegin},
-		{"begin;", StmtBegin},
-		{"begin ;", StmtBegin},
-		{"begin; /*...*/", StmtBegin},
 		{"start transaction", StmtBegin},
 		{"commit", StmtCommit},
 		{"commit /*...*/", StmtCommit},
diff --git a/go/vt/sqlparser/parse_test.go b/go/vt/sqlparser/parse_test.go
index 8ae27a8594..564edccefc 100644
--- a/go/vt/sqlparser/parse_test.go
+++ b/go/vt/sqlparser/parse_test.go
@@ -1442,9 +1442,6 @@ var (
 		input: "stream /* comment */ * from t",
 	}, {
 		input: "begin",
-	}, {
-		input:  "begin;",
-		output: "begin",
 	}, {
 		input:  "start transaction",
 		output: "begin",
diff --git a/go/vt/tlstest/tlstest.go b/go/vt/tlstest/tlstest.go
index 810aa4c279..89b715cf52 100644
--- a/go/vt/tlstest/tlstest.go
+++ b/go/vt/tlstest/tlstest.go
@@ -147,51 +147,3 @@ func CreateSignedCert(root, parent, serial, name, commonName string) {
 		"-extfile", config,
 		"-out", cert)
 }
-
-type ClientServerKeyPairs struct {
-	ServerCert string
-	ServerKey  string
-	ServerCA   string
-	ServerName string
-	ClientCert string
-	ClientKey  string
-	ClientCA   string
-}
-
-var serialCounter = 0
-
-func CreateClientServerCertPairs(root string) ClientServerKeyPairs {
-	// Create the certs and configs.
-	CreateCA(root)
-
-	serverSerial := fmt.Sprintf("%03d", serialCounter*2+1)
-	clientSerial := fmt.Sprintf("%03d", serialCounter*2+2)
-
-	serialCounter = serialCounter + 1
-
-	serverName := fmt.Sprintf("server-%s", serverSerial)
-	serverCACommonName := fmt.Sprintf("Server %s CA", serverSerial)
-	serverCertName := fmt.Sprintf("server-instance-%s", serverSerial)
-	serverCertCommonName := fmt.Sprintf("server%s.example.com", serverSerial)
-
-	clientName := fmt.Sprintf("clients-%s", serverSerial)
-	clientCACommonName := fmt.Sprintf("Clients %s CA", serverSerial)
-	clientCertName := fmt.Sprintf("client-instance-%s", serverSerial)
-	clientCertCommonName := fmt.Sprintf("Client Instance %s", serverSerial)
-
-	CreateSignedCert(root, CA, serverSerial, serverName, serverCACommonName)
-	CreateSignedCert(root, serverName, serverSerial, serverCertName, serverCertCommonName)
-
-	CreateSignedCert(root, CA, clientSerial, clientName, clientCACommonName)
-	CreateSignedCert(root, clientName, serverSerial, clientCertName, clientCertCommonName)
-
-	return ClientServerKeyPairs{
-		ServerCert: path.Join(root, fmt.Sprintf("%s-cert.pem", serverCertName)),
-		ServerKey:  path.Join(root, fmt.Sprintf("%s-key.pem", serverCertName)),
-		ServerCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", serverName)),
-		ClientCert: path.Join(root, fmt.Sprintf("%s-cert.pem", clientCertName)),
-		ClientKey:  path.Join(root, fmt.Sprintf("%s-key.pem", clientCertName)),
-		ClientCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", clientName)),
-		ServerName: serverCertCommonName,
-	}
-}
diff --git a/go/vt/tlstest/tlstest_test.go b/go/vt/tlstest/tlstest_test.go
index 746ab161ce..1f22403cdd 100644
--- a/go/vt/tlstest/tlstest_test.go
+++ b/go/vt/tlstest/tlstest_test.go
@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
 	"strings"
 	"sync"
 	"testing"
@@ -46,20 +47,20 @@ func TestClientServer(t *testing.T) {
 	}
 	defer os.RemoveAll(root)
 
-	clientServerKeyPairs := CreateClientServerCertPairs(root)
+	clientServerKeyPairs := createClientServerCertPairs(root)
 
 	serverConfig, err := vttls.ServerConfig(
-		clientServerKeyPairs.ServerCert,
-		clientServerKeyPairs.ServerKey,
-		clientServerKeyPairs.ClientCA)
+		clientServerKeyPairs.serverCert,
+		clientServerKeyPairs.serverKey,
+		clientServerKeyPairs.clientCA)
 	if err != nil {
 		t.Fatalf("TLSServerConfig failed: %v", err)
 	}
 	clientConfig, err := vttls.ClientConfig(
-		clientServerKeyPairs.ClientCert,
-		clientServerKeyPairs.ClientKey,
-		clientServerKeyPairs.ServerCA,
-		clientServerKeyPairs.ServerName)
+		clientServerKeyPairs.clientCert,
+		clientServerKeyPairs.clientKey,
+		clientServerKeyPairs.serverCA,
+		clientServerKeyPairs.serverName)
 	if err != nil {
 		t.Fatalf("TLSClientConfig failed: %v", err)
 	}
@@ -116,10 +117,10 @@ func TestClientServer(t *testing.T) {
 	//
 
 	badClientConfig, err := vttls.ClientConfig(
-		clientServerKeyPairs.ServerCert,
-		clientServerKeyPairs.ServerKey,
-		clientServerKeyPairs.ServerCA,
-		clientServerKeyPairs.ServerName)
+		clientServerKeyPairs.serverCert,
+		clientServerKeyPairs.serverKey,
+		clientServerKeyPairs.serverCA,
+		clientServerKeyPairs.serverName)
 	if err != nil {
 		t.Fatalf("TLSClientConfig failed: %v", err)
 	}
@@ -164,19 +165,69 @@ func TestClientServer(t *testing.T) {
 	}
 }
 
-func getServerConfig(keypairs ClientServerKeyPairs) (*tls.Config, error) {
+var serialCounter = 0
+
+type clientServerKeyPairs struct {
+	serverCert string
+	serverKey  string
+	serverCA   string
+	serverName string
+	clientCert string
+	clientKey  string
+	clientCA   string
+}
+
+func createClientServerCertPairs(root string) clientServerKeyPairs {
+
+	// Create the certs and configs.
+	CreateCA(root)
+
+	serverSerial := fmt.Sprintf("%03d", serialCounter*2+1)
+	clientSerial := fmt.Sprintf("%03d", serialCounter*2+2)
+
+	serialCounter = serialCounter + 1
+
+	serverName := fmt.Sprintf("server-%s", serverSerial)
+	serverCACommonName := fmt.Sprintf("Server %s CA", serverSerial)
+	serverCertName := fmt.Sprintf("server-instance-%s", serverSerial)
+	serverCertCommonName := fmt.Sprintf("server%s.example.com", serverSerial)
+
+	clientName := fmt.Sprintf("clients-%s", serverSerial)
+	clientCACommonName := fmt.Sprintf("Clients %s CA", serverSerial)
+	clientCertName := fmt.Sprintf("client-instance-%s", serverSerial)
+	clientCertCommonName := fmt.Sprintf("Client Instance %s", serverSerial)
+
+	CreateSignedCert(root, CA, serverSerial, serverName, serverCACommonName)
+	CreateSignedCert(root, serverName, serverSerial, serverCertName, serverCertCommonName)
+
+	CreateSignedCert(root, CA, clientSerial, clientName, clientCACommonName)
+	CreateSignedCert(root, clientName, serverSerial, clientCertName, clientCertCommonName)
+
+	return clientServerKeyPairs{
+		serverCert: path.Join(root, fmt.Sprintf("%s-cert.pem", serverCertName)),
+		serverKey:  path.Join(root, fmt.Sprintf("%s-key.pem", serverCertName)),
+		serverCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", serverName)),
+		clientCert: path.Join(root, fmt.Sprintf("%s-cert.pem", clientCertName)),
+		clientKey:  path.Join(root, fmt.Sprintf("%s-key.pem", clientCertName)),
+		clientCA:   path.Join(root, fmt.Sprintf("%s-cert.pem", clientName)),
+		serverName: serverCertCommonName,
+	}
+
+}
+
+func getServerConfig(keypairs clientServerKeyPairs) (*tls.Config, error) {
 	return vttls.ServerConfig(
-		keypairs.ClientCert,
-		keypairs.ClientKey,
-		keypairs.ServerCA)
+		keypairs.clientCert,
+		keypairs.clientKey,
+		keypairs.serverCA)
 }
 
-func getClientConfig(keypairs ClientServerKeyPairs) (*tls.Config, error) {
+func getClientConfig(keypairs clientServerKeyPairs) (*tls.Config, error) {
 	return vttls.ClientConfig(
-		keypairs.ClientCert,
-		keypairs.ClientKey,
-		keypairs.ServerCA,
-		keypairs.ServerName)
+		keypairs.clientCert,
+		keypairs.clientKey,
+		keypairs.serverCA,
+		keypairs.serverName)
 }
 
 func TestServerTLSConfigCaching(t *testing.T) {
@@ -191,7 +242,7 @@ func TestClientTLSConfigCaching(t *testing.T) {
 	})
 }
 
-func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(ClientServerKeyPairs) (*tls.Config, error), getCertPool func(tlsConfig *tls.Config) *x509.CertPool) {
+func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(clientServerKeyPairs) (*tls.Config, error), getCertPool func(tlsConfig *tls.Config) *x509.CertPool) {
 	// Our test root.
 	root, err := ioutil.TempDir("", rootPrefix)
 	if err != nil {
@@ -201,8 +252,8 @@ func testConfigGeneration(t *testing.T, rootPrefix string, generateConfig func(C
 
 	const configsToGenerate = 1
 
-	firstClientServerKeyPairs := CreateClientServerCertPairs(root)
-	secondClientServerKeyPairs := CreateClientServerCertPairs(root)
+	firstClientServerKeyPairs := createClientServerCertPairs(root)
+	secondClientServerKeyPairs := createClientServerCertPairs(root)
 
 	firstExpectedConfig, _ := generateConfig(firstClientServerKeyPairs)
 	secondExpectedConfig, _ := generateConfig(secondClientServerKeyPairs)
diff --git a/go/vt/topo/etcd2topo/server_test.go b/go/vt/topo/etcd2topo/server_test.go
index 7e2a0784b9..c782e9638a 100644
--- a/go/vt/topo/etcd2topo/server_test.go
+++ b/go/vt/topo/etcd2topo/server_test.go
@@ -25,10 +25,6 @@ import (
 	"testing"
 	"time"
 
-	"vitess.io/vitess/go/vt/tlstest"
-
-	"github.com/coreos/etcd/pkg/transport"
-
 	"golang.org/x/net/context"
 
 	"github.com/coreos/etcd/clientv3"
@@ -93,130 +89,6 @@ func startEtcd(t *testing.T) (*exec.Cmd, string, string) {
 	return cmd, dataDir, clientAddr
 }
 
-// startEtcdWithTLS starts an etcd subprocess with TLS setup, and waits for it to be ready.
-func startEtcdWithTLS(t *testing.T) (string, *tlstest.ClientServerKeyPairs, func()) {
-	// Create a temporary directory.
-	dataDir, err := ioutil.TempDir("", "etcd")
-	if err != nil {
-		t.Fatalf("cannot create tempdir: %v", err)
-	}
-
-	// Get our two ports to listen to.
-	port := testfiles.GoVtTopoEtcd2topoPort
-	name := "vitess_unit_test"
-	clientAddr := fmt.Sprintf("https://localhost:%v", port+2)
-	peerAddr := fmt.Sprintf("https://localhost:%v", port+3)
-	initialCluster := fmt.Sprintf("%v=%v", name, peerAddr)
-
-	certs := tlstest.CreateClientServerCertPairs(dataDir)
-
-	cmd := exec.Command("etcd",
-		"-name", name,
-		"-advertise-client-urls", clientAddr,
-		"-initial-advertise-peer-urls", peerAddr,
-		"-listen-client-urls", clientAddr,
-		"-listen-peer-urls", peerAddr,
-		"-initial-cluster", initialCluster,
-		"-cert-file", certs.ServerCert,
-		"-key-file", certs.ServerKey,
-		"-trusted-ca-file", certs.ClientCA,
-		"-peer-trusted-ca-file", certs.ClientCA,
-		"-peer-cert-file", certs.ServerCert,
-		"-peer-key-file", certs.ServerKey,
-		"-client-cert-auth",
-		"-data-dir", dataDir)
-
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-	err = cmd.Start()
-	if err != nil {
-		t.Fatalf("failed to start etcd: %v", err)
-	}
-
-	// Safe now to build up TLS info.
-	tlsInfo := transport.TLSInfo{
-		CertFile:      certs.ClientCert,
-		KeyFile:       certs.ClientKey,
-		TrustedCAFile: certs.ServerCA,
-	}
-
-	tlsConfig, err := tlsInfo.ClientConfig()
-
-	var cli *clientv3.Client
-	// Create client
-	start := time.Now()
-	for {
-		// Create a client to connect to the created etcd.
-		cli, err = clientv3.New(clientv3.Config{
-			Endpoints:   []string{clientAddr},
-			TLS:         tlsConfig,
-			DialTimeout: 5 * time.Second,
-		})
-		if err == nil {
-			break
-		}
-		t.Logf("error establishing client for etcd tls test: %v", err)
-		if time.Since(start) > 60*time.Second {
-			t.Fatalf("failed to start client for etcd tls test in time")
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	defer cli.Close()
-
-	// Wait until we can list "/", or timeout.
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	start = time.Now()
-	for {
-		if _, err := cli.Get(ctx, "/"); err == nil {
-			break
-		}
-		if time.Since(start) > 60*time.Second {
-			t.Fatalf("failed to start etcd daemon in time")
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-
-	stopEtcd := func() {
-		cmd.Process.Kill()
-		cmd.Wait()
-		os.RemoveAll(dataDir)
-	}
-
-	return clientAddr, &certs, stopEtcd
-}
-
-func TestEtcd2TLS(t *testing.T) {
-	// Start a single etcd in the background.
-	clientAddr, certs, stopEtcd := startEtcdWithTLS(t)
-	defer stopEtcd()
-
-	testIndex := 0
-	testRoot := fmt.Sprintf("/test-%v", testIndex)
-
-	// Create the server on the new root.
-	server, err := NewServerWithOpts(clientAddr, testRoot, certs.ClientCert, certs.ClientKey, certs.ServerCA)
-	if err != nil {
-		t.Fatalf("NewServerWithOpts failed: %v", err)
-	}
-	defer server.Close()
-
-	testCtx := context.Background()
-	testKey := "testkey"
-	testVal := "testval"
-	_, err = server.Create(testCtx, testKey, []byte(testVal))
-	if err != nil {
-		t.Fatalf("Failed to set key value pair: %v", err)
-	}
-	val, _, err := server.Get(testCtx, testKey)
-	if err != nil {
-		t.Fatalf("Failed to retrieve value at key we just set: %v", err)
-	}
-	if string(val) != testVal {
-		t.Fatalf("Value returned doesn't match %s, err: %v", testVal, err)
-	}
-}
-
 func TestEtcd2Topo(t *testing.T) {
 	// Start a single etcd in the background.
 	cmd, dataDir, clientAddr := startEtcd(t)
diff --git a/go/vt/vtgate/endtoend/deletetest/delete_test.go b/go/vt/vtgate/endtoend/deletetest/delete_test.go
index de6614ed7c..9f0cb6ceb7 100644
--- a/go/vt/vtgate/endtoend/deletetest/delete_test.go
+++ b/go/vt/vtgate/endtoend/deletetest/delete_test.go
@@ -127,7 +127,7 @@ func TestMain(m *testing.M) {
 				}},
 			}},
 		}
-		cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")}
+		cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")}
 		if err := cfg.InitSchemas("ks", schema, vschema); err != nil {
 			fmt.Fprintf(os.Stderr, "%v\n", err)
 			os.RemoveAll(cfg.SchemaDir)
diff --git a/go/vt/vtgate/endtoend/main_test.go b/go/vt/vtgate/endtoend/main_test.go
index a3b69cdb6e..da52636dce 100644
--- a/go/vt/vtgate/endtoend/main_test.go
+++ b/go/vt/vtgate/endtoend/main_test.go
@@ -171,7 +171,7 @@ func TestMain(m *testing.M) {
 				}},
 			}},
 		}
-		cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")}
+		cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")}
 		if err := cfg.InitSchemas("ks", schema, vschema); err != nil {
 			fmt.Fprintf(os.Stderr, "%v\n", err)
 			os.RemoveAll(cfg.SchemaDir)
diff --git a/go/vt/vtgate/engine/fake_primitive_test.go b/go/vt/vtgate/engine/fake_primitive_test.go
index 2c0641a979..2d00e3e735 100644
--- a/go/vt/vtgate/engine/fake_primitive_test.go
+++ b/go/vt/vtgate/engine/fake_primitive_test.go
@@ -40,12 +40,6 @@ type fakePrimitive struct {
 	log []string
 }
 
-func (f *fakePrimitive) Inputs() []Primitive {
-	return []Primitive{}
-}
-
-var _ Primitive = (*fakePrimitive)(nil)
-
 func (f *fakePrimitive) rewind() {
 	f.curResult = 0
 	f.log = nil
diff --git a/go/vt/vtgate/executor_scatter_stats_test.go b/go/vt/vtgate/executor_scatter_stats_test.go
deleted file mode 100644
index 333a06be91..0000000000
--- a/go/vt/vtgate/executor_scatter_stats_test.go
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
-Copyright 2019 The Vitess Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package vtgate
-
-import (
-	"context"
-	"testing"
-
-	"net/http/httptest"
-
-	"github.com/stretchr/testify/assert"
-
-	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
-)
-
-func TestScatterStatsWithNoScatterQuery(t *testing.T) {
-	executor, _, _, _ := createExecutorEnv()
-	session := NewSafeSession(&vtgatepb.Session{TargetString: "@master"})
-
-	_, err := executor.Execute(context.Background(), "TestExecutorResultsExceeded", session, "select * from main1", nil)
-	assert.NoError(t, err)
-
-	result, err := executor.gatherScatterStats()
-	assert.NoError(t, err)
-	assert.Equal(t, 0, len(result.Items))
-}
-
-func TestScatterStatsWithSingleScatterQuery(t *testing.T) {
-	executor, _, _, _ := createExecutorEnv()
-	session := NewSafeSession(&vtgatepb.Session{TargetString: "@master"})
-
-	_, err := executor.Execute(context.Background(), "TestExecutorResultsExceeded", session, "select * from user", nil)
-	assert.NoError(t, err)
-
-	result, err := executor.gatherScatterStats()
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(result.Items))
-}
-
-func TestScatterStatsHttpWriting(t *testing.T) {
-	executor, _, _, _ := createExecutorEnv()
-	session := NewSafeSession(&vtgatepb.Session{TargetString: "@master"})
-
-	_, err := executor.Execute(context.Background(), "TestExecutorResultsExceeded", session, "select * from user", nil)
-	assert.NoError(t, err)
-
-	_, err = executor.Execute(context.Background(), "TestExecutorResultsExceeded", session, "select * from user where Id = 15", nil)
-	assert.NoError(t, err)
-
-	_, err = executor.Execute(context.Background(), "TestExecutorResultsExceeded", session, "select * from user where Id > 15", nil)
-	assert.NoError(t, err)
-
-	query4 := "select * from user as u1 join  user as u2 on u1.Id = u2.Id"
-	_, err = executor.Execute(context.Background(), "TestExecutorResultsExceeded", session, query4, nil)
-	assert.NoError(t, err)
-
-	recorder := httptest.NewRecorder()
-	executor.WriteScatterStats(recorder)
-
-	// Here we are checking that the template was executed correctly.
-	// If it wasn't, instead of html, we'll get an error message
-	assert.Contains(t, recorder.Body.String(), query4)
-	assert.NoError(t, err)
-}
diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go
index 8a42e884bc..8767cd0928 100644
--- a/go/vt/vtgate/executor_test.go
+++ b/go/vt/vtgate/executor_test.go
@@ -890,13 +890,6 @@ func TestExecutorShow(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-
-	// Test SHOW FULL COLUMNS FROM where query has a qualifier
-	_, err = executor.Execute(context.Background(), "TestExecute", session, fmt.Sprintf("show full columns from %v.table1", KsTestUnsharded), nil)
-	if err != nil {
-		t.Errorf("Unexpected error: %v", err)
-	}
-
 	// Just test for first & last.
 	qr.Rows = [][]sqltypes.Value{qr.Rows[0], qr.Rows[len(qr.Rows)-1]}
 	wantqr = &sqltypes.Result{
diff --git a/go/vt/vtgate/planbuilder/testdata/filter_cases.txt b/go/vt/vtgate/planbuilder/testdata/filter_cases.txt
index 53fae69ba0..0130fe08b7 100644
--- a/go/vt/vtgate/planbuilder/testdata/filter_cases.txt
+++ b/go/vt/vtgate/planbuilder/testdata/filter_cases.txt
@@ -233,17 +233,13 @@
 {
   "Original": "select user_extra.id from user left join user_extra on user.id = user_extra.user_id where user_extra.user_id = 5",
   "Instructions": {
-    "Opcode": "SelectEqualUnique",
+    "Opcode": "SelectScatter",
     "Keyspace": {
       "Name": "user",
       "Sharded": true
     },
     "Query": "select user_extra.id from user left join user_extra on user.id = user_extra.user_id where user_extra.user_id = 5",
     "FieldQuery": "select user_extra.id from user left join user_extra on user.id = user_extra.user_id where 1 != 1",
-    "Vindex": "user_index",
-    "Values": [
-      5
-    ],
     "Table": "user"
   }
 }
diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.txt b/go/vt/vtgate/planbuilder/testdata/from_cases.txt
index 67943ac9df..6428be6a65 100644
--- a/go/vt/vtgate/planbuilder/testdata/from_cases.txt
+++ b/go/vt/vtgate/planbuilder/testdata/from_cases.txt
@@ -927,19 +927,35 @@
   }
 }
 
-# reference table can merge with other opcodes left to right.
+# reference table doesn't merge with other opcodes yet.
 "select ref.col from ref join user"
 {
   "Original": "select ref.col from ref join user",
   "Instructions": {
-    "Opcode": "SelectScatter",
-    "Keyspace": {
-      "Name": "user",
-      "Sharded": true
+    "Opcode": "Join",
+    "Left": {
+      "Opcode": "SelectReference",
+      "Keyspace": {
+        "Name": "user",
+        "Sharded": true
+      },
+      "Query": "select ref.col from ref",
+      "FieldQuery": "select ref.col from ref where 1 != 1",
+      "Table": "ref"
     },
-    "Query": "select ref.col from ref join user",
-    "FieldQuery": "select ref.col from ref join user where 1 != 1",
-    "Table": "user"
+    "Right": {
+      "Opcode": "SelectScatter",
+      "Keyspace": {
+        "Name": "user",
+        "Sharded": true
+      },
+      "Query": "select 1 from user",
+      "FieldQuery": "select 1 from user where 1 != 1",
+      "Table": "user"
+    },
+    "Cols": [
+      -1
+    ]
   }
 }
 
diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt
index 67a05f8aee..77cee7da74 100644
--- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt
+++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt
@@ -432,3 +432,8 @@
 
 "select func(keyspace_id) from user_index where id = :id"
 "unsupported: expression on results of a vindex function"
+
+# Multi-table unique vindex constraint on left table of left join, two levels of join, simple aggregation
+# This should work, but doesn't. See https://github.com/vitessio/vitess/issues/4772
+"select user.id, count(*) from user left join user_extra ue1 on user.id = ue1.user_id left join user_extra ue2 on ue1.user_id = ue2.user_id group by user.id"
+"unsupported: cross-shard query with aggregates"
diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go
similarity index 100%
rename from go/vt/vttablet/tabletmanager/vreplication/vcopier_flaky_test.go
rename to go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go
diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go
index 84b2365a75..599cf3304e 100644
--- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go
+++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go
@@ -92,7 +92,7 @@ func Init() (*Env, error) {
 				},
 			},
 		},
-		ExtraMyCnf: []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")},
+		ExtraMyCnf: []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")},
 		OnlyMySQL:  true,
 	}
 	te.cluster = &vttest.LocalCluster{
diff --git a/go/vt/vttest/environment.go b/go/vt/vttest/environment.go
index 130791bce4..551cc4e1d9 100644
--- a/go/vt/vttest/environment.go
+++ b/go/vt/vttest/environment.go
@@ -119,7 +119,7 @@ func GetMySQLOptions(flavor string) (string, []string, error) {
 	mycnf = append(mycnf, "config/mycnf/default-fast.cnf")
 
 	for i, cnf := range mycnf {
-		mycnf[i] = path.Join(os.Getenv("VTROOT"), cnf)
+		mycnf[i] = path.Join(os.Getenv("VTTOP"), cnf)
 	}
 
 	return flavor, mycnf, nil
@@ -139,7 +139,7 @@ func (env *LocalTestEnv) BinaryPath(binary string) string {
 func (env *LocalTestEnv) MySQLManager(mycnf []string, snapshot string) (MySQLManager, error) {
 	return &Mysqlctl{
 		Binary:    env.BinaryPath("mysqlctl"),
-		InitFile:  path.Join(os.Getenv("VTROOT"), "config/init_db.sql"),
+		InitFile:  path.Join(os.Getenv("VTTOP"), "config/init_db.sql"),
 		Directory: env.TmpPath,
 		Port:      env.PortForProtocol("mysql", ""),
 		MyCnf:     append(env.DefaultMyCnf, mycnf...),
diff --git a/java/client/src/test/java/io/vitess/client/TestEnv.java b/java/client/src/test/java/io/vitess/client/TestEnv.java
index 1b12162fcc..1e631e8bfb 100644
--- a/java/client/src/test/java/io/vitess/client/TestEnv.java
+++ b/java/client/src/test/java/io/vitess/client/TestEnv.java
@@ -72,13 +72,13 @@ public class TestEnv {
    * Get setup command to launch a cluster.
    */
   public List<String> getSetupCommand(int port) {
-    String vtRoot = System.getenv("VTROOT");
-    if (vtRoot == null) {
-      throw new RuntimeException("cannot find env variable: VTROOT");
+    String vtTop = System.getenv("VTTOP");
+    if (vtTop == null) {
+      throw new RuntimeException("cannot find env variable: VTTOP");
     }
     String schemaDir = getTestDataPath() + "/schema";
     List<String> command = new ArrayList<String>();
-    command.add(vtRoot + "/py/vttest/run_local_database.py");
+    command.add(vtTop + "/py/vttest/run_local_database.py");
     command.add("--port");
     command.add(Integer.toString(port));
     command.add("--proto_topo");
@@ -89,11 +89,11 @@ public class TestEnv {
   }
 
   public String getTestDataPath() {
-    String vtRoot = System.getenv("VTROOT");
-    if (vtRoot == null) {
-      throw new RuntimeException("cannot find env variable: VTROOT");
+    String vtTop = System.getenv("VTTOP");
+    if (vtTop == null) {
+      throw new RuntimeException("cannot find env variable: VTTOP");
     }
-    return vtRoot + "/data/test";
+    return vtTop + "/data/test";
   }
 
   public String getTestOutputPath() {
diff --git a/py/vttest/mysql_db_mysqlctl.py b/py/vttest/mysql_db_mysqlctl.py
index c1ecc8d049..f79f5c6504 100644
--- a/py/vttest/mysql_db_mysqlctl.py
+++ b/py/vttest/mysql_db_mysqlctl.py
@@ -40,7 +40,7 @@ class MySqlDBMysqlctl(mysql_db.MySqlDB):
         '-mysql_port', str(self._port),
         'init',
         '-init_db_sql_file',
-        os.path.join(os.environ['VTROOT'], 'config/init_db.sql'),
+        os.path.join(os.environ['VTTOP'], 'config/init_db.sql'),
     ]
     env = os.environ
     env['VTDATAROOT'] = self._directory
diff --git a/py/vttest/mysql_flavor.py b/py/vttest/mysql_flavor.py
index c28bf979e0..3da1570e05 100644
--- a/py/vttest/mysql_flavor.py
+++ b/py/vttest/mysql_flavor.py
@@ -23,15 +23,17 @@ import os
 import sys
 
 
-# For now, vtroot is only used in this module. If other people
+# For now, vttop is only used in this module. If other people
 # need this, we should move it to environment.
-if "VTROOT" not in os.environ:
+if "VTTOP" not in os.environ:
   sys.stderr.write(
       "ERROR: Vitess environment not set up. "
       'Please run "source dev.env" first.\n')
   sys.exit(1)
 
-vtroot = os.environ["VTROOT"]
+# vttop is the toplevel of the vitess source tree
+vttop = os.environ["VTTOP"]
+
 
 class MysqlFlavor(object):
   """Base class with default SQL statements."""
@@ -46,7 +48,7 @@ class MariaDB(MysqlFlavor):
 
   def my_cnf(self):
     files = [
-        os.path.join(vtroot, "config/mycnf/default-fast.cnf"),
+        os.path.join(vttop, "config/mycnf/default-fast.cnf"),
     ]
     return ":".join(files)
 
@@ -55,7 +57,7 @@ class MariaDB103(MysqlFlavor):
 
   def my_cnf(self):
     files = [
-      os.path.join(vtroot, "config/mycnf/default-fast.cnf"),
+      os.path.join(vttop, "config/mycnf/default-fast.cnf"),
     ]
     return ":".join(files)
 
@@ -64,7 +66,7 @@ class MySQL56(MysqlFlavor):
 
   def my_cnf(self):
     files = [
-        os.path.join(vtroot, "config/mycnf/default-fast.cnf"),
+        os.path.join(vttop, "config/mycnf/default-fast.cnf"),
     ]
     return ":".join(files)
 
@@ -73,7 +75,7 @@ class MySQL80(MysqlFlavor):
 
   def my_cnf(self):
     files = [
-        os.path.join(vtroot, "config/mycnf/default-fast.cnf"),
+        os.path.join(vttop, "config/mycnf/default-fast.cnf"),
     ]
     return ":".join(files)
 
diff --git a/test.go b/test.go
index 846cb0c875..73b75e9df3 100755
--- a/test.go
+++ b/test.go
@@ -27,7 +27,7 @@ run against a given flavor, it may take some time for the corresponding
 bootstrap image (vitess/bootstrap:<flavor>) to be downloaded.
 
 It is meant to be run from the Vitess root, like so:
-  $ go run test.go [args]
+  ~/src/vitess.io/vitess$ go run test.go [args]
 
 For a list of options, run:
   $ go run test.go --help
@@ -195,7 +195,6 @@ func (t *Test) run(dir, dataDir string) ([]byte, error) {
 	// Also try to make them use different port ranges
 	// to mitigate failures due to zombie processes.
 	cmd.Env = updateEnv(os.Environ(), map[string]string{
-		"VTROOT":      "/vt/src/vitess.io/vitess",
 		"VTDATAROOT":  dataDir,
 		"VTPORTSTART": strconv.FormatInt(int64(getPortStart(100)), 10),
 	})
@@ -371,7 +370,7 @@ func main() {
 	}
 	tests = dup
 
-	vtRoot := "."
+	vtTop := "."
 	tmpDir := ""
 	if *docker {
 		// Copy working repo to tmpDir.
@@ -388,7 +387,7 @@ func main() {
 		if out, err := exec.Command("chmod", "-R", "go=u", tmpDir).CombinedOutput(); err != nil {
 			log.Printf("Can't set permissions on temp dir %v: %v: %s", tmpDir, err, out)
 		}
-		vtRoot = tmpDir
+		vtTop = tmpDir
 	} else {
 		// Since we're sharing the working dir, do the build once for all tests.
 		log.Printf("Running make build...")
@@ -474,7 +473,7 @@ func main() {
 
 					// Run the test.
 					start := time.Now()
-					output, err := test.run(vtRoot, dataDir)
+					output, err := test.run(vtTop, dataDir)
 					duration := time.Since(start)
 
 					// Save/print test output.
diff --git a/test/backup.py b/test/backup.py
index 51b7a94eef..823e2a2ba2 100755
--- a/test/backup.py
+++ b/test/backup.py
@@ -95,7 +95,7 @@ def setUpModule():
     # Create a new init_db.sql file that sets up passwords for all users.
     # Then we use a db-credentials-file with the passwords.
     new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
-    with open(environment.vtroot + '/config/init_db.sql') as fd:
+    with open(environment.vttop + '/config/init_db.sql') as fd:
       init_db = fd.read()
     with open(new_init_db, 'w') as fd:
       fd.write(init_db)
diff --git a/test/backup_only.py b/test/backup_only.py
index 04f2db4dfd..dd9b547dac 100755
--- a/test/backup_only.py
+++ b/test/backup_only.py
@@ -95,7 +95,7 @@ def setUpModule():
     # Create a new init_db.sql file that sets up passwords for all users.
     # Then we use a db-credentials-file with the passwords.
     new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
-    with open(environment.vtroot + '/config/init_db.sql') as fd:
+    with open(environment.vttop + '/config/init_db.sql') as fd:
       init_db = fd.read()
     with open(new_init_db, 'w') as fd:
       fd.write(init_db)
diff --git a/test/backup_transform.py b/test/backup_transform.py
index 00f3ef9223..b0e665b6fa 100755
--- a/test/backup_transform.py
+++ b/test/backup_transform.py
@@ -79,7 +79,7 @@ def setUpModule():
     # Create a new init_db.sql file that sets up passwords for all users.
     # Then we use a db-credentials-file with the passwords.
     new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
-    with open(environment.vtroot + '/config/init_db.sql') as fd:
+    with open(environment.vttop + '/config/init_db.sql') as fd:
       init_db = fd.read()
     with open(new_init_db, 'w') as fd:
       fd.write(init_db)
diff --git a/test/client_test.sh b/test/client_test.sh
index 6ad69b2023..82be7996c8 100755
--- a/test/client_test.sh
+++ b/test/client_test.sh
@@ -17,10 +17,8 @@
 # This runs client tests. It used to be part of local_example,
 # but has been moved to its own test. It hijacks the public examples scripts
 
-source build.env
-
 set -xe
-cd "$VTROOT/examples/local"
+cd "$VTTOP/examples/local"
 
 CELL=test ./etcd-up.sh
 CELL=test ./vtctld-up.sh
diff --git a/test/cluster/k8s_environment.py b/test/cluster/k8s_environment.py
index d0026fb70b..0951ae58a0 100644
--- a/test/cluster/k8s_environment.py
+++ b/test/cluster/k8s_environment.py
@@ -121,7 +121,7 @@ class K8sEnvironment(base_environment.BaseEnvironment):
     if 'VITESS_NAME' not in kwargs:
       kwargs['VITESS_NAME'] = getpass.getuser()
     kwargs['TEST_MODE'] = '1'
-    self.script_dir = os.path.join(os.environ['VTROOT'], 'examples/kubernetes')
+    self.script_dir = os.path.join(os.environ['VTTOP'], 'examples/kubernetes')
     try:
       subprocess.check_output(['gcloud', 'config', 'list'])
     except OSError:
diff --git a/test/cluster/sandbox/create_schema.py b/test/cluster/sandbox/create_schema.py
index d60839bd01..198858eeff 100755
--- a/test/cluster/sandbox/create_schema.py
+++ b/test/cluster/sandbox/create_schema.py
@@ -33,7 +33,7 @@ def main():
   parser.add_option(
       '-s', '--sql_file', help='File containing sql schema',
       default=os.path.join(
-          os.environ['VTROOT'], 'examples/kubernetes/create_test_table.sql'))
+          os.environ['VTTOP'], 'examples/kubernetes/create_test_table.sql'))
   logging.getLogger().setLevel(logging.INFO)
 
   options, _ = parser.parse_args()
diff --git a/test/cluster/sandbox/kubernetes_components.py b/test/cluster/sandbox/kubernetes_components.py
index 94d3ac5e94..e67c4bf379 100755
--- a/test/cluster/sandbox/kubernetes_components.py
+++ b/test/cluster/sandbox/kubernetes_components.py
@@ -81,7 +81,7 @@ class HelmComponent(sandlet.SandletComponent):
     logging.info('Installing helm.')
     try:
       subprocess.check_output(
-          ['helm', 'install', os.path.join(os.environ['VTROOT'], 'helm/vitess'),
+          ['helm', 'install', os.path.join(os.environ['VTTOP'], 'helm/vitess'),
            '-n', self.sandbox_name, '--namespace', self.sandbox_name,
            '--replace', '--values', self.helm_config],
           stderr=subprocess.STDOUT)
diff --git a/test/cluster/sandbox/vitess_kubernetes_sandbox.py b/test/cluster/sandbox/vitess_kubernetes_sandbox.py
index 016a8d9991..734f8d1ac4 100755
--- a/test/cluster/sandbox/vitess_kubernetes_sandbox.py
+++ b/test/cluster/sandbox/vitess_kubernetes_sandbox.py
@@ -46,7 +46,7 @@ class VitessKubernetesSandbox(sandbox.Sandbox):
     """Creates a sandlet encompassing the guestbook app built on Vitess."""
     guestbook_sandlet = sandlet.Sandlet('guestbook')
     guestbook_sandlet.dependencies = ['helm']
-    template_dir = os.path.join(os.environ['VTROOT'], 'examples/kubernetes')
+    template_dir = os.path.join(os.environ['VTTOP'], 'examples/kubernetes')
     guestbook_sandlet.components.add_component(
         self.cluster_env.Port('%s-guestbook' % self.name, 80))
     for keyspace in self.app_options.keyspaces:
@@ -54,7 +54,7 @@ class VitessKubernetesSandbox(sandbox.Sandbox):
           'create_schema_%s' % keyspace['name'], self.name, 'create_schema.py',
           self.log_dir, namespace=self.name, keyspace=keyspace['name'],
           drop_table='messages', sql_file=os.path.join(
-              os.environ['VTROOT'], 'examples/kubernetes/create_test_table.sql'))
+              os.environ['VTTOP'], 'examples/kubernetes/create_test_table.sql'))
       guestbook_sandlet.components.add_component(create_schema_subprocess)
     guestbook_sandlet.components.add_component(
         kubernetes_components.KubernetesResource(
diff --git a/test/cluster/sandbox/vtctl_sandbox.py b/test/cluster/sandbox/vtctl_sandbox.py
index 3e495b414a..ef0f1d978e 100755
--- a/test/cluster/sandbox/vtctl_sandbox.py
+++ b/test/cluster/sandbox/vtctl_sandbox.py
@@ -68,7 +68,7 @@ def execute_vtctl_command(vtctl_args, namespace='default', timeout_s=180):
     # Default to trying to use kvtctl.sh if a forwarded port cannot be found.
     os.environ['VITESS_NAME'] = namespace
     vtctl_cmd_args = (
-        [os.path.join(os.environ['VTROOT'], 'examples/kubernetes/kvtctl.sh')]
+        [os.path.join(os.environ['VTTOP'], 'examples/kubernetes/kvtctl.sh')]
         + vtctl_args)
 
   start_time = time.time()
diff --git a/test/config.json b/test/config.json
index ea0d559f6f..f484e326b7 100644
--- a/test/config.json
+++ b/test/config.json
@@ -121,7 +121,7 @@
 			"Args": [],
 			"Command": [],
 			"Manual": false,
-			"Shard": 2,
+			"Shard": 5,
 			"RetryMax": 0,
 			"Tags": [
 				"worker_test"
@@ -132,7 +132,7 @@
 			"Args": [],
 			"Command": [],
 			"Manual": false,
-			"Shard": 3,
+			"Shard": 5,
 			"RetryMax": 0,
 			"Tags": [
 				"worker_test"
@@ -143,7 +143,7 @@
 			"Args": [],
 			"Command": [],
 			"Manual": false,
-			"Shard": 2,
+			"Shard": 5,
 			"RetryMax": 0,
 			"Tags": []
 		},
@@ -261,7 +261,7 @@
 			"Args": [],
 			"Command": [],
 			"Manual": false,
-			"Shard": 5,
+			"Shard": 4,
 			"RetryMax": 0,
 			"Tags": []
 		},
diff --git a/test/environment.py b/test/environment.py
index eee51cc452..ce3b58db68 100644
--- a/test/environment.py
+++ b/test/environment.py
@@ -53,6 +53,14 @@ if os.getuid() == 1:
       'ERROR: Vitess and mysqld '
       'should not be run as root.\n')
   sys.exit(1)
+if 'VTTOP' not in os.environ:
+  sys.stderr.write(
+      'ERROR: Vitess environment not set up. '
+      'Please run "source dev.env" first.\n')
+  sys.exit(1)
+
+# vttop is the toplevel of the vitess source tree
+vttop = os.environ['VTTOP']
 
 # vtroot is where everything gets installed
 vtroot = os.environ['VTROOT']
@@ -154,7 +162,7 @@ def prog_compile(name):
     return
   compiled_progs.append(name)
   logging.debug('Compiling %s', name)
-  run(['go', 'install'], cwd=os.path.join(vtroot, 'go', 'cmd', name))
+  run(['go', 'install'], cwd=os.path.join(vttop, 'go', 'cmd', name))
 
 
 # binary management: returns the full path for a binary this should
diff --git a/test/initial_sharding_multi.py b/test/initial_sharding_multi.py
index 8edad5a73e..af0b167ed2 100755
--- a/test/initial_sharding_multi.py
+++ b/test/initial_sharding_multi.py
@@ -127,7 +127,7 @@ def setUpModule():
     # Create a new init_db.sql file that sets up passwords for all users.
     # Then we use a db-credentials-file with the passwords.
     new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
-    with open(environment.vtroot + '/config/init_db.sql') as fd:
+    with open(environment.vttop + '/config/init_db.sql') as fd:
       init_db = fd.read()
     with open(new_init_db, 'w') as fd:
       fd.write(init_db)
diff --git a/test/local_example.sh b/test/local_example.sh
index 26c41abf15..f0ba0f4278 100755
--- a/test/local_example.sh
+++ b/test/local_example.sh
@@ -18,11 +18,9 @@
 # It should be kept in sync with the steps in https://vitess.io/docs/get-started/local/
 # So we can detect if a regression affecting a tutorial is introduced.
 
-source build.env
-
 set -xe
 
-cd "$VTROOT/examples/local"
+cd "$VTTOP/examples/local"
 
 ./101_initial_cluster.sh
 
diff --git a/test/tablet.py b/test/tablet.py
index ba7ff2716e..552c7d5031 100644
--- a/test/tablet.py
+++ b/test/tablet.py
@@ -55,7 +55,7 @@ def get_backup_storage_flags():
 
 
 def get_all_extra_my_cnf(extra_my_cnf):
-  all_extra_my_cnf = [environment.vtroot + '/config/mycnf/default-fast.cnf']
+  all_extra_my_cnf = [environment.vttop + '/config/mycnf/default-fast.cnf']
   flavor_my_cnf = mysql_flavor().extra_my_cnf()
   if flavor_my_cnf:
     all_extra_my_cnf.append(flavor_my_cnf)
@@ -186,12 +186,12 @@ class Tablet(object):
     """
     if use_rbr:
       if extra_my_cnf:
-        extra_my_cnf += ':' + environment.vtroot + '/config/mycnf/rbr.cnf'
+        extra_my_cnf += ':' + environment.vttop + '/config/mycnf/rbr.cnf'
       else:
-        extra_my_cnf = environment.vtroot + '/config/mycnf/rbr.cnf'
+        extra_my_cnf = environment.vttop + '/config/mycnf/rbr.cnf'
 
     if not init_db:
-      init_db = environment.vtroot + '/config/init_db.sql'
+      init_db = environment.vttop + '/config/init_db.sql'
 
     if self.use_mysqlctld:
       self.mysqlctld_process = self.mysqlctld(['-init_db_sql_file', init_db],
diff --git a/test/utils.py b/test/utils.py
index 8f645c6cbd..0a162375ff 100644
--- a/test/utils.py
+++ b/test/utils.py
@@ -1214,8 +1214,8 @@ class Vtctld(object):
     args = environment.binary_args('vtctld') + [
         '-enable_queries',
         '-cell', 'test_nj',
-        '-web_dir', environment.vtroot + '/web/vtctld',
-        '-web_dir2', environment.vtroot + '/web/vtctld2',
+        '-web_dir', environment.vttop + '/web/vtctld',
+        '-web_dir2', environment.vttop + '/web/vtctld2',
         '--log_dir', environment.vtlogroot,
         '--port', str(self.port),
         '-tablet_manager_protocol',
diff --git a/test/vtbackup.py b/test/vtbackup.py
index fecaca6612..f42831ea4e 100644
--- a/test/vtbackup.py
+++ b/test/vtbackup.py
@@ -35,7 +35,7 @@ def get_backup_storage_flags():
           os.path.join(environment.tmproot, 'backupstorage')]
 
 def get_all_extra_my_cnf(extra_my_cnf):
-    all_extra_my_cnf = [environment.vtroot + '/config/mycnf/default-fast.cnf']
+    all_extra_my_cnf = [environment.vttop + '/config/mycnf/default-fast.cnf']
     flavor_my_cnf = mysql_flavor().extra_my_cnf()
     if flavor_my_cnf:
         all_extra_my_cnf.append(flavor_my_cnf)
diff --git a/test/vtctld_web_test.py b/test/vtctld_web_test.py
index 33fd8e0086..19738abf77 100755
--- a/test/vtctld_web_test.py
+++ b/test/vtctld_web_test.py
@@ -86,12 +86,12 @@ class TestVtctldWeb(unittest.TestCase):
 
     cls.db = local_database.LocalDatabase(
         topology,
-        os.path.join(environment.vtroot, 'test/vttest_schema'),
+        os.path.join(environment.vttop, 'test/vttest_schema'),
         False, None,
-        web_dir=os.path.join(environment.vtroot, 'web/vtctld'),
+        web_dir=os.path.join(environment.vttop, 'web/vtctld'),
         default_schema_dir=os.path.join(
-            environment.vtroot, 'test/vttest_schema/default'),
-        web_dir2=os.path.join(environment.vtroot, 'web/vtctld2/app'))
+            environment.vttop, 'test/vttest_schema/default'),
+        web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
     cls.db.setup()
     cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
     utils.pause('Paused test after vtcombo was started.\n'
diff --git a/test/vttest_sample_test.py b/test/vttest_sample_test.py
index a4a7de4639..98888a9777 100755
--- a/test/vttest_sample_test.py
+++ b/test/vttest_sample_test.py
@@ -82,9 +82,9 @@ class TestMysqlctl(unittest.TestCase):
             '--port', str(port),
             '--proto_topo', text_format.MessageToString(topology,
                                                         as_one_line=True),
-            '--schema_dir', os.path.join(environment.vtroot, 'test',
+            '--schema_dir', os.path.join(environment.vttop, 'test',
                                          'vttest_schema'),
-            '--web_dir', environment.vtroot + '/web/vtctld',
+            '--web_dir', environment.vttop + '/web/vtctld',
            ]
     sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
     config = json.loads(sp.stdout.readline())
diff --git a/tools/e2e_test_cluster.sh b/tools/e2e_test_cluster.sh
index 991cd29ce5..d9ca94f655 100755
--- a/tools/e2e_test_cluster.sh
+++ b/tools/e2e_test_cluster.sh
@@ -19,9 +19,6 @@
 
 # All Go packages with test files.
 # Output per line: <full Go package name> <all _test.go files in the package>*
-
-source build.env
-
 packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/.../endtoend/... | sort)
 
 cluster_tests=$(echo "$packages_with_tests" | grep -E "go/test/endtoend" | cut -d" " -f1)
diff --git a/tools/e2e_test_race.sh b/tools/e2e_test_race.sh
index 32374a25c1..f3a31ac3af 100755
--- a/tools/e2e_test_race.sh
+++ b/tools/e2e_test_race.sh
@@ -14,11 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source build.env
-
 temp_log_file="$(mktemp --suffix .unit_test_race.log)"
 trap '[ -f "$temp_log_file" ] && rm $temp_log_file' EXIT
 
+# This can be removed once the docker images are rebuilt
+export GO111MODULE=on
+
 # Wrapper around go test -race.
 
 # This script exists because the -race test doesn't allow to distinguish
diff --git a/tools/e2e_test_runner.sh b/tools/e2e_test_runner.sh
index c581957a36..e2b9da256a 100755
--- a/tools/e2e_test_runner.sh
+++ b/tools/e2e_test_runner.sh
@@ -29,9 +29,6 @@
 # Set VT_GO_PARALLEL variable in the same way as the Makefile does.
 # We repeat this here because this script is called directly by test.go
 # and not via the Makefile.
-
-source build.env
-
 if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then
   VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE"
 fi
diff --git a/tools/unit_test_race.sh b/tools/unit_test_race.sh
index 2253cab77b..6fee1b9a9e 100755
--- a/tools/unit_test_race.sh
+++ b/tools/unit_test_race.sh
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source build.env
-
 if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then
   VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE"
 fi
diff --git a/tools/unit_test_runner.sh b/tools/unit_test_runner.sh
index 879626d7bf..382dd2f0b7 100755
--- a/tools/unit_test_runner.sh
+++ b/tools/unit_test_runner.sh
@@ -29,9 +29,6 @@
 # Set VT_GO_PARALLEL variable in the same way as the Makefile does.
 # We repeat this here because this script is called directly by test.go
 # and not via the Makefile.
-
-source build.env
-
 if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then
   VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE"
 fi
@@ -55,8 +52,6 @@ if [ $? -ne 0 ]; then
   exit 1
 fi
 
-echo '# Flaky tests (3 attempts permitted)'
-
 # Run flaky tests sequentially. Retry when necessary.
 for pkg in $flaky_tests; do
   max_attempts=3