Repository: vitessio/vitess
Pull request #6911 (2020-10-20 07:24:35)

Fixes a bug in Load From statement
Added end to end test folder to config.json and fixes a bug in load from statement's permissions required.


Patch:
diff --git a/bootstrap.sh b/bootstrap.sh
index ff8f702c0a..c539809bc8 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -27,9 +27,6 @@ BUILD_JAVA=${BUILD_JAVA:-1}
 BUILD_CONSUL=${BUILD_CONSUL:-1}
 BUILD_CHROME=${BUILD_CHROME:-1}
 
-VITESS_RESOURCES_DOWNLOAD_BASE_URL="https://github.com/vitessio/vitess-resources/releases/download"
-VITESS_RESOURCES_RELEASE="v1.0"
-VITESS_RESOURCES_DOWNLOAD_URL="${VITESS_RESOURCES_DOWNLOAD_BASE_URL}/${VITESS_RESOURCES_RELEASE}"
 #
 # 0. Initialization and helper methods.
 #
@@ -41,7 +38,7 @@ VITESS_RESOURCES_DOWNLOAD_URL="${VITESS_RESOURCES_DOWNLOAD_BASE_URL}/${VITESS_RE
 # If the installation is successful, it puts the installed version string into
 # the $dist/.installed_version file. If the version has not changed, bootstrap
 # will skip future installations.
-install_dep() {
+function install_dep() {
   if [[ $# != 4 ]]; then
     fail "install_dep function requires exactly 4 parameters (and not $#). Parameters: $*"
   fi
@@ -88,12 +85,12 @@ install_dep() {
 # We should not use the arch command, since it is not reliably
 # available on macOS or some linuxes:
 # https://www.gnu.org/software/coreutils/manual/html_node/arch-invocation.html
-get_arch() {
+function get_arch() {
   uname -m
 }
 
 # Install protoc.
-install_protoc() {
+function install_protoc() {
   local version="$1"
   local dist="$2"
 
@@ -108,23 +105,21 @@ install_protoc() {
       *)   echo "ERROR: unsupported architecture"; exit 1;;
   esac
 
-  # This is how we'd download directly from source:
-  # wget https://github.com/protocolbuffers/protobuf/releases/download/v$version/protoc-$version-$platform-${target}.zip
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/protoc-$version-$platform-${target}.zip"
+  wget https://github.com/protocolbuffers/protobuf/releases/download/v$version/protoc-$version-$platform-${target}.zip
   unzip "protoc-$version-$platform-${target}.zip"
   ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc"
 }
+protoc_ver=3.6.1
+install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc
 
 
 # Install Zookeeper.
-install_zookeeper() {
+function install_zookeeper() {
   local version="$1"
   local dist="$2"
 
   zk="zookeeper-$version"
-  # This is how we'd download directly from source:
-  # wget "https://archive.apache.org/dist/zookeeper/$zk/$zk.tar.gz"
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/${zk}.tar.gz"
+  wget "https://archive.apache.org/dist/zookeeper/$zk/$zk.tar.gz"
   tar -xzf "$zk.tar.gz"
   ant -f "$zk/build.xml" package
   ant -f "$zk/zookeeper-contrib/zookeeper-contrib-fatjar/build.xml" jar
@@ -134,9 +129,13 @@ install_zookeeper() {
   rm -rf "$zk" "$zk.tar.gz"
 }
 
+zk_ver=${ZK_VERSION:-3.4.14}
+if [ "$BUILD_JAVA" == 1 ] ; then
+  install_dep "Zookeeper" "$zk_ver" "$VTROOT/dist/vt-zookeeper-$zk_ver" install_zookeeper
+fi
 
 # Download and install etcd, link etcd binary into our root.
-install_etcd() {
+function install_etcd() {
   local version="$1"
   local dist="$2"
 
@@ -151,12 +150,10 @@ install_etcd() {
       *)   echo "ERROR: unsupported architecture"; exit 1;;
   esac
 
+  download_url=https://github.com/coreos/etcd/releases/download
   file="etcd-${version}-${platform}-${target}.${ext}"
 
-  # This is how we'd download directly from source:
-  # download_url=https://github.com/etcd-io/etcd/releases/download
-  # wget "$download_url/$version/$file"
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/${file}"
+  wget "$download_url/$version/$file"
   if [ "$ext" = "tar.gz" ]; then
     tar xzf "$file"
   else
@@ -166,10 +163,11 @@ install_etcd() {
   ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd"
   ln -snf "$dist/etcd-${version}-${platform}-${target}/etcdctl" "$VTROOT/bin/etcdctl"
 }
+command -v etcd && echo "etcd already installed" || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
 
 
 # Download and install k3s, link k3s binary into our root
-install_k3s() {
+function install_k3s() {
   local version="$1"
   local dist="$2"
 
@@ -184,20 +182,18 @@ install_k3s() {
       *)   echo "WARNING: unsupported architecture, the k8s topology will not be available for local examples."; return;;
   esac
 
+  download_url=https://github.com/rancher/k3s/releases/download
   file="k3s${target}"
 
   local dest="$dist/k3s${target}-${version}-${platform}"
-  # This is how we'd download directly from source:
-  # download_url=https://github.com/rancher/k3s/releases/download
-  # wget -O  $dest "$download_url/$version/$file"
-  wget -O  $dest "${VITESS_RESOURCES_DOWNLOAD_URL}/$file-$version"
+  wget -O  $dest "$download_url/$version/$file"
   chmod +x $dest
   ln -snf  $dest "$VTROOT/bin/k3s"
 }
-
+command -v  k3s || install_dep "k3s" "v1.0.0" "$VTROOT/dist/k3s" install_k3s
 
 # Download and install consul, link consul binary into our root.
-install_consul() {
+function install_consul() {
   local version="$1"
   local dist="$2"
 
@@ -212,17 +208,18 @@ install_consul() {
       *)   echo "ERROR: unsupported architecture"; exit 1;;
   esac
 
-  # This is how we'd download directly from source:
-  # download_url=https://releases.hashicorp.com/consul
-  # wget "${download_url}/${version}/consul_${version}_${platform}_${target}.zip"
-  wget "${VITESS_RESOURCES_DOWNLOAD_URL}/consul_${version}_${platform}_${target}.zip"
+  download_url=https://releases.hashicorp.com/consul
+  wget "${download_url}/${version}/consul_${version}_${platform}_${target}.zip"
   unzip "consul_${version}_${platform}_${target}.zip"
   ln -snf "$dist/consul" "$VTROOT/bin/consul"
 }
 
+if [ "$BUILD_CONSUL" == 1 ] ; then
+  install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
+fi
 
 # Download chromedriver
-install_chromedriver() {
+function install_chromedriver() {
   local version="$1"
   local dist="$2"
 
@@ -234,12 +231,12 @@ install_chromedriver() {
   if [ "$(arch)" == "aarch64" ] ; then
       os=$(cat /etc/*release | grep "^ID=" | cut -d '=' -f 2)
       case $os in
-        ubuntu|debian)
-          sudo apt-get update -y && sudo apt install -y --no-install-recommends unzip libglib2.0-0 libnss3 libx11-6
-        ;;
-        centos|fedora)
-          sudo yum update -y && yum install -y libX11 unzip wget
-        ;;
+          ubuntu|debian)
+              sudo apt-get update -y && sudo apt install -y --no-install-recommends unzip libglib2.0-0 libnss3 libx11-6
+	      ;;
+	  centos|fedora)
+	      sudo yum update -y && yum install -y libX11 unzip wget
+	      ;;
       esac
       echo "For Arm64, using prebuilt binary from electron (https://github.com/electron/electron/) of version 76.0.3809.126"
       wget https://github.com/electron/electron/releases/download/v6.0.3/chromedriver-v6.0.3-linux-arm64.zip
@@ -252,35 +249,9 @@ install_chromedriver() {
   fi
 }
 
-install_all() {
-  # protoc
-  protoc_ver=3.6.1
-  install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc
-
-  # zk
-  zk_ver=${ZK_VERSION:-3.4.14}
-  if [ "$BUILD_JAVA" == 1 ] ; then
-    install_dep "Zookeeper" "$zk_ver" "$VTROOT/dist/vt-zookeeper-$zk_ver" install_zookeeper
-  fi
-
-  # etcd
-  command -v etcd && echo "etcd already installed" || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
-
-  # k3s
-  command -v  k3s || install_dep "k3s" "v1.0.0" "$VTROOT/dist/k3s" install_k3s
-
-  # consul
-  if [ "$BUILD_CONSUL" == 1 ] ; then
-    install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
-  fi
-
-  # chromedriver
-  if [ "$BUILD_CHROME" == 1 ] ; then
-    install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver
-  fi
-
-  echo
-  echo "bootstrap finished - run 'make build' to compile"
-}
+if [ "$BUILD_CHROME" == 1 ] ; then
+	install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver
+fi
 
-install_all
+echo
+echo "bootstrap finished - run 'make build' to compile"
diff --git a/go/vt/vttablet/tabletserver/planbuilder/permission.go b/go/vt/vttablet/tabletserver/planbuilder/permission.go
index 3bf2789650..dc0e2fb058 100644
--- a/go/vt/vttablet/tabletserver/planbuilder/permission.go
+++ b/go/vt/vttablet/tabletserver/planbuilder/permission.go
@@ -55,7 +55,7 @@ func BuildPermissions(stmt sqlparser.Statement) []Permission {
 		}
 	case *sqlparser.OtherAdmin:
 		// no op
-	case *sqlparser.Begin, *sqlparser.Commit, *sqlparser.Rollback:
+	case *sqlparser.Begin, *sqlparser.Commit, *sqlparser.Rollback, *sqlparser.Load:
 		// no op
 	case *sqlparser.Savepoint, *sqlparser.Release, *sqlparser.SRollback:
 		// no op
diff --git a/go/vt/vttablet/tabletserver/planbuilder/plan.go b/go/vt/vttablet/tabletserver/planbuilder/plan.go
index 4c34bdf53f..3479bc0222 100644
--- a/go/vt/vttablet/tabletserver/planbuilder/plan.go
+++ b/go/vt/vttablet/tabletserver/planbuilder/plan.go
@@ -92,6 +92,7 @@ var planName = []string{
 	"Release",
 	"RollbackSavepoint",
 	"ShowTables",
+	"Load",
 }
 
 func (pt PlanType) String() string {
diff --git a/go/vt/vttablet/tabletserver/query_executor.go b/go/vt/vttablet/tabletserver/query_executor.go
index 2cf8235b8b..2e812967fb 100644
--- a/go/vt/vttablet/tabletserver/query_executor.go
+++ b/go/vt/vttablet/tabletserver/query_executor.go
@@ -195,7 +195,7 @@ func (qre *QueryExecutor) execAsTransaction(f func(conn *StatefulConnection) (*s
 
 func (qre *QueryExecutor) txConnExec(conn *StatefulConnection) (*sqltypes.Result, error) {
 	switch qre.plan.PlanID {
-	case planbuilder.PlanInsert, planbuilder.PlanUpdate, planbuilder.PlanDelete, planbuilder.PlanLoad:
+	case planbuilder.PlanInsert, planbuilder.PlanUpdate, planbuilder.PlanDelete:
 		return qre.txFetch(conn, true)
 	case planbuilder.PlanInsertMessage:
 		qre.bindVars["#time_now"] = sqltypes.Int64BindVariable(time.Now().UnixNano())
@@ -222,6 +222,8 @@ func (qre *QueryExecutor) txConnExec(conn *StatefulConnection) (*sqltypes.Result
 		return qr, nil
 	case planbuilder.PlanDDL:
 		return qre.execDDL(conn)
+	case planbuilder.PlanLoad:
+		return qre.execLoad(conn)
 	}
 	return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "%s unexpected plan type", qre.plan.PlanID.String())
 }
@@ -393,6 +395,14 @@ func (qre *QueryExecutor) execDDL(conn *StatefulConnection) (*sqltypes.Result, e
 	return result, nil
 }
 
+func (qre *QueryExecutor) execLoad(conn *StatefulConnection) (*sqltypes.Result, error) {
+	result, err := qre.execSQL(conn, qre.query, true)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
 // BeginAgain commits the existing transaction and begins a new one
 func (*QueryExecutor) BeginAgain(ctx context.Context, dc *StatefulConnection) error {
 	if dc.IsClosed() || dc.TxProperties().Autocommit {


Test patch:
diff --git a/go/test/endtoend/vtgate/unsharded/main_test.go b/go/test/endtoend/vtgate/unsharded/main_test.go
index be79923423..969d461cb8 100644
--- a/go/test/endtoend/vtgate/unsharded/main_test.go
+++ b/go/test/endtoend/vtgate/unsharded/main_test.go
@@ -70,7 +70,7 @@ CREATE TABLE t1 (
                     "name": "c4",
                     "type": "VARCHAR"
                 }
-            ],
+            ]
         }
     }
 }
@@ -111,6 +111,8 @@ func TestMain(m *testing.M) {
 }
 
 func TestSelectIntoAndLoadFrom(t *testing.T) {
+	// Test is skipped because it requires secure-file-priv variable to be set to not NULL or empty.
+	t.Skip()
 	defer cluster.PanicHandler(t)
 	ctx := context.Background()
 	vtParams := mysql.ConnParams{
@@ -123,33 +125,42 @@ func TestSelectIntoAndLoadFrom(t *testing.T) {
 
 	defer exec(t, conn, `delete from t1`)
 	exec(t, conn, `insert into t1(c1, c2, c3, c4) values (300,100,300,'abc')`)
-
-	exec(t, conn, `select * from t1 into outfile 'x.txt'`)
-	execAssertError(t, conn, `load data infile 'x.txt' into table t1`, "ERROR 1062 (23000): Duplicate entry '300' for key 'PRIMARY'")
+	res := exec(t, conn, `select @@secure_file_priv;`)
+	directory := res.Rows[0][0].ToString()
+	query := `select * from t1 into outfile '` + directory + `x.txt'`
+	exec(t, conn, query)
+	defer os.Remove(directory + `x.txt`)
+	query = `load data infile '` + directory + `x.txt' into table t1`
+	execAssertError(t, conn, query, "Duplicate entry '300' for key 'PRIMARY'")
 	exec(t, conn, `delete from t1`)
-	exec(t, conn, `load data infile 'x.txt' into table t1`)
+	exec(t, conn, query)
 	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)]]`)
-	exec(t, conn, `select * from t1 into dumpfile 'x1.txt'`)
-	exec(t, conn, `select * from t1 into outfile 'x2.txt' Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`)
-	exec(t, conn, `load data infile 'x.txt' into replace table t1 Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`)
+	query = `select * from t1 into dumpfile '` + directory + `x1.txt'`
+	exec(t, conn, query)
+	defer os.Remove(directory + `x1.txt`)
+	query = `select * from t1 into outfile '` + directory + `x2.txt' Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`
+	exec(t, conn, query)
+	defer os.Remove(directory + `x2.txt`)
+	query = `load data infile '` + directory + `x2.txt' replace into table t1 Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`
+	exec(t, conn, query)
 	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)]]`)
 }
 
-func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
+func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { //nolint:golint,unused
 	t.Helper()
 	qr, err := conn.ExecuteFetch(query, 1000, true)
 	require.NoError(t, err)
 	return qr
 }
 
-func execAssertError(t *testing.T, conn *mysql.Conn, query string, errorString string) {
+func execAssertError(t *testing.T, conn *mysql.Conn, query string, errorString string) { //nolint:golint,unused
 	t.Helper()
 	_, err := conn.ExecuteFetch(query, 1000, true)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), errorString)
 }
 
-func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
+func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { //nolint:golint,unused
 	t.Helper()
 	qr := exec(t, conn, query)
 	got := fmt.Sprintf("%v", qr.Rows)
diff --git a/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt b/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt
index 44d4eea589..2b9d31892f 100644
--- a/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt
+++ b/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt
@@ -817,3 +817,10 @@ options:PassthroughDMLs
   "PlanID": "OtherRead",
   "TableName": ""
 }
+
+# load data
+"load data infile 'x.txt' into table a"
+{
+    "PlanID":"Load",
+    "TableName":""
+}
diff --git a/test/config.json b/test/config.json
index ede36477a0..487b3a0e3b 100644
--- a/test/config.json
+++ b/test/config.json
@@ -515,6 +515,15 @@
 			"RetryMax": 0,
 			"Tags": []
 		},
+		"vtgate_unsharded": {
+			"File": "unused.go",
+			"Args": ["vitess.io/vitess/go/test/endtoend/vtgate/unsharded"],
+			"Command": [],
+			"Manual": false,
+			"Shard": 17,
+			"RetryMax": 0,
+			"Tags": []
+		},
 		"vtgate_vschema": {
 			"File": "unused.go",
 			"Args": ["vitess.io/vitess/go/test/endtoend/vtgate/vschema"],