Repository: snowflakedb/gosnowflake
Pull request #474 (2021-08-30 22:47:39)

SNOW-444264 Make PUT overwrite default
### Description
Make overwrite = true the default for PUT operations unless otherwise specified.

### Checklist
- [x] Code compiles correctly
- [x] Run ``make fmt`` to fix inconsistent formats
- [x] Run ``make lint`` to get lint errors and fix all of them
- [x] Created tests which fail without the change (if possible)
- [x] All tests passing
- [x] Extended the README / documentation, if necessary



Patch:
diff --git a/connection_util.go b/connection_util.go
index 554ee6f..604a1a1 100644
--- a/connection_util.go
+++ b/connection_util.go
@@ -89,8 +89,8 @@ func (sc *snowflakeConn) processFileTransfer(
 	if op := getFileTransferOptions(ctx); op != nil {
 		sfa.options = op
 	}
-	if sfa.options.multiPartThreshold == 0 {
-		sfa.options.multiPartThreshold = dataSizeThreshold
+	if sfa.options.MultiPartThreshold == 0 {
+		sfa.options.MultiPartThreshold = dataSizeThreshold
 	}
 	if err := sfa.execute(); err != nil {
 		return nil, err
diff --git a/file_transfer_agent.go b/file_transfer_agent.go
index 426a88b..7b0519d 100644
--- a/file_transfer_agent.go
+++ b/file_transfer_agent.go
@@ -82,14 +82,14 @@ func (rs resultStatus) isSet() bool {
 // files transfers such as PUT/GET
 type SnowflakeFileTransferOptions struct {
 	showProgressBar    bool
-	raisePutGetError   bool
-	multiPartThreshold int64
+	RaisePutGetError   bool
+	MultiPartThreshold int64
 
 	/* streaming PUT */
 	compressSourceFromStream bool
 
 	/* PUT */
-	forcePutOverwrite       bool
+	DisablePutOverwrite     bool
 	putCallback             *snowflakeProgressPercentage
 	putAzureCallback        *snowflakeProgressPercentage
 	putCallbackOutputStream *io.Writer
@@ -171,8 +171,8 @@ func (sfa *snowflakeFileTransferAgent) execute() error {
 		meta.sfa = sfa
 		meta.options = sfa.options
 		if sfa.stageLocationType != local {
-			sizeThreshold := sfa.options.multiPartThreshold
-			meta.options.multiPartThreshold = sizeThreshold
+			sizeThreshold := sfa.options.MultiPartThreshold
+			meta.options.MultiPartThreshold = sizeThreshold
 			if meta.srcFileSize > sizeThreshold {
 				meta.parallel = sfa.parallel
 				largeFileMetas = append(largeFileMetas, meta)
@@ -259,7 +259,7 @@ func (sfa *snowflakeFileTransferAgent) parseCommand() error {
 	if sfa.data.Parallel != 0 {
 		sfa.parallel = sfa.data.Parallel
 	}
-	sfa.overwrite = sfa.data.Overwrite || sfa.options.forcePutOverwrite
+	sfa.overwrite = !sfa.options.DisablePutOverwrite
 	sfa.stageLocationType = cloudType(strings.ToUpper(sfa.data.StageInfo.LocationType))
 	sfa.stageInfo = &sfa.data.StageInfo
 	sfa.presignedURLs = make([]string, 0)
@@ -981,7 +981,7 @@ func (sfa *snowflakeFileTransferAgent) result() (*execResponse, error) {
 				errorDetails := meta.errorDetails
 				srcFileSize := meta.srcFileSize
 				dstFileSize := meta.dstFileSize
-				if sfa.options.raisePutGetError && errorDetails != nil {
+				if sfa.options.RaisePutGetError && errorDetails != nil {
 					return nil, (&SnowflakeError{
 						Number:   ErrFailedToUploadToStage,
 						SQLState: sfa.data.SQLState,
@@ -1046,7 +1046,7 @@ func (sfa *snowflakeFileTransferAgent) result() (*execResponse, error) {
 			for _, meta := range sfa.results {
 				dstFileSize := meta.dstFileSize
 				errorDetails := meta.errorDetails
-				if sfa.options.raisePutGetError && errorDetails != nil {
+				if sfa.options.RaisePutGetError && errorDetails != nil {
 					return nil, (&SnowflakeError{
 						Number:   ErrFailedToDownloadFromStage,
 						SQLState: sfa.data.SQLState,
diff --git a/query.go b/query.go
index 54a0391..a917c93 100644
--- a/query.go
+++ b/query.go
@@ -112,7 +112,6 @@ type execResponseData struct {
 	Parallel                int64                 `json:"parallel,omitempty"`
 	Threshold               int64                 `json:"threshold,omitempty"`
 	AutoCompress            bool                  `json:"autoCompress,omitempty"`
-	Overwrite               bool                  `json:"overwrite,omitempty"`
 	SourceCompression       string                `json:"sourceCompression,omitempty"`
 	ShowEncryptionParameter bool                  `json:"clientShowEncryptionParameter,omitempty"`
 	EncryptionMaterial      encryptionWrapper     `json:"encryptionMaterial,omitempty"`
diff --git a/storage_client.go b/storage_client.go
index 94caa85..9f45f49 100644
--- a/storage_client.go
+++ b/storage_client.go
@@ -95,7 +95,7 @@ func (rsu *remoteStorageUtil) uploadOneFile(meta *fileMetadata) error {
 			}
 		}
 		if meta.overwrite || meta.resStatus == notFoundFile {
-			utilClass.uploadFile(dataFile, meta, encryptMeta, maxConcurrency, meta.options.multiPartThreshold)
+			utilClass.uploadFile(dataFile, meta, encryptMeta, maxConcurrency, meta.options.MultiPartThreshold)
 		}
 		if meta.resStatus == uploaded || meta.resStatus == renewToken || meta.resStatus == renewPresignedURL {
 			return nil


Test patch:
diff --git a/put_get_test.go b/put_get_test.go
index df358d9..64bbfa7 100644
--- a/put_get_test.go
+++ b/put_get_test.go
@@ -44,7 +44,7 @@ func TestPutError(t *testing.T) {
 	fta := &snowflakeFileTransferAgent{
 		data: data,
 		options: &SnowflakeFileTransferOptions{
-			raisePutGetError: false,
+			RaisePutGetError: false,
 		},
 	}
 	if err := fta.execute(); err != nil {
@@ -57,7 +57,7 @@ func TestPutError(t *testing.T) {
 	fta = &snowflakeFileTransferAgent{
 		data: data,
 		options: &SnowflakeFileTransferOptions{
-			raisePutGetError: true,
+			RaisePutGetError: true,
 		},
 	}
 	if err := fta.execute(); err != nil {
@@ -269,8 +269,12 @@ func TestPutOverwrite(t *testing.T) {
 		}
 
 		f, _ = os.Open(testData)
+		ctx := WithFileTransferOptions(context.Background(),
+			&SnowflakeFileTransferOptions{
+				DisablePutOverwrite: true,
+			})
 		rows = dbt.mustQueryContext(
-			WithFileStream(context.Background(), f),
+			WithFileStream(ctx, f),
 			fmt.Sprintf("put 'file://%v' @~/test_put_overwrite",
 				strings.ReplaceAll(testData, "\\", "\\\\")))
 		f.Close()
diff --git a/s3_util_test.go b/s3_util_test.go
index 0e59480..d24f987 100644
--- a/s3_util_test.go
+++ b/s3_util_test.go
@@ -72,7 +72,7 @@ func TestUploadOneFileToS3WSAEConnAborted(t *testing.T) {
 		srcFileName:       path.Join(dir, "/test_data/put_get_1.txt"),
 		overwrite:         true,
 		options: &SnowflakeFileTransferOptions{
-			multiPartThreshold: dataSizeThreshold,
+			MultiPartThreshold: dataSizeThreshold,
 		},
 		mockUploader: mockUploadObjectAPI(func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 			return nil, &smithy.GenericAPIError{
@@ -137,7 +137,7 @@ func TestUploadOneFileToS3ConnReset(t *testing.T) {
 		srcFileName:       path.Join(dir, "/test_data/put_get_1.txt"),
 		overwrite:         true,
 		options: &SnowflakeFileTransferOptions{
-			multiPartThreshold: dataSizeThreshold,
+			MultiPartThreshold: dataSizeThreshold,
 		},
 		mockUploader: mockUploadObjectAPI(func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 			return nil, &smithy.GenericAPIError{
@@ -184,7 +184,7 @@ func TestUploadFileWithS3UploadFailedError(t *testing.T) {
 		srcFileName:       path.Join(dir, "/test_data/put_get_1.txt"),
 		overwrite:         true,
 		options: &SnowflakeFileTransferOptions{
-			multiPartThreshold: dataSizeThreshold,
+			MultiPartThreshold: dataSizeThreshold,
 		},
 		mockUploader: mockUploadObjectAPI(func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
 			return nil, &smithy.GenericAPIError{