Repository: duckdb/duckdb
File: extension/core_functions/scalar/blob/encode.cpp (C++)

Code before:
	template <class INPUT_TYPE, class RESULT_TYPE>
	static RESULT_TYPE Operation(INPUT_TYPE input) {
		auto input_data = input.GetData();
		auto input_length = input.GetSize();
		if (Utf8Proc::Analyze(input_data, input_length) == UnicodeType::INVALID) {
			throw ConversionException(
			    "Failure in decode: could not convert blob to UTF8 string, the blob contained invalid UTF8 characters");
		}
		return input;
	}
};

void UnaryDecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
	// decode is also a nop cast, but requires verification if the provided string is actually
	UnaryExecutor::Execute<string_t, string_t, UnaryBlobDecodeOperator>(args.data[0], result, args.size());
	StringVector::AddHeapReference(result, args.data[0]);
}

void BinaryDecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
	// decode is also a nop cast, but requires verification if the provided string is actually
	BinaryExecutor::ExecuteWithNulls<string_t, string_t, string_t>(
	    args.data[0], args.data[1], result, args.size(),
	    [&](string_t input, string_t error_option, ValidityMask &mask, idx_t idx) {
		    auto input_data = input.GetDataWriteable();
		    auto input_length = input.GetSize();
		    auto const error_behavior = GetDecodeErrorBehavior(error_option.GetData());
		    switch (error_behavior) {
		    case DecodeErrorBehavior::ABORT:
			    if (Utf8Proc::Analyze(input_data, input_length) == UnicodeType::INVALID) {
				    throw ConversionException("Failure in decode: could not convert blob to UTF8 string, the blob "
				                              "contained invalid UTF8 characters");
			    }
			    return input;

		    case DecodeErrorBehavior::STRICT:
			    if (Utf8Proc::Analyze(input_data, input_length) == UnicodeType::INVALID) {
				    mask.SetInvalid(idx);
				    return StringVector::EmptyString(result, input_length);
			    } else {
				    return input;
			    }
		    case DecodeErrorBehavior::REPLACE:
			    if (Utf8Proc::Analyze(input_data, input_length) == UnicodeType::INVALID) {
				    Utf8Proc::MakeValid(input_data, input_length);
			    }
			    return input;
		    }
	    });
	StringVector::AddHeapReference(result, args.data[0]);
}


Review comment (suggestion):
Python has as behavior that the error string is only parsed if the unicode is actually invalid. I think that makes sense, i.e. we can do this:


```cpp
if (Utf8Proc::Analyze(input_data, input_length) != UnicodeType::INVALID) {
    return input;
}
// handle error
auto const error_behavior = GetDecodeErrorBehavior(error_option.GetData());
```

Code after:
			    "contained invalid UTF8 characters. \n"
			    "Use try(decode(BLOB)) to return NULL and continue instead of returning an error. "
			    "Specify decode(BLOB, 'replace') to replace invalid characters with '?'. "
			    "Specify decode(BLOB, 'ignore') to remove invalid characters when encountered.");
		}
		return input;
	}
};

void UnaryDecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
	// decode is also a nop cast, but requires verification if the provided string is actually
	UnaryExecutor::Execute<string_t, string_t, UnaryBlobDecodeOperator>(args.data[0], result, args.size());
	StringVector::AddHeapReference(result, args.data[0]);
}

void BinaryDecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
	// decode is also a nop cast, but requires verification if the provided string is actually
	BinaryExecutor::Execute<string_t, string_t, string_t>(
	    args.data[0], args.data[1], result, args.size(), [&](string_t input, string_t error_option) {
		    auto input_data = input.GetDataWriteable();
		    auto input_length = input.GetSize();

		    if (Utf8Proc::Analyze(input_data, input_length) != UnicodeType::INVALID) {
			    return input;
		    }
		    auto const error_behavior = GetDecodeErrorBehavior(error_option);

		    switch (error_behavior) {
		    case DecodeErrorBehavior::REPLACE:
			    Utf8Proc::MakeValid(input_data, input_length);
			    return input;

		    case DecodeErrorBehavior::IGNORE: {
			    auto new_str = Utf8Proc::RemoveInvalid(input_data, input_length);
			    auto target = StringVector::EmptyString(result, new_str.size());
			    auto output = target.GetDataWriteable();
			    memcpy(output, new_str.data(), new_str.size());
			    target.Finalize();
			    return target;
		    }

		    case DecodeErrorBehavior::STRICT:
			    throw ConversionException(
			        "Failure in decode: could not convert blob to UTF8 string, the blob "
			        "contained invalid UTF8 characters. \n"
			        "Use try(decode(BLOB)) to return NULL and continue instead of returning an error. "
			        "Specify decode(BLOB, 'replace') to replace invalid characters with '?'. "
			        "Specify decode(BLOB, 'ignore') to remove invalid characters when encountered.");

		    default:
			    throw InternalException("Unimplemented decode error behavior");