Repository: hyperledger/firefly
Pull request #207 (2021-09-25 06:00:40)

Allow null value with blob data, and allow "none" validator to tag datatype on unstructured data
Fix for #198 

- Fixes handling of `null` on `data.value` - including as part of a FORM post upload
  - Stored as the bytes `null` in the database (as today)
  - Does not contribute to the hash, when there is a blob
  - Returned as `null` in the JSON payload on `GET`
- Adds a new `validator` of `none`
  - Must be set explicitly - `json` remains the default
  - Disabled any checking of the `datatype.name`/`datatype.value` when set
  - Allows you to just use the `datatype` for tagging purposes, without validating the `value` JSON payload itself


Patch:
diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml
index bc2bb9a48..f16189fde 100644
--- a/docs/swagger/swagger.yaml
+++ b/docs/swagger/swagger.yaml
@@ -665,6 +665,7 @@ paths:
                 validator:
                   enum:
                   - json
+                  - none
                   - definition
                   type: string
                 value:
@@ -1639,6 +1640,7 @@ paths:
                 validator:
                   enum:
                   - json
+                  - none
                   - definition
                   type: string
                 value:
diff --git a/internal/apiserver/route_post_data.go b/internal/apiserver/route_post_data.go
index c936cf510..322966faa 100644
--- a/internal/apiserver/route_post_data.go
+++ b/internal/apiserver/route_post_data.go
@@ -55,6 +55,16 @@ var postData = &oapispec.Route{
 	},
 	FormUploadHandler: func(r *oapispec.APIRequest) (output interface{}, err error) {
 		data := &fftypes.DataRefOrValue{}
+		validator := r.FP["validator"]
+		if len(validator) > 0 {
+			data.Validator = fftypes.ValidatorType(validator)
+		}
+		if r.FP["datatype.name"] != "" {
+			data.Datatype = &fftypes.DatatypeRef{
+				Name:    r.FP["datatype.name"],
+				Version: r.FP["datatype.version"],
+			}
+		}
 		metadata := r.FP["metadata"]
 		if len(metadata) > 0 {
 			// The metadata might be JSON, or just a simple string. Try to unmarshal and see
@@ -63,13 +73,6 @@ var postData = &oapispec.Route{
 				metadata = fmt.Sprintf(`"%s"`, metadata)
 			}
 			data.Value = fftypes.Byteable(metadata)
-			data.Validator = fftypes.ValidatorType(r.FP["validator"])
-			if r.FP["datatype.name"] != "" {
-				data.Datatype = &fftypes.DatatypeRef{
-					Name:    r.FP["datatype.name"],
-					Version: r.FP["datatype.version"],
-				}
-			}
 		}
 		output, err = r.Or.Data().UploadBLOB(r.Ctx, r.PP["ns"], data, r.Part, strings.EqualFold(r.FP["autometa"], "true"))
 		return output, err
diff --git a/internal/data/data_manager.go b/internal/data/data_manager.go
index 82d36e3b5..3a05a4aa6 100644
--- a/internal/data/data_manager.go
+++ b/internal/data/data_manager.go
@@ -157,7 +157,7 @@ func (dm *dataManager) GetMessageData(ctx context.Context, msg *fftypes.Message,
 
 func (dm *dataManager) ValidateAll(ctx context.Context, data []*fftypes.Data) (valid bool, err error) {
 	for _, d := range data {
-		if d.Datatype != nil {
+		if d.Datatype != nil && d.Validator != fftypes.ValidatorTypeNone {
 			v, err := dm.getValidatorForDatatype(ctx, d.Namespace, d.Validator, d.Datatype)
 			if err != nil {
 				return false, err
diff --git a/pkg/fftypes/byteable.go b/pkg/fftypes/byteable.go
index 30b9fe87c..1f1112f30 100644
--- a/pkg/fftypes/byteable.go
+++ b/pkg/fftypes/byteable.go
@@ -21,9 +21,14 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 
+	"github.com/hyperledger/firefly/internal/i18n"
 	"github.com/hyperledger/firefly/internal/log"
 )
 
+const (
+	nullString = "null"
+)
+
 // Byteable uses raw encode/decode to preserve field order, and can handle any types of field.
 // It validates the JSON can be unmarshalled, but does not change the order.
 // It does however trim out whitespace
@@ -41,7 +46,7 @@ func (h *Byteable) UnmarshalJSON(b []byte) error {
 
 func (h Byteable) MarshalJSON() ([]byte, error) {
 	if h == nil {
-		return []byte("null"), nil
+		return []byte(nullString), nil
 	}
 	return h, nil
 }
@@ -72,3 +77,19 @@ func (h Byteable) JSONObject() JSONObject {
 	jo, _ := h.JSONObjectOk()
 	return jo
 }
+
+// Scan implements sql.Scanner
+func (h *Byteable) Scan(src interface{}) error {
+	switch src := src.(type) {
+	case nil:
+		nullVal := []byte(nullString)
+		*h = nullVal
+		return nil
+	case []byte:
+		return h.UnmarshalJSON(src)
+	case string:
+		return h.UnmarshalJSON([]byte(src))
+	default:
+		return i18n.NewError(context.Background(), i18n.MsgScanFailed, src, h)
+	}
+}
diff --git a/pkg/fftypes/data.go b/pkg/fftypes/data.go
index c94b1b493..966dd1e85 100644
--- a/pkg/fftypes/data.go
+++ b/pkg/fftypes/data.go
@@ -58,7 +58,7 @@ type DatatypeRef struct {
 
 func (dr *DatatypeRef) String() string {
 	if dr == nil {
-		return "null"
+		return nullString
 	}
 	return fmt.Sprintf("%s/%s", dr.Name, dr.Version)
 }
@@ -72,16 +72,20 @@ func (d DataRefs) Hash() *Bytes32 {
 }
 
 func (d *Data) CalcHash(ctx context.Context) (*Bytes32, error) {
-	if (d.Value == nil || d.Value.String() == "null") && (d.Blob == nil || d.Blob.Hash == nil) {
+	if d.Value == nil {
+		d.Value = Byteable(nullString)
+	}
+	valueIsNull := d.Value.String() == nullString
+	if valueIsNull && (d.Blob == nil || d.Blob.Hash == nil) {
 		return nil, i18n.NewError(ctx, i18n.MsgDataValueIsNull)
 	}
 	// The hash is either the blob hash, the value hash, or if both are supplied
 	// (e.g. a blob with associated metadata) it a hash of the two HEX hashes
 	// concattenated together (no spaces or separation).
 	switch {
-	case d.Value != nil && (d.Blob == nil || d.Blob.Hash == nil):
+	case !valueIsNull && (d.Blob == nil || d.Blob.Hash == nil):
 		return d.Value.Hash(), nil
-	case d.Value == nil && d.Blob != nil && d.Blob.Hash != nil:
+	case valueIsNull && d.Blob != nil && d.Blob.Hash != nil:
 		return d.Blob.Hash, nil
 	default:
 		hash := sha256.New()
diff --git a/pkg/fftypes/datatype.go b/pkg/fftypes/datatype.go
index 8b4de5c5d..b76a7c71f 100644
--- a/pkg/fftypes/datatype.go
+++ b/pkg/fftypes/datatype.go
@@ -27,6 +27,8 @@ type ValidatorType = FFEnum
 var (
 	// ValidatorTypeJSON is the validator type for JSON Schema validation
 	ValidatorTypeJSON ValidatorType = ffEnum("validatortype", "json")
+	// ValidatorTypeNone explicitly disables validation, even when a datatype is set. Allowing categorization of datatype without validation.
+	ValidatorTypeNone ValidatorType = ffEnum("validatortype", "none")
 	// ValidatorTypeSystemDefinition is the validator type for system definitions
 	ValidatorTypeSystemDefinition ValidatorType = ffEnum("validatortype", "definition")
 )


Test patch:
diff --git a/internal/orchestrator/orchestrator_test.go b/internal/orchestrator/orchestrator_test.go
index 7b15472f9..0bc6fa0fe 100644
--- a/internal/orchestrator/orchestrator_test.go
+++ b/internal/orchestrator/orchestrator_test.go
@@ -274,7 +274,9 @@ func TestBadDataExchangeInitFail(t *testing.T) {
 
 func TestBadTokensPlugin(t *testing.T) {
 	or := newTestOrchestrator()
-	tokensConfig.AddKnownKey(tokens.TokensConfigName, "test")
+	tokensConfig = config.NewPluginConfig("tokens").Array()
+	tifactory.InitPrefix(tokensConfig)
+	tokensConfig.AddKnownKey(tokens.TokensConfigName, "text")
 	tokensConfig.AddKnownKey(tokens.TokensConfigConnector, "wrong")
 	config.Set("tokens", []fftypes.JSONObject{{}})
 	or.tokens = nil
@@ -294,6 +296,8 @@ func TestBadTokensPlugin(t *testing.T) {
 
 func TestBadTokensPluginNoName(t *testing.T) {
 	or := newTestOrchestrator()
+	tokensConfig = config.NewPluginConfig("tokens").Array()
+	tifactory.InitPrefix(tokensConfig)
 	tokensConfig.AddKnownKey(tokens.TokensConfigName)
 	tokensConfig.AddKnownKey(tokens.TokensConfigConnector, "wrong")
 	config.Set("tokens", []fftypes.JSONObject{{}})
@@ -312,8 +316,31 @@ func TestBadTokensPluginNoName(t *testing.T) {
 	assert.Regexp(t, "FF10273", err)
 }
 
+func TestBadTokensPluginNoType(t *testing.T) {
+	or := newTestOrchestrator()
+	tokensConfig = config.NewPluginConfig("tokens").Array()
+	tifactory.InitPrefix(tokensConfig)
+	tokensConfig.AddKnownKey(tokens.TokensConfigName, "text")
+	tokensConfig.AddKnownKey(tokens.TokensConfigConnector)
+	config.Set("tokens", []fftypes.JSONObject{{}})
+	or.tokens = nil
+	or.mdi.On("GetConfigRecords", mock.Anything, mock.Anything, mock.Anything).Return([]*fftypes.ConfigRecord{}, nil, nil)
+	or.mdi.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	or.mbi.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	or.mii.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	or.mbi.On("VerifyIdentitySyntax", mock.Anything, mock.Anything, mock.Anything).Return("", nil)
+	or.mps.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	or.mdx.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	or.mdi.On("GetNamespace", mock.Anything, mock.Anything).Return(nil, nil)
+	or.mdi.On("UpsertNamespace", mock.Anything, mock.Anything, true).Return(nil)
+	ctx, cancelCtx := context.WithCancel(context.Background())
+	err := or.Init(ctx, cancelCtx)
+	assert.Regexp(t, "FF10273", err)
+}
+
 func TestGoodTokensPlugin(t *testing.T) {
 	or := newTestOrchestrator()
+	tokensConfig = config.NewPluginConfig("tokens").Array()
 	tifactory.InitPrefix(tokensConfig)
 	tokensConfig.AddKnownKey(tokens.TokensConfigName, "test")
 	tokensConfig.AddKnownKey(tokens.TokensConfigConnector, "https")
diff --git a/pkg/fftypes/byteable_test.go b/pkg/fftypes/byteable_test.go
index 3017352b4..e2cb08775 100644
--- a/pkg/fftypes/byteable_test.go
+++ b/pkg/fftypes/byteable_test.go
@@ -77,7 +77,7 @@ func TestByteableMarshalNull(t *testing.T) {
 	var pb Byteable
 	b, err := pb.MarshalJSON()
 	assert.NoError(t, err)
-	assert.Equal(t, "null", string(b))
+	assert.Equal(t, nullString, string(b))
 }
 
 func TestByteableUnmarshalFail(t *testing.T) {
@@ -89,3 +89,19 @@ func TestByteableUnmarshalFail(t *testing.T) {
 	jo := b.JSONObject()
 	assert.Equal(t, JSONObject{}, jo)
 }
+
+func TestScan(t *testing.T) {
+
+	var h Byteable
+	assert.NoError(t, h.Scan(nil))
+	assert.Equal(t, []byte(nullString), []byte(h))
+
+	assert.NoError(t, h.Scan(`{"some": "stuff"}`))
+	assert.Equal(t, "stuff", h.JSONObject().GetString("some"))
+
+	assert.NoError(t, h.Scan([]byte(`{"some": "stuff"}`)))
+	assert.Equal(t, "stuff", h.JSONObject().GetString("some"))
+
+	assert.Regexp(t, "FF10125", h.Scan(12345))
+
+}
diff --git a/pkg/fftypes/data_test.go b/pkg/fftypes/data_test.go
index 6fb8f6ff7..b17204789 100644
--- a/pkg/fftypes/data_test.go
+++ b/pkg/fftypes/data_test.go
@@ -19,6 +19,7 @@ package fftypes
 import (
 	"context"
 	"crypto/sha256"
+	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -27,7 +28,7 @@ import (
 func TestDatatypeReference(t *testing.T) {
 
 	var dr *DatatypeRef
-	assert.Equal(t, "null", dr.String())
+	assert.Equal(t, nullString, dr.String())
 	dr = &DatatypeRef{
 		Name:    "customer",
 		Version: "0.0.1",
@@ -61,7 +62,7 @@ func TestSealBlobOnly(t *testing.T) {
 	}
 	err := d.Seal(context.Background())
 	assert.NoError(t, err)
-	assert.Equal(t, d.Hash.String(), "22440fcf4ee9ac8c1a83de36c3a9ef39f838d960971dc79b274718392f1735f9")
+	assert.Equal(t, "22440fcf4ee9ac8c1a83de36c3a9ef39f838d960971dc79b274718392f1735f9", d.Hash.String())
 }
 
 func TestSealBlobAndHashOnly(t *testing.T) {
@@ -77,3 +78,32 @@ func TestSealBlobAndHashOnly(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, d.Hash[:], h[:])
 }
+
+func TestHashDataNull(t *testing.T) {
+
+	jd := []byte(`{
+		"id": "a64addf8-00e2-4210-9474-477e93b7f8dc",
+		"validator": "none",
+		"namespace": "default",
+		"hash": "3e12f246f0d16ab3fe6d15d15161ca5de8a00991c98fa12cea1b9733ea9832da",
+		"created": "2021-09-25T05:07:53.5847572Z",
+		"datatype": {
+			"name": "myblob"
+		},
+		"value": null,
+		"blob": {
+			"hash": "6014cbaf6bde9f9d755f347cb326db88859475e9d1a215d5dc4ccd8ae9caec7c"
+		}
+	}`)
+	var d Data
+	err := json.Unmarshal(jd, &d)
+	assert.NoError(t, err)
+
+	// Note that the processing of "null" means the value does not contribute to the hash
+	expectedHash, err := ParseBytes32(context.Background(), "6014cbaf6bde9f9d755f347cb326db88859475e9d1a215d5dc4ccd8ae9caec7c")
+	assert.NoError(t, err)
+	hash, err := d.CalcHash(context.Background())
+	assert.NoError(t, err)
+	assert.Equal(t, expectedHash.String(), hash.String())
+
+}
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index 0c7f47638..957250d7d 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -328,7 +328,7 @@ func TestE2EBroadcastBlob(t *testing.T) {
 
 }
 
-func TestE2EPrivateBlob(t *testing.T) {
+func TestE2EPrivateBlobDatatypeTagged(t *testing.T) {
 
 	ts := beforeE2ETest(t)
 	defer ts.done()
@@ -338,7 +338,7 @@ func TestE2EPrivateBlob(t *testing.T) {
 
 	var resp *resty.Response
 
-	resp, err := PrivateBlobMessage(t, ts.client1, []string{
+	resp, err := PrivateBlobMessageDatatypeTagged(t, ts.client1, []string{
 		ts.org1.Name,
 		ts.org2.Name,
 	})
@@ -346,12 +346,10 @@ func TestE2EPrivateBlob(t *testing.T) {
 	assert.Equal(t, 202, resp.StatusCode())
 
 	<-received1
-	val1 := validateReceivedMessages(ts, ts.client1, fftypes.MessageTypePrivate, fftypes.TransactionTypeBatchPin, 1, 0)
-	assert.Regexp(t, "myfile.txt", string(val1))
+	_ = validateReceivedMessages(ts, ts.client1, fftypes.MessageTypePrivate, fftypes.TransactionTypeBatchPin, 1, 0)
 
 	<-received2
-	val2 := validateReceivedMessages(ts, ts.client2, fftypes.MessageTypePrivate, fftypes.TransactionTypeBatchPin, 1, 0)
-	assert.Regexp(t, "myfile.txt", string(val2))
+	_ = validateReceivedMessages(ts, ts.client2, fftypes.MessageTypePrivate, fftypes.TransactionTypeBatchPin, 1, 0)
 }
 
 func TestE2ETokenPool(t *testing.T) {
diff --git a/test/e2e/restclient.go b/test/e2e/restclient.go
index b9d72db70..5c9f88fe8 100644
--- a/test/e2e/restclient.go
+++ b/test/e2e/restclient.go
@@ -162,7 +162,7 @@ func BroadcastMessage(client *resty.Client, data *fftypes.DataRefOrValue) (*rest
 		Post(urlBroadcastMessage)
 }
 
-func CreateBlob(t *testing.T, client *resty.Client) *fftypes.Data {
+func CreateBlob(t *testing.T, client *resty.Client, dt *fftypes.DatatypeRef) *fftypes.Data {
 	r, _ := rand.Int(rand.Reader, big.NewInt(1024*1024))
 	blob := make([]byte, r.Int64()+1024*1024)
 	for i := 0; i < len(blob); i++ {
@@ -171,25 +171,40 @@ func CreateBlob(t *testing.T, client *resty.Client) *fftypes.Data {
 	var blobHash fftypes.Bytes32 = sha256.Sum256(blob)
 	t.Logf("Blob size=%d hash=%s", len(blob), &blobHash)
 	var data fftypes.Data
+	formData := map[string]string{}
+	if dt == nil {
+		// If there's no datatype, tell FireFly to automatically add a data payload
+		formData["autometa"] = "true"
+		formData["metadata"] = `{"mymeta": "data"}`
+	} else {
+		// Otherwise use a tagging only approach, where we allow a nil value, specify that this should
+		// not be validated, but still set a datatype for classification of the data.
+		formData["validator"] = "none"
+		formData["datatype.name"] = dt.Name
+		formData["datatype.version"] = dt.Version
+	}
 	resp, err := client.R().
-		SetFormData(map[string]string{
-			"autometa": "true",
-			"metadata": `{"mymeta": "data"}`,
-		}).
+		SetFormData(formData).
 		SetFileReader("file", "myfile.txt", bytes.NewReader(blob)).
 		SetResult(&data).
 		Post(urlUploadData)
 	require.NoError(t, err)
 	require.Equal(t, 201, resp.StatusCode(), "POST %s [%d]: %s", urlUploadData, resp.StatusCode(), resp.String())
-	assert.Equal(t, "data", data.Value.JSONObject().GetString("mymeta"))
-	assert.Equal(t, "myfile.txt", data.Value.JSONObject().GetString("filename"))
-	assert.Equal(t, float64(len(blob)), data.Value.JSONObject()["size"])
+	t.Logf("Data created: %s", data.ID)
+	if dt == nil {
+		assert.Equal(t, "data", data.Value.JSONObject().GetString("mymeta"))
+		assert.Equal(t, "myfile.txt", data.Value.JSONObject().GetString("filename"))
+		assert.Equal(t, float64(len(blob)), data.Value.JSONObject()["size"])
+	} else {
+		assert.Equal(t, fftypes.ValidatorTypeNone, data.Validator)
+		assert.Equal(t, *dt, *data.Datatype)
+	}
 	assert.Equal(t, blobHash, *data.Blob.Hash)
 	return &data
 }
 
 func BroadcastBlobMessage(t *testing.T, client *resty.Client) (*resty.Response, error) {
-	data := CreateBlob(t, client)
+	data := CreateBlob(t, client, nil)
 	return client.R().
 		SetBody(fftypes.MessageInOut{
 			InlineData: fftypes.InlineData{
@@ -199,8 +214,8 @@ func BroadcastBlobMessage(t *testing.T, client *resty.Client) (*resty.Response,
 		Post(urlBroadcastMessage)
 }
 
-func PrivateBlobMessage(t *testing.T, client *resty.Client, orgNames []string) (*resty.Response, error) {
-	data := CreateBlob(t, client)
+func PrivateBlobMessageDatatypeTagged(t *testing.T, client *resty.Client, orgNames []string) (*resty.Response, error) {
+	data := CreateBlob(t, client, &fftypes.DatatypeRef{Name: "myblob"})
 	members := make([]fftypes.MemberInput, len(orgNames))
 	for i, oName := range orgNames {
 		// We let FireFly resolve the friendly name of the org to the identity