Repository: amzn/ion-go
Pull request #145 (2020-08-19 20:23:26)

Create SymbolToken and incorporate to SymbolTable

Issue #, if available:

Description of changes:

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.




Patch:
diff --git a/ion/symboltable.go b/ion/symboltable.go
index b366d4d..140c3e6 100644
--- a/ion/symboltable.go
+++ b/ion/symboltable.go
@@ -28,7 +28,8 @@ type SymbolTable interface {
 	Symbols() []string
 	// MaxID returns the maximum ID this symbol table defines.
 	MaxID() uint64
-
+	// Find finds the SymbolToken by its name.
+	Find(symbol string) *SymbolToken
 	// FindByName finds the ID of a symbol by its name.
 	FindByName(symbol string) (uint64, bool)
 	// FindByID finds the name of a symbol given its ID.
@@ -128,6 +129,20 @@ func (s *sst) Adjust(maxID uint64) SharedSymbolTable {
 	}
 }
 
+func (s *sst) Find(sym string) *SymbolToken {
+	id, ok := s.FindByName(sym)
+	if !ok {
+		return nil
+	}
+
+	text, ok := s.FindByID(id)
+	if !ok {
+		return nil
+	}
+
+	return &SymbolToken{Text: &text, LocalSID: SymbolIDUnknown}
+}
+
 func (s *sst) FindByName(sym string) (uint64, bool) {
 	id, ok := s.index[sym]
 	return id, ok
@@ -243,6 +258,10 @@ func (s *bogusSST) Adjust(maxID uint64) SharedSymbolTable {
 	}
 }
 
+func (s *bogusSST) Find(sym string) *SymbolToken {
+	return nil
+}
+
 func (s *bogusSST) FindByName(sym string) (uint64, bool) {
 	return 0, false
 }
@@ -318,6 +337,24 @@ func (t *lst) MaxID() uint64 {
 	return t.maxImportID + uint64(len(t.symbols))
 }
 
+func (t *lst) Find(s string) *SymbolToken {
+	// Check import
+	for _, imp := range t.imports {
+		symbolToken := imp.Find(s)
+		if symbolToken != nil {
+			return symbolToken
+		}
+	}
+
+	// Check local
+	_, ok := t.index[s]
+	if ok {
+		return &SymbolToken{Text: &s, LocalSID: SymbolIDUnknown}
+	}
+
+	return nil
+}
+
 func (t *lst) FindByName(s string) (uint64, bool) {
 	for i, imp := range t.imports {
 		if id, ok := imp.FindByName(s); ok {
diff --git a/ion/symboltoken.go b/ion/symboltoken.go
new file mode 100644
index 0000000..c9a0abd
--- /dev/null
+++ b/ion/symboltoken.go
@@ -0,0 +1,118 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package ion
+
+import "fmt"
+
+const (
+	// SymbolIDUnknown is placeholder for when a symbol token has no symbol ID.
+	SymbolIDUnknown = -1
+)
+
+// ImportSource is a reference to a SID within a shared symbol table.
+type ImportSource struct {
+
+	// The name of the shared symbol table that this token refers to.
+	Table string
+
+	// The ID of the interned symbol text within the shared SymbolTable.
+	// This must be greater or equal to 1.
+	SID int64
+}
+
+func newSource(table string, sid int64) *ImportSource {
+	value := ImportSource{
+		Table: table,
+		SID:   sid,
+	}
+	return &value
+}
+
+// Equal figures out if two import sources are equal for each component.
+func (is *ImportSource) Equal(o *ImportSource) bool {
+	return is.Table == o.Table && is.SID == o.SID
+}
+
+// SymbolToken is the representation for annotations, field names, and the textual content of Ion symbol values.
+// The `nil` value for SymbolToken is the SID `$0`.
+type SymbolToken struct {
+	// The string text of the token or nil if unknown.
+	Text *string
+	// Local symbol ID associated with the token.
+	LocalSID int64
+	// The shared symbol table location that this token came from, or nil if undefined.
+	Source *ImportSource
+}
+
+var (
+	// symbolTokenUndefined is the sentinel for invalid tokens.
+	// The `nil` value is actually SID `$0` which is a defined token.
+	symbolTokenUndefined = SymbolToken{
+		LocalSID: SymbolIDUnknown,
+	}
+)
+
+func (st *SymbolToken) String() string {
+	text := "nil"
+	if st.Text != nil {
+		text = fmt.Sprintf("%q", *st.Text)
+	}
+
+	source := "nil"
+	if st.Source != nil {
+		source = fmt.Sprintf("{%q %d}", st.Source.Table, st.Source.SID)
+	}
+
+	return fmt.Sprintf("{%s %d %s}", text, st.LocalSID, source)
+}
+
+// Equal figures out if two symbol tokens are equivalent.
+func (st *SymbolToken) Equal(o *SymbolToken) bool {
+	if st.Text == nil && o.Text == nil {
+		if st.Source == nil && o.Source == nil {
+			return true
+		}
+		if st.Source != nil && o.Source != nil {
+			return st.Source.Equal(o.Source)
+		}
+		return false
+	}
+
+	if st.Text != nil && o.Text != nil {
+		return *st.Text == *o.Text
+	}
+	return false
+}
+
+// NewSymbolToken will check and return a symbol token if it exists in a symbol table,
+// otherwise return a new symbol token.
+func NewSymbolToken(symbolTable SymbolTable, text string) SymbolToken {
+	token := symbolTable.Find(text)
+	if token == nil {
+		token = &SymbolToken{Text: &text, LocalSID: SymbolIDUnknown}
+	}
+	return *token
+}
+
+// NewSymbolTokens will check and return a list of symbol tokens if they exists in a symbol table,
+// otherwise return a list of new symbol tokens.
+func NewSymbolTokens(symbolTable SymbolTable, textVals []string) []SymbolToken {
+	var tokens []SymbolToken
+	for _, text := range textVals {
+		tokens = append(tokens, NewSymbolToken(symbolTable, text))
+	}
+	return tokens
+}


Test patch:
diff --git a/ion/symboltable_test.go b/ion/symboltable_test.go
index 1928551..78c59d7 100644
--- a/ion/symboltable_test.go
+++ b/ion/symboltable_test.go
@@ -49,6 +49,9 @@ func TestSharedSymbolTable(t *testing.T) {
 	testFindByID(t, st, 4, "null")
 	testFindByID(t, st, 7, "")
 
+	testFindSymbolToken(t, st, "def", SymbolToken{Text: newString("def"), LocalSID: -1})
+	testFindSymbolToken(t, st, "foo'bar", SymbolToken{Text: newString("foo'bar"), LocalSID: -1})
+
 	testString(t, st, `$ion_shared_symbol_table::{name:"test",version:2,symbols:["abc","def","foo'bar","null","def","ghi"]}`)
 }
 
@@ -70,6 +73,10 @@ func TestLocalSymbolTable(t *testing.T) {
 	testFindByID(t, st, 11, "bar")
 	testFindByID(t, st, 12, "")
 
+	testFindSymbolToken(t, st, "foo", SymbolToken{Text: newString("foo"), LocalSID: -1})
+	testFindSymbolToken(t, st, "bar", SymbolToken{Text: newString("bar"), LocalSID: -1})
+	testFindSymbolToken(t, st, "$ion", SymbolToken{Text: newString("$ion"), LocalSID: -1})
+
 	testString(t, st, `$ion_symbol_table::{symbols:["foo","bar"]}`)
 }
 
@@ -106,6 +113,11 @@ func TestLocalSymbolTableWithImports(t *testing.T) {
 	testFindByID(t, st, 13, "bar2")
 	testFindByID(t, st, 14, "")
 
+	testFindSymbolToken(t, st, "foo", SymbolToken{Text: newString("foo"), LocalSID: -1})
+	testFindSymbolToken(t, st, "bar", SymbolToken{Text: newString("bar"), LocalSID: -1})
+	testFindSymbolToken(t, st, "foo2", SymbolToken{Text: newString("foo2"), LocalSID: -1})
+	testFindSymbolToken(t, st, "bar2", SymbolToken{Text: newString("bar2"), LocalSID: -1})
+
 	testString(t, st, `$ion_symbol_table::{imports:[{name:"shared",version:1,max_id:2}],symbols:["foo2","bar2"]}`)
 }
 
@@ -186,6 +198,20 @@ func testFindByID(t *testing.T, st SymbolTable, id uint64, expected string) {
 	})
 }
 
+func testFindSymbolToken(t *testing.T, st SymbolTable, sym string, expected SymbolToken) {
+	t.Run("Find("+sym+")", func(t *testing.T) {
+		actual := st.Find(sym)
+
+		if actual == nil {
+			t.Fatal("unexpectedly not found")
+		}
+
+		if !actual.Equal(&expected) {
+			t.Errorf("expected %v, got %v", expected, actual)
+		}
+	})
+}
+
 func testString(t *testing.T, st SymbolTable, expected string) {
 	t.Run("String()", func(t *testing.T) {
 		actual := st.String()
diff --git a/ion/symboltoken_test.go b/ion/symboltoken_test.go
new file mode 100644
index 0000000..2290e56
--- /dev/null
+++ b/ion/symboltoken_test.go
@@ -0,0 +1,201 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package ion
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func newString(value string) *string {
+	return &value
+}
+
+var symbolTokenEqualsTestData = []struct {
+	text1   *string
+	sid1    int64
+	source1 *ImportSource
+	text2   *string
+	sid2    int64
+	source2 *ImportSource
+	equals  bool
+}{
+	{nil, 123, nil, nil, 123, nil, true},
+	{nil, 123, newSource("table", 1), nil, 456, newSource("table", 1), true},
+	{newString("text1"), 123, nil, newString("text1"), 123, nil, true},
+	{newString("text2"), 123, newSource("table", 1), newString("text2"), 456, newSource("table", 1), true},
+	{nil, 123, newSource("table", 1), nil, 123, nil, false},
+	{nil, 123, nil, newString("text1"), 456, nil, false},
+	{nil, 123, newSource("table", 1), nil, 123, newSource("table2", 1), false},
+	{nil, 123, newSource("table", 1), nil, 456, newSource("table", 2), false},
+	{newString("text2"), 123, nil, newString("text3"), 123, nil, false},
+}
+
+func TestSymbolTokenEqualsOperator(t *testing.T) {
+	for _, testData := range symbolTokenEqualsTestData {
+		st1 := SymbolToken{Text: testData.text1, LocalSID: testData.sid1, Source: testData.source1}
+		st2 := SymbolToken{Text: testData.text2, LocalSID: testData.sid2, Source: testData.source2}
+
+		if testData.equals {
+			if !st1.Equal(&st2) {
+				t.Errorf("expected %v, got %v", true, false)
+			}
+		} else {
+			if st1.Equal(&st2) {
+				t.Errorf("expected %v, got %v", true, false)
+			}
+		}
+	}
+}
+
+// Make sure SymbolToken conforms to Stringer
+var _ fmt.Stringer = &SymbolToken{}
+
+func TestSymbolToken_String(t *testing.T) {
+	cases := []struct {
+		desc     string
+		token    SymbolToken
+		expected string
+	}{
+		{
+			desc: "Text and SID",
+			token: SymbolToken{
+				Text:     newString("hello"),
+				LocalSID: 10,
+				Source:   nil,
+			},
+			expected: `{"hello" 10 nil}`,
+		},
+		{
+			desc: "nil Text",
+			token: SymbolToken{
+				Text:     nil,
+				LocalSID: 11,
+				Source:   nil,
+			},
+			expected: `{nil 11 nil}`,
+		},
+		{
+			desc: "Text and SID with Import",
+			token: SymbolToken{
+				Text:     newString("world"),
+				LocalSID: 12,
+				Source:   newSource("foobar", 3),
+			},
+			expected: `{"world" 12 {"foobar" 3}}`,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.desc, func(t *testing.T) {
+			if diff := cmp.Diff(c.expected, c.token.String()); diff != "" {
+				t.Errorf("Token String() differs (-expected, +actual):\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestNewImportSource(t *testing.T) {
+	is := newSource("table", 1)
+	if is.Table != "table" {
+		t.Errorf("expected %v, got %v", "table", is.Table)
+	}
+	if is.SID != 1 {
+		t.Errorf("expected %v, got %v", 1, is.SID)
+	}
+}
+
+var importSourceEqualsTestData = []struct {
+	text1  string
+	sid1   int64
+	text2  string
+	sid2   int64
+	equals bool
+}{
+	{"text1", 123, "text1", 123, true},
+	{"text2", 456, "text2", 456, true},
+	{"text1", 123, "text1", 456, false},
+	{"text2", 456, "text3", 456, false},
+}
+
+func TestImportSourceEqualsOperator(t *testing.T) {
+	for _, testData := range importSourceEqualsTestData {
+		is1 := newSource(testData.text1, testData.sid1)
+		is2 := newSource(testData.text2, testData.sid2)
+
+		if testData.equals {
+			if !is1.Equal(is2) {
+				t.Errorf("expected %v, got %v", true, false)
+			}
+		} else {
+			if is1.Equal(is2) {
+				t.Errorf("expected %v, got %v", false, true)
+			}
+		}
+	}
+}
+
+func TestNewSymbolTokenThatAlreadyExistInSymbolTable(t *testing.T) {
+	expectedSymbolToken := SymbolToken{Text: newString("$ion"), LocalSID: SymbolIDUnknown}
+
+	actualSymbolToken := NewSymbolToken(V1SystemSymbolTable, "$ion")
+
+	if !actualSymbolToken.Equal(&expectedSymbolToken) {
+		t.Errorf("expected %v, got %v", expectedSymbolToken, actualSymbolToken)
+	}
+}
+
+func TestNewSymbolTokenThatDoesNotExistInSymbolTable(t *testing.T) {
+	expectedSymbolToken := SymbolToken{Text: newString("newToken"), LocalSID: SymbolIDUnknown}
+
+	actualSymbolToken := NewSymbolToken(V1SystemSymbolTable, "newToken")
+
+	if !actualSymbolToken.Equal(&expectedSymbolToken) {
+		t.Errorf("expected %v, got %v", expectedSymbolToken, actualSymbolToken)
+	}
+}
+
+func TestNewSymbolTokensThatAlreadyExistInSymbolTable(t *testing.T) {
+	expectedSymbolTokens := []SymbolToken{
+		{Text: newString("$ion"), LocalSID: SymbolIDUnknown},
+		{Text: newString("$ion_1_0"), LocalSID: SymbolIDUnknown}}
+
+	actualSymbolTokens := NewSymbolTokens(V1SystemSymbolTable, []string{"$ion", "$ion_1_0"})
+
+	for index, actualSymbolToken := range actualSymbolTokens {
+		if !actualSymbolToken.Equal(&expectedSymbolTokens[index]) {
+			t.Errorf("expected %v, got %v", &expectedSymbolTokens[index], actualSymbolToken)
+
+		}
+	}
+}
+
+func TestNewSymbolTokensThatDoNotExistInSymbolTable(t *testing.T) {
+	expectedSymbolTokens := []SymbolToken{
+		{Text: newString("newToken1"), LocalSID: SymbolIDUnknown},
+		{Text: newString("newToken2"), LocalSID: SymbolIDUnknown}}
+
+	actualSymbolTokens := NewSymbolTokens(V1SystemSymbolTable, []string{"newToken1", "newToken2"})
+
+	for index, actualSymbolToken := range actualSymbolTokens {
+		if !actualSymbolToken.Equal(&expectedSymbolTokens[index]) {
+			t.Errorf("expected %v, got %v", &expectedSymbolTokens[index], actualSymbolToken)
+
+		}
+	}
+}