Repository: ably/ably-go (go)

Issue:
ID and Client ID is blank on realtime incoming messages.
Found a possible bug that is causing trouble for me. Incoming realtime messages have a blank message ID and some other blank fields like Client ID. Historical messages DO have an ID. Other SDKs such as the Javascript library do report IDs in realtime.

Here is some test code that you can run, you should see that the message IDs are blank.

```
func main() {
	options := ably.NewClientOptions("APIKEY")
	var err error
	ablyClient, err := ably.NewRealtimeClient(options)
	ablyChannel := ablyClient.Channels.Get("test")

	go func() {
		number := 1
		for {
			time.Sleep(time.Second * 2)
			ablyChannel.Publish("test", fmt.Sprintf("Message number %v", number))
			number++
		}
	}()

	sub, err := ablyChannel.Subscribe("test")
	if err != nil {
		panic(err)
	}
	for msg := range sub.MessageChannel() {
		fmt.Printf("recv:\n\tName: %s\n\tData: %s\n\tID: %s \n\tClient ID: %s\n", msg.Name, msg.Data, msg.ID, msg.ClientID)
	}
}
```

┆sprint: SDK 2022-Q1-S8


Pull request:
Update empty message fields on receive 
Fixes #58 #377 #293 

Patch:
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index b1574bc..8093ed7 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.17, 1.18]
+        go-version: [1.16, 1.17]
 
     steps:
       - uses: actions/checkout@v2
diff --git a/EXAMPLES.md b/EXAMPLES.md
deleted file mode 100644
index 2052890..0000000
--- a/EXAMPLES.md
+++ /dev/null
@@ -1,279 +0,0 @@
-# Examples
-
-## Using the Realtime API
-
-### Creating a client
-
-```go
-client, err := ably.NewRealtime(ably.WithKey("xxx:xxx"))
-if err != nil {
-        panic(err)
-}
-
-channel := client.Channels.Get("test")
-```
-
-### Subscribing to events
-
-You may monitor events on connections and channels.
-
-```go
-client, err = ably.NewRealtime(
-ably.WithKey("xxx:xxx"),
-ably.WithAutoConnect(false), // Set this option to avoid missing state changes.
-)
-if err != nil {
-        panic(err)
-}
-
-// Set up connection events handler.
-client.Connection.OnAll(func(change ably.ConnectionStateChange) {
-        fmt.Printf("Connection event: %s state=%s reason=%s", change.Event, change.Current, change.Reason)
-})
-
-// Then connect.
-client.Connect()
-
-channel = client.Channels.Get("test")
-
-channel.OnAll(func(change ably.ChannelStateChange) {
-        fmt.Printf("Channel event event: %s channel=%s state=%s reason=%s", channel.Name, change.Event, change.Current, change.Reason)
-})
-```
-
-### Subscribing to a channel for all messages
-
-```go
-unsubscribe, err := channel.SubscribeAll(ctx, func(msg *ably.Message) {
-        fmt.Printf("Received message: name=%s data=%v\n", msg.Name, msg.Data)
-})
-if err != nil {
-        panic(err)
-}
-```
-
-### Subscribing to a channel for `EventName1` and `EventName2` message names
-
-```go
-unsubscribe1, err := channel.Subscribe(ctx, "EventName1", func(msg *ably.Message) {
-        fmt.Printf("Received message: name=%s data=%v\n", msg.Name, msg.Data)
-})
-if err != nil {
-        panic(err)
-}
-
-unsubscribe2, err := channel.Subscribe(ctx, "EventName2", func(msg *ably.Message) {
-        fmt.Printf("Received message: name=%s data=%v\n", msg.Name, msg.Data)
-})
-if err != nil {
-        panic(err)
-}
-```
-
-### Publishing to a channel
-
-```go
-err = channel.Publish(ctx, "EventName1", "EventData1")
-if err != nil {
-        panic(err)
-}
-```
-
-### Handling errors
-
-Errors returned by this library may have an underlying `*ErrorInfo` type.
-
-[See Ably documentation for ErrorInfo.](https://www.ably.io/documentation/realtime/types#error-info)
-
-```go
-badClient, err := ably.NewRealtime(ably.WithKey("invalid:key"))
-if err != nil {
-        panic(err)
-}
-
-err = badClient.Channels.Get("test").Publish(ctx, "event", "data")
-if errInfo := (*ably.ErrorInfo)(nil); errors.As(err, &errInfo) {
-        fmt.Printf("Error publishing message: code=%v status=%v cause=%v", errInfo.Code, errInfo.StatusCode, errInfo.Cause)
-} else if err != nil {
-        panic(err)
-}
-```
-
-### Announcing presence on a channel
-
-```go
-err = channel.Presence.Enter(ctx, "presence data")
-if err != nil {
-        panic(err)
-}
-```
-
-### Announcing presence on a channel on behalf of other client
-
-```go
-err = channel.Presence.EnterClient(ctx, "clientID", "presence data")
-if err != nil {
-        panic(err)
-}
-```
-
-### Updating and leaving presence
-
-```go
-// Update also has an UpdateClient variant.
-err = channel.Presence.Update(ctx, "new presence data")
-if err != nil {
-        panic(err)
-}
-
-// Leave also has an LeaveClient variant.
-err = channel.Presence.Leave(ctx, "last presence data")
-if err != nil {
-        panic(err)
-}
-```
-
-### Getting all clients present on a channel
-
-```go
-clients, err := channel.Presence.Get(ctx)
-if err != nil {
-        panic(err)
-}
-
-for _, client := range clients {
-        fmt.Println("Present client:", client)
-}
-```
-
-### Subscribing to all presence messages
-
-```go
-unsubscribe, err = channel.Presence.SubscribeAll(ctx, func(msg *ably.PresenceMessage) {
-        fmt.Printf("Presence event: action=%v data=%v", msg.Action, msg.Data)
-})
-if err != nil {
-        panic(err)
-}
-```
-
-### Subscribing to 'Enter' presence messages only
-
-```go
-unsubscribe, err = channel.Presence.Subscribe(ctx, ably.PresenceActionEnter, func(msg *ably.PresenceMessage) {
-        fmt.Printf("Presence event: action=%v data=%v", msg.Action, msg.Data)
-})
-if err != nil {
-        panic(err)
-}
-```
-
-## Using the REST API
-
-### Introduction
-
-All examples assume a client and/or channel has been created as follows:
-
-```go
-client, err := ably.NewREST(ably.WithKey("xxx:xxx"))
-if err != nil {
-        panic(err)
-}
-
-channel := client.Channels.Get("test")
-```
-
-### Publishing a message to a channel
-
-```go
-err = channel.Publish(ctx, "HelloEvent", "Hello!")
-if err != nil {
-        panic(err)
-}
-
-// You can also publish multiple messages in a single request.
-err = channel.PublishMultiple(ctx, []*ably.Message{
-        {Name: "HelloEvent", Data: "Hello!"},
-        {Name: "ByeEvent", Data: "Bye!"},
-})
-if err != nil {
-        panic(err)
-}
-
-// A REST client can publish messages on behalf of another client
-// by providing the connection key of that client.
-err := channel.Publish(ctx, "temperature", "12.7", ably.PublishWithConnectionKey("connectionKeyOfAnotherClient"))
-if err != nil {
-        panic(err)
-}
-```
-
-### Querying the History
-
-```go
-pages, err := channel.History().Pages(ctx)
-if err != nil {
-        panic(err)
-}
-for pages.Next(ctx) {
-        for _, message := range pages.Items() {
-                fmt.Println(message)
-        }
-}
-if err := pages.Err(); err != nil {
-        panic(err)
-}
-
-```
-
-### Presence on a channel
-
-```go
-pages, err := channel.Presence.Get().Pages(ctx)
-if err != nil {
-        panic(err)
-}
-for pages.Next(ctx) {
-        for _, presence := range pages.Items() {
-                fmt.Println(presence)
-        }
-}
-if err := pages.Err(); err != nil {
-        panic(err)
-}
-```
-
-### Querying the Presence History
-
-```go
-pages, err := channel.Presence.History().Pages(ctx)
-if err != nil {
-        panic(err)
-}
-for pages.Next(ctx) {
-        for _, presence := range pages.Items() {
-                fmt.Println(presence)
-        }
-}
-if err := pages.Err(); err != nil {
-        panic(err)
-}
-```
-
-### Fetching your application's stats
-
-```go
-pages, err := client.Stats().Pages(ctx)
-if err != nil {
-        panic(err)
-}
-for pages.Next(ctx) {
-        for _, stat := range pages.Items() {
-                fmt.Println(stat)
-        }
-}
-if err := pages.Err(); err != nil {
-        panic(err)
-}
-```
-
diff --git a/README.md b/README.md
index 2d8311b..138e891 100644
--- a/README.md
+++ b/README.md
@@ -25,9 +25,282 @@ Please see our [Upgrade / Migration Guide](UPDATING.md) for notes on changes you
 Users updating from version 1.1.5 of this library will note that there are significant breaking changes to the API.
 Our [current approach to versioning](https://ably.com/documentation/client-lib-development-guide/versioning) is not compliant with semantic versioning, which is why these changes are breaking despite presenting only a change in the `minor` component of the version number.
 
-## Examples
+## Using the Realtime API
 
-See [EXAMPLES.md](EXAMPLES.md) for examples on how to use the ably API.
+### Creating a client
+
+```go
+client, err := ably.NewRealtime(ably.WithKey("xxx:xxx"))
+if err != nil {
+        panic(err)
+}
+
+channel := client.Channels.Get("test")
+```
+
+### Subscribing to events
+
+You may monitor events on connections and channels.
+
+```go
+client, err = ably.NewRealtime(
+ably.WithKey("xxx:xxx"),
+ably.WithAutoConnect(false), // Set this option to avoid missing state changes.
+)
+if err != nil {
+        panic(err)
+}
+
+// Set up connection events handler.
+client.Connection.OnAll(func(change ably.ConnectionStateChange) {
+        fmt.Printf("Connection event: %s state=%s reason=%s", change.Event, change.Current, change.Reason)
+})
+
+// Then connect.
+client.Connect()
+
+channel = client.Channels.Get("test")
+
+channel.OnAll(func(change ably.ChannelStateChange) {
+        fmt.Printf("Channel event event: %s channel=%s state=%s reason=%s", channel.Name, change.Event, change.Current, change.Reason)
+})
+```
+
+### Subscribing to a channel for all messages
+
+```go
+unsubscribe, err := channel.SubscribeAll(ctx, func(msg *ably.Message) {
+        fmt.Printf("Received message: name=%s data=%v\n", msg.Name, msg.Data)
+})
+if err != nil {
+        panic(err)
+}
+```
+
+### Subscribing to a channel for `EventName1` and `EventName2` message names
+
+```go
+unsubscribe1, err := channel.Subscribe(ctx, "EventName1", func(msg *ably.Message) {
+        fmt.Printf("Received message: name=%s data=%v\n", msg.Name, msg.Data)
+})
+if err != nil {
+        panic(err)
+}
+
+unsubscribe2, err := channel.Subscribe(ctx, "EventName2", func(msg *ably.Message) {
+        fmt.Printf("Received message: name=%s data=%v\n", msg.Name, msg.Data)
+})
+if err != nil {
+        panic(err)
+}
+```
+
+### Publishing to a channel
+
+```go
+err = channel.Publish(ctx, "EventName1", "EventData1")
+if err != nil {
+        panic(err)
+}
+```
+
+### Handling errors
+
+Errors returned by this library may have an underlying `*ErrorInfo` type.
+
+[See Ably documentation for ErrorInfo.](https://www.ably.io/documentation/realtime/types#error-info)
+
+```go
+badClient, err := ably.NewRealtime(ably.WithKey("invalid:key"))
+if err != nil {
+        panic(err)
+}
+
+err = badClient.Channels.Get("test").Publish(ctx, "event", "data")
+if errInfo := (*ably.ErrorInfo)(nil); errors.As(err, &errInfo) {
+        fmt.Printf("Error publishing message: code=%v status=%v cause=%v", errInfo.Code, errInfo.StatusCode, errInfo.Cause)
+} else if err != nil {
+        panic(err)
+}
+```
+
+### Announcing presence on a channel
+
+```go
+err = channel.Presence.Enter(ctx, "presence data")
+if err != nil {
+        panic(err)
+}
+```
+
+### Announcing presence on a channel on behalf of other client
+
+```go
+err = channel.Presence.EnterClient(ctx, "clientID", "presence data")
+if err != nil {
+        panic(err)
+}
+```
+
+### Updating and leaving presence
+
+```go
+// Update also has an UpdateClient variant.
+err = channel.Presence.Update(ctx, "new presence data")
+if err != nil {
+        panic(err)
+}
+
+// Leave also has an LeaveClient variant.
+err = channel.Presence.Leave(ctx, "last presence data")
+if err != nil {
+        panic(err)
+}
+```
+
+### Getting all clients present on a channel
+
+```go
+clients, err := channel.Presence.Get(ctx)
+if err != nil {
+        panic(err)
+}
+
+for _, client := range clients {
+        fmt.Println("Present client:", client)
+}
+```
+
+### Subscribing to all presence messages
+
+```go
+unsubscribe, err = channel.Presence.SubscribeAll(ctx, func(msg *ably.PresenceMessage) {
+        fmt.Printf("Presence event: action=%v data=%v", msg.Action, msg.Data)
+})
+if err != nil {
+        panic(err)
+}
+```
+
+### Subscribing to 'Enter' presence messages only
+
+```go
+unsubscribe, err = channel.Presence.Subscribe(ctx, ably.PresenceActionEnter, func(msg *ably.PresenceMessage) {
+        fmt.Printf("Presence event: action=%v data=%v", msg.Action, msg.Data)
+})
+if err != nil {
+        panic(err)
+}
+```
+
+## Using the REST API
+
+### Introduction
+
+All examples assume a client and/or channel has been created as follows:
+
+```go
+client, err := ably.NewREST(ably.WithKey("xxx:xxx"))
+if err != nil {
+        panic(err)
+}
+
+channel := client.Channels.Get("test")
+```
+
+### Publishing a message to a channel
+
+```go
+err = channel.Publish(ctx, "HelloEvent", "Hello!")
+if err != nil {
+        panic(err)
+}
+
+// You can also publish multiple messages in a single request.
+err = channel.PublishMultiple(ctx, []*ably.Message{
+        {Name: "HelloEvent", Data: "Hello!"},
+        {Name: "ByeEvent", Data: "Bye!"},
+})
+if err != nil {
+        panic(err)
+}
+
+// A REST client can publish messages on behalf of another client
+// by providing the connection key of that client.
+err := channel.Publish(ctx, "temperature", "12.7", ably.PublishWithConnectionKey("connectionKeyOfAnotherClient"))
+if err != nil {
+        panic(err)
+}
+```
+
+### Querying the History
+
+```go
+pages, err := channel.History().Pages(ctx)
+if err != nil {
+        panic(err)
+}
+for pages.Next(ctx) {
+        for _, message := range pages.Items() {
+                fmt.Println(message)
+        }
+}
+if err := pages.Err(); err != nil {
+        panic(err)
+}
+
+```
+
+### Presence on a channel
+
+```go
+pages, err := channel.Presence.Get().Pages(ctx)
+if err != nil {
+        panic(err)
+}
+for pages.Next(ctx) {
+        for _, presence := range pages.Items() {
+                fmt.Println(presence)
+        }
+}
+if err := pages.Err(); err != nil {
+        panic(err)
+}
+```
+
+### Querying the Presence History
+
+```go
+pages, err := channel.Presence.History().Pages(ctx)
+if err != nil {
+        panic(err)
+}
+for pages.Next(ctx) {
+        for _, presence := range pages.Items() {
+                fmt.Println(presence)
+        }
+}
+if err := pages.Err(); err != nil {
+        panic(err)
+}
+```
+
+### Fetching your application's stats
+
+```go
+pages, err := client.Stats().Pages(ctx)
+if err != nil {
+        panic(err)
+}
+for pages.Next(ctx) {
+        for _, stat := range pages.Items() {
+                fmt.Println(stat)
+        }
+}
+if err := pages.Err(); err != nil {
+        panic(err)
+}
+```
 
 ## Feature support
 
@@ -154,4 +427,4 @@ Because this package uses `internal` packages, all fork development has to happe
 5. commit your changes (`git commit -am 'Add some feature'`)
 6. ensure you have added suitable tests and the test suite is passing for both JSON and MessagePack protocols (see [workflow](.github/workflows/check.yml) for the test commands executed in the CI environment).
 7. push to the branch: `git push fork my-new-feature`
-8. create a new Pull Request.
+8. create a new Pull Request
diff --git a/ably/doc.go b/ably/doc.go
index da31933..2d7a41f 100644
--- a/ably/doc.go
+++ b/ably/doc.go
@@ -58,13 +58,6 @@
 // from the iterator object. Finally, once it returns false, the Err method must
 // be called to check if the iterator stopped due to some error, or else, it
 // just finished going through all pages.
-
-// Calling the First method on the PaginatedResults returns the first page of the
-// results. However, the Next method has to be called before inspecting the items.
-//
-// For every page in the PaginatedResults, the HasNext method can be called to check if there
-// are more page(s) available. IsLast method checks if the page is the
-// last page. Both methods return a true or false value.
 //
 // See the PaginatedResults example.
 package ably
diff --git a/ably/proto_message.go b/ably/proto_message.go
index 03caac9..042f513 100644
--- a/ably/proto_message.go
+++ b/ably/proto_message.go
@@ -31,6 +31,28 @@ type Message struct {
 	Extras        map[string]interface{} `json:"extras,omitempty" codec:"extras,omitempty"`
 }
 
+func (p *protocolMessage) updateInnerMessageEmptyFields(m *Message, index int) {
+	if empty(m.ID) {
+		m.ID = fmt.Sprintf("%s:%d", p.ID, index)
+	}
+	if empty(m.ConnectionID) {
+		m.ConnectionID = p.ConnectionID
+	}
+	if m.Timestamp == 0 {
+		m.Timestamp = p.Timestamp
+	}
+}
+
+//updateInnerMessagesEmptyFields - Update inner Message empty Id, connectionId and Timestamp
+func (p *protocolMessage) updateInnerMessagesEmptyFields() {
+	for i, m := range p.Messages {
+		p.updateInnerMessageEmptyFields(m, i)
+	}
+	for i, m := range p.Presence {
+		p.updateInnerMessageEmptyFields(&m.Message, i)
+	}
+}
+
 func (m Message) String() string {
 	return fmt.Sprintf("<Message %q data=%v>", m.Name, m.Data)
 }
@@ -39,6 +61,8 @@ func unencodableDataErr(data interface{}) error {
 	return fmt.Errorf("message data type %T must be string, []byte, or a value that can be encoded as a JSON object or array", data)
 }
 
+//withEncodedData - Used to encode string, binary([]byte) or json data (TM3).
+//Updates/Mutates Message.Data and Message.Encoding
 func (m Message) withEncodedData(cipher channelCipher) (Message, error) {
 	if m.Data == nil {
 		return m, nil
@@ -96,9 +120,8 @@ func (m Message) withEncodedData(cipher channelCipher) (Message, error) {
 	return m, nil
 }
 
+//withDecodedData - Used to decode received encoded data into string, binary([]byte) or json (TM3).
 func (m Message) withDecodedData(cipher channelCipher) (Message, error) {
-	// TODO: Unexport once proto gets merged into package ably.
-
 	// strings.Split on empty string returns []string{""}
 	if m.Data == nil || m.Encoding == "" {
 		return m, nil
diff --git a/ably/realtime_conn.go b/ably/realtime_conn.go
index a1d102d..99d7be1 100644
--- a/ably/realtime_conn.go
+++ b/ably/realtime_conn.go
@@ -711,6 +711,7 @@ func (c *Connection) eventloop() {
 			return
 		}
 		lastActivityAt = c.opts.Now()
+		msg.updateInnerMessagesEmptyFields() // TM2a, TM2c, TM2f
 		if msg.ConnectionSerial != 0 {
 			c.mtx.Lock()
 			c.setSerial(&msg.ConnectionSerial)
diff --git a/ably/rest_channel.go b/ably/rest_channel.go
index c8a0747..3579a6d 100644
--- a/ably/rest_channel.go
+++ b/ably/rest_channel.go
@@ -236,20 +236,6 @@ func (p *MessagesPaginatedResult) Next(ctx context.Context) bool {
 	return p.next(ctx, &p.items)
 }
 
-// IsLast returns true if the page is last page.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *MessagesPaginatedResult) IsLast(ctx context.Context) bool {
-	return !p.HasNext(ctx)
-}
-
-// HasNext returns true is there are more pages available.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *MessagesPaginatedResult) HasNext(ctx context.Context) bool {
-	return p.nextLink != ""
-}
-
 // Items returns the current page of results.
 //
 // See the "Paginated results" section in the package-level documentation.
diff --git a/ably/rest_client.go b/ably/rest_client.go
index 564724a..add5cb4 100644
--- a/ably/rest_client.go
+++ b/ably/rest_client.go
@@ -244,20 +244,6 @@ func (p *StatsPaginatedResult) Next(ctx context.Context) bool {
 	return p.next(ctx, &p.items)
 }
 
-// IsLast returns true if the page is last page.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *StatsPaginatedResult) IsLast(ctx context.Context) bool {
-	return !p.HasNext(ctx)
-}
-
-// HasNext returns true is there are more pages available.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *StatsPaginatedResult) HasNext(ctx context.Context) bool {
-	return p.nextLink != ""
-}
-
 // Items returns the current page of results.
 //
 // See the "Paginated results" section in the package-level documentation.
@@ -441,20 +427,6 @@ func (p *HTTPPaginatedResponse) Next(ctx context.Context) bool {
 	return p.next(ctx, &p.items)
 }
 
-// IsLast returns true if the page is last page.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *HTTPPaginatedResponse) IsLast(ctx context.Context) bool {
-	return !p.HasNext(ctx)
-}
-
-// HasNext returns true is there are more pages available.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *HTTPPaginatedResponse) HasNext(ctx context.Context) bool {
-	return p.nextLink != ""
-}
-
 // Items unmarshals the current page of results as JSON into the provided
 // variable.
 //
@@ -498,20 +470,6 @@ func (p *RESTPaginatedItems) Next(ctx context.Context) bool {
 	return true
 }
 
-// IsLast returns true if the page is last page.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *RESTPaginatedItems) IsLast(ctx context.Context) bool {
-	return !p.HasNext(ctx)
-}
-
-// HasNext returns true is there are more pages available.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *RESTPaginatedItems) HasNext(ctx context.Context) bool {
-	return p.nextLink != ""
-}
-
 // Item unmarshal the current result as JSON into the provided variable.
 //
 // See the "Paginated results" section in the package-level documentation.
diff --git a/ably/rest_presence.go b/ably/rest_presence.go
index 715f8d8..bc0a6ba 100644
--- a/ably/rest_presence.go
+++ b/ably/rest_presence.go
@@ -140,17 +140,6 @@ func (p *PresencePaginatedResult) Next(ctx context.Context) bool {
 	return p.next(ctx, p.decoder(&p.items))
 }
 
-// IsLast returns true if the page is last page.
-//
-// See "Paginated results" section in the package-level documentation.
-func (p *PresencePaginatedResult) IsLast(ctx context.Context) bool {
-	return !p.HasNext(ctx)
-}
-
-func (p *PresencePaginatedResult) HasNext(ctx context.Context) bool {
-	return p.nextLink != ""
-}
-
 // Items returns the current page of results.
 //
 // See the "Paginated results" section in the package-level documentation.
diff --git a/go.mod b/go.mod
index 97ff2d6..8d361bc 100644
--- a/go.mod
+++ b/go.mod
@@ -5,12 +5,7 @@ require (
 	github.com/ugorji/go/codec v1.1.7
 	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
 	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
+	golang.org/x/tools v0.1.8 // indirect
 )
 
-require (
-	github.com/davecgh/go-spew v1.1.0 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
-)
-
-go 1.17
+go 1.16
diff --git a/go.sum b/go.sum
index fa9b8ee..2a660ef 100644
--- a/go.sum
+++ b/go.sum
@@ -9,15 +9,40 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/yuin/goldmark v1.4.1 h1:/vn0k+RBvwlxEmP5E7SZMqNxPhfMVFEJiykr15/0XKM=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM=
+golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
+golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=


Test patch:
diff --git a/ably/auth_internal_test.go b/ably/auth_internal_test.go
index cfa1505..79f783c 100644
--- a/ably/auth_internal_test.go
+++ b/ably/auth_internal_test.go
@@ -4,8 +4,10 @@
 package ably
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"log"
 	"net/http"
 	"net/url"
 	"testing"
@@ -13,6 +15,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var (
+	buffer     bytes.Buffer
+	mocklogger = log.New(&buffer, "logger: ", log.Lshortfile)
+)
+
 func TestAddParams(t *testing.T) {
 	tests := map[string]struct {
 		leftHandSide   url.Values
diff --git a/ably/doc_test.go b/ably/doc_test.go
index 6437163..7a7670b 100644
--- a/ably/doc_test.go
+++ b/ably/doc_test.go
@@ -1,46 +1,8 @@
 package ably_test
 
-import (
-	"fmt"
-
-	"context"
-	"github.com/ably/ably-go/ably"
-)
+import "fmt"
 
 func Example_paginatedResults() {
-	ctx := context.Background()
-	client, err := ably.NewRealtime(ably.WithKey("xxx:xxx"))
-	if err != nil {
-		panic(err)
-	}
-	channel := client.Channels.Get("persisted:test")
-
-	err = channel.Publish(ctx, "EventName1", "EventData1")
-	if err != nil {
-		panic(err)
-	}
-
-	pages, err := channel.History().Pages(ctx)
-	if err != nil {
-		panic(err)
-	}
-
-	// Returning and iterating over the first page
-	pages.Next(ctx)
-	pages.First(ctx)
-	for _, message := range pages.Items() {
-		fmt.Println(message)
-	}
-
-	// Iteration over pages in PaginatedResult
-	for pages.Next(ctx) {
-		fmt.Println(pages.HasNext(ctx))
-		fmt.Println(pages.IsLast(ctx))
-		for _, presence := range pages.Items() {
-			fmt.Println(presence)
-		}
-	}
-	if err := pages.Err(); err != nil {
-		panic(err)
-	}
+	fmt.Println("TODO once the Requests method is adapted")
+	// Output: TODO once the Requests method is adapted
 }
diff --git a/ably/export_test.go b/ably/export_test.go
index 6475815..2668fb9 100644
--- a/ably/export_test.go
+++ b/ably/export_test.go
@@ -248,6 +248,10 @@ func MessageWithDecodedData(m Message, cipher channelCipher) (Message, error) {
 	return m.withDecodedData(cipher)
 }
 
+func (p *ProtocolMessage) UpdateEmptyFields() {
+	p.updateInnerMessagesEmptyFields()
+}
+
 func ChannelModeFromFlag(flags ProtoFlag) []ChannelMode {
 	return channelModeFromFlag(flags)
 }
diff --git a/ably/mock_test.go b/ably/mock_test.go
deleted file mode 100644
index 1c6ffdf..0000000
--- a/ably/mock_test.go
+++ /dev/null
@@ -1,32 +0,0 @@
-// mocks and helpers for unit tests.
-
-package ably
-
-import (
-	"bytes"
-	"log"
-)
-
-var (
-	buffer     bytes.Buffer
-	mocklogger = log.New(&buffer, "logger: ", log.Lshortfile)
-)
-
-// mockChannelWithState is a test helper that returns a mock channel in a specified state
-func mockChannelWithState(channelState *ChannelState, connectionState *ConnectionState) *RealtimeChannel {
-	mockChannel := RealtimeChannel{
-		client: &Realtime{
-			rest: &REST{
-				log: logger{l: &stdLogger{mocklogger}},
-			},
-			Connection: &Connection{},
-		},
-	}
-	if channelState != nil {
-		mockChannel.state = *channelState
-	}
-	if connectionState != nil {
-		mockChannel.client.Connection.state = *connectionState
-	}
-	return &mockChannel
-}
diff --git a/ably/proto_protocol_message_test.go b/ably/proto_protocol_message_test.go
index ef6c4d2..8783c4c 100644
--- a/ably/proto_protocol_message_test.go
+++ b/ably/proto_protocol_message_test.go
@@ -5,6 +5,7 @@ package ably_test
 
 import (
 	"bytes"
+	"strconv"
 	"testing"
 
 	"github.com/ably/ably-go/ably"
@@ -29,6 +30,82 @@ func TestProtocolMessageEncodeZeroSerials(t *testing.T) {
 		"unexpected msgpack encoding\nexpected: %x\nactual:   %x", expected, encoded)
 }
 
+func TestUpdateEmptyMessageFields_TM2a_TM2c_TM2f(t *testing.T) {
+	messages := []*ably.Message{
+		{
+			ID:           "",
+			ConnectionID: "",
+			Timestamp:    0,
+		},
+		{
+			ID:           "",
+			ConnectionID: "",
+			Timestamp:    0,
+		},
+		{
+			ID:           "",
+			ConnectionID: "",
+			Timestamp:    0,
+		},
+	}
+
+	presenceMessages := []*ably.PresenceMessage{
+		{
+			Message: ably.Message{
+				ID:           "",
+				ConnectionID: "",
+				Timestamp:    0,
+			},
+			Action: 0,
+		},
+		{
+			Message: ably.Message{
+				ID:           "",
+				ConnectionID: "",
+				Timestamp:    0,
+			},
+			Action: 0,
+		},
+		{
+			Message: ably.Message{
+				ID:           "",
+				ConnectionID: "",
+				Timestamp:    0,
+			},
+			Action: 0,
+		},
+		{
+			Message: ably.Message{
+				ID:           "",
+				ConnectionID: "",
+				Timestamp:    0,
+			},
+			Action: 0,
+		},
+	}
+	protoMsg := ably.ProtocolMessage{
+		Messages:     messages,
+		Presence:     presenceMessages,
+		ID:           "msg-id",
+		ConnectionID: "conn-id",
+		Timestamp:    3453,
+	}
+
+	protoMsg.UpdateEmptyFields()
+
+	for msgIndex, msg := range protoMsg.Messages {
+		assert.Equal(t, protoMsg.ID+":"+strconv.Itoa(msgIndex), msg.ID)
+		assert.Equal(t, protoMsg.ConnectionID, msg.ConnectionID)
+		assert.Equal(t, protoMsg.Timestamp, msg.Timestamp)
+	}
+
+	for presenceMsgIndex, presenceMessage := range protoMsg.Presence {
+		assert.Equal(t, protoMsg.ID+":"+strconv.Itoa(presenceMsgIndex), presenceMessage.ID)
+		assert.Equal(t, protoMsg.ConnectionID, presenceMessage.ConnectionID)
+		assert.Equal(t, protoMsg.Timestamp, presenceMessage.Timestamp)
+	}
+}
+
 func TestIfFlagIsSet(t *testing.T) {
 	flags := ably.FlagAttachResume
 	flags.Set(ably.FlagPresence)
diff --git a/ably/realtime_channel_spec_integration_test.go b/ably/realtime_channel_spec_integration_test.go
index 8951af3..262e133 100644
--- a/ably/realtime_channel_spec_integration_test.go
+++ b/ably/realtime_channel_spec_integration_test.go
@@ -2774,6 +2774,78 @@ func TestRealtimeChannel_RTL17_IgnoreMessagesWhenNotAttached(t *testing.T) {
 	})
 }
 
+func Test_UpdateEmptyMessageFields_TM2a_TM2c_TM2f(t *testing.T) {
+	const channelRetryTimeout = 123 * time.Millisecond
+	setup := func(t *testing.T) (
+		in, out chan *ably.ProtocolMessage,
+		msg chan *ably.Message,
+		channel *ably.RealtimeChannel,
+	) {
+		in = make(chan *ably.ProtocolMessage, 1)
+		out = make(chan *ably.ProtocolMessage, 16)
+		msg = make(chan *ably.Message, 1)
+
+		c, _ := ably.NewRealtime(
+			ably.WithToken("fake:token"),
+			ably.WithAutoConnect(false),
+			ably.WithChannelRetryTimeout(channelRetryTimeout),
+			ably.WithDial(MessagePipe(in, out)),
+		)
+
+		in <- &ably.ProtocolMessage{
+			Action:            ably.ActionConnected,
+			ConnectionID:      "connection-id",
+			ConnectionDetails: &ably.ConnectionDetails{},
+		}
+
+		err := ablytest.Wait(ablytest.ConnWaiter(c, c.Connect, ably.ConnectionEventConnected), nil)
+		assert.NoError(t, err)
+
+		channel = c.Channels.Get("test")
+
+		stateChanges := make(ably.ChannelStateChanges, 10)
+		channel.OnAll(stateChanges.Receive)
+
+		channel.Attach(canceledCtx)
+		ablytest.Instantly.Recv(t, nil, stateChanges, t.Fatalf) // Consume ATTACHING
+		in <- &ably.ProtocolMessage{
+			Action:  ably.ActionAttached,
+			Channel: channel.Name,
+		}
+		ablytest.Instantly.Recv(t, nil, stateChanges, t.Fatalf) // Consume ATTACHED
+		return
+	}
+
+	in, _, msg, channel := setup(t)
+
+	channel.SubscribeAll(context.Background(), func(message *ably.Message) {
+		msg <- message
+	})
+
+	// receive message when state is ATTACHED
+	protoMsg := &ably.ProtocolMessage{
+		Action:        ably.ActionMessage,
+		Channel:       channel.Name,
+		MsgSerial:     3,
+		ChannelSerial: "channelSerial",
+		ID:            "protoId",
+		ConnectionID:  "protoConnectionId",
+		Timestamp:     23454,
+		Messages: []*ably.Message{{
+			ID:           "",
+			ConnectionID: "",
+			Timestamp:    0,
+		}},
+	}
+	in <- protoMsg
+
+	var message *ably.Message
+	ablytest.Instantly.Recv(t, &message, msg, t.Fatalf)
+	assert.Equal(t, protoMsg.ID+":0", message.ID)
+	assert.Equal(t, protoMsg.ConnectionID, message.ConnectionID)
+	assert.Equal(t, protoMsg.Timestamp, message.Timestamp)
+}
+
 func TestRealtimeChannel_RTL14_HandleChannelError(t *testing.T) {
 
 	const channelRetryTimeout = 123 * time.Millisecond
diff --git a/ably/realtime_presence_internal_test.go b/ably/realtime_presence_internal_test.go
deleted file mode 100644
index ad112fd..0000000
--- a/ably/realtime_presence_internal_test.go
+++ /dev/null
@@ -1,89 +0,0 @@
-//go:build !integration
-// +build !integration
-
-package ably
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestVerifyChanState(t *testing.T) {
-	tests := map[string]struct {
-		channel     *RealtimeChannel
-		expectedErr error
-	}{
-		`No error if the channel is in state: "INITIALIZED"`: {
-			channel:     mockChannelWithState(&ChannelStateInitialized, nil),
-			expectedErr: nil,
-		},
-		`No error if the channel is in state: "ATTACHING"`: {
-			channel:     mockChannelWithState(&ChannelStateAttaching, nil),
-			expectedErr: nil,
-		},
-		`No error if the channel is in state: "ATTACHED"`: {
-			channel:     mockChannelWithState(&ChannelStateAttached, nil),
-			expectedErr: nil,
-		},
-		`No error if the channel is in state: "SUSPENDED"`: {
-			channel:     mockChannelWithState(&ChannelStateSuspended, nil),
-			expectedErr: nil,
-		},
-		`Error if the channel is in state: "DETACHING"`: {
-			channel:     mockChannelWithState(&ChannelStateDetaching, nil),
-			expectedErr: newError(91001, errors.New("unable to enter presence channel (invalid channel state: DETACHING)")),
-		},
-		`Error if the channel is in state: "DETACHED"`: {
-			channel:     mockChannelWithState(&ChannelStateDetached, nil),
-			expectedErr: newError(91001, errors.New("unable to enter presence channel (invalid channel state: DETACHED)")),
-		},
-		`Error if the channel is in state: "FAILED"`: {
-			channel:     mockChannelWithState(&ChannelStateFailed, nil),
-			expectedErr: newError(91001, errors.New("unable to enter presence channel (invalid channel state: FAILED)")),
-		},
-	}
-
-	for testName, test := range tests {
-		t.Run(testName, func(t *testing.T) {
-			presence := newRealtimePresence(test.channel)
-			err := presence.verifyChanState()
-			assert.Equal(t, test.expectedErr, err)
-		})
-	}
-}
-
-func TestSend(t *testing.T) {
-	tests := map[string]struct {
-		channel        *RealtimeChannel
-		msg            PresenceMessage
-		expectedResult result
-		expectedErr    error
-	}{
-		`No error sending presence if the channel is in state: "ATTACHED"`: {
-			channel: mockChannelWithState(&ChannelStateAttached, nil),
-			msg: PresenceMessage{
-				Message: Message{Name: "Hello"},
-				Action:  PresenceActionEnter,
-			},
-			expectedErr: nil,
-		},
-		`Error if channel is: "DETACHED" and connection is :"CLOSED"`: {
-			channel: mockChannelWithState(&ChannelStateDetached, &ConnectionStateClosed),
-			msg: PresenceMessage{
-				Message: Message{Name: "Hello"},
-				Action:  PresenceActionEnter,
-			},
-			expectedErr: newError(80000, errors.New("cannot Attach channel because connection is in CLOSED state")),
-		},
-	}
-
-	for testName, test := range tests {
-		t.Run(testName, func(t *testing.T) {
-			presence := newRealtimePresence(test.channel)
-			_, err := presence.send(&test.msg)
-			assert.Equal(t, test.expectedErr, err)
-		})
-	}
-}
diff --git a/ablytest/pagination.go b/ablytest/pagination.go
index 244afc0..acd8dcd 100644
--- a/ablytest/pagination.go
+++ b/ablytest/pagination.go
@@ -73,11 +73,7 @@ func testPagination(request reflect.Value, expectedItems []interface{}, perPage
 		var gotItems []interface{}
 		pageNum := 1
 		expectedFullPages := len(expectedItems) / perPage
-		pageMap := make(map[int]map[string]bool, expectedFullPages)
 		for pages.next() {
-			pageMap[pageNum] = map[string]bool{}
-			pageMap[pageNum]["hasNext"] = pages.hasNext()
-			pageMap[pageNum]["isLast"] = pages.isLast()
 			if (pageNum <= expectedFullPages && len(pages.items()) != perPage) ||
 				(pageNum > expectedFullPages && len(pages.items()) >= perPage) {
 				return fmt.Errorf("page #%d got %d items, expected at most %d", pageNum, len(pages.items()), perPage)
@@ -85,20 +81,6 @@ func testPagination(request reflect.Value, expectedItems []interface{}, perPage
 			gotItems = append(gotItems, pages.items()...)
 			pageNum++
 		}
-
-		lastPage := pageNum - 1
-		for idx, page := range pageMap {
-			if idx == lastPage {
-				if page["hasNext"] || !page["isLast"] {
-					return fmt.Errorf("page #%d got incorrect HasNext and IsLast boolean values", idx)
-				}
-			} else {
-				if !page["hasNext"] || page["isLast"] {
-					return fmt.Errorf("page #%d got incorrect HasNext and IsLast boolean values", idx)
-				}
-			}
-		}
-
 		if err := pages.err(); err != nil {
 			return fmt.Errorf("iterating pages: %w", err)
 		}
@@ -142,11 +124,9 @@ func testPagination(request reflect.Value, expectedItems []interface{}, perPage
 }
 
 type paginated struct {
-	next    func() bool
-	first   func() error
-	err     func() error
-	hasNext func() bool
-	isLast  func() bool
+	next  func() bool
+	first func() error
+	err   func() error
 }
 
 type paginatedResult struct {
@@ -181,12 +161,6 @@ func generalizePaginatedRequest(request reflect.Value) (
 				err, _ := r.MethodByName("First").Call([]reflect.Value{ctx})[0].Interface().(error)
 				return err
 			},
-			hasNext: func() bool {
-				return r.MethodByName("HasNext").Call([]reflect.Value{ctx})[0].Bool()
-			},
-			isLast: func() bool {
-				return r.MethodByName("IsLast").Call([]reflect.Value{ctx})[0].Bool()
-			},
 			err: func() error {
 				err, _ := r.MethodByName("Err").Call(nil)[0].Interface().(error)
 				return err